From 30ac1b674bf022f282a783be3cd075af955b8863 Mon Sep 17 00:00:00 2001 From: Openclaw Date: Fri, 25 Sep 2026 08:52:53 +0200 Subject: [PATCH 1/9] feat(anomaly): add experimental offline traffic-anomaly detector library Adds internal/anomaly, an encryption-independent anomaly detector for normalized MeshCore transmissions. It is inactive: nothing in CoreScope imports it, and there is no ingestor, server, API, UI, database, config or notification wiring. It has no default thresholds; New rejects the zero config. - Normalizer: observations -> final Events with an explicit settle policy; late observations are rejected, and capacity cuts mark the event Truncated. - Stream keys: channel + route class + first hop with its width. Route groups use flood relays only; the full route is separate evidence. - Detector (Observe/Advance/Flush/Snapshot, data time only): dedup with an explicit horizon, a reorder buffer, and multi-scale rate windows at stream, channel, route-group and global scope. - Also: new-stream volume with left censoring, and causal periodicity with harmonic handling and a union-bound Poisson null. - An episode state machine with cooldown, stable IDs and reason codes. - Every piece of state is capped by Limits, and coverage loss is reported. DecidedAt comes from a monotone decision clock. Co-Authored-By: Claude Opus 5.5 --- internal/anomaly/bounds_test.go | 286 +++++++ internal/anomaly/candidate.go | 249 +++++++ internal/anomaly/config.go | 372 +++++++++ internal/anomaly/contract_test.go | 446 +++++++++++ internal/anomaly/detector.go | 912 +++++++++++++++++++++++ internal/anomaly/doc.go | 119 +++ internal/anomaly/event.go | 340 +++++++++ internal/anomaly/go.mod | 3 + internal/anomaly/helpers_test.go | 142 ++++ internal/anomaly/key.go | 367 +++++++++ internal/anomaly/key_test.go | 146 ++++ internal/anomaly/newstream.go | 60 ++ internal/anomaly/newstream_state_test.go | 271 +++++++ internal/anomaly/normalizer.go | 412 ++++++++++ internal/anomaly/normalizer_test.go | 306 ++++++++ internal/anomaly/periodic.go | 411 ++++++++++ internal/anomaly/periodic_test.go | 363 +++++++++ internal/anomaly/rate_test.go | 296 ++++++++ internal/anomaly/state.go | 150 ++++ internal/anomaly/types.go | 310 ++++++++ internal/anomaly/window.go | 130 ++++ 21 files changed, 6091 insertions(+) create mode 100644 internal/anomaly/bounds_test.go create mode 100644 internal/anomaly/candidate.go create mode 100644 internal/anomaly/config.go create mode 100644 internal/anomaly/contract_test.go create mode 100644 internal/anomaly/detector.go create mode 100644 internal/anomaly/doc.go create mode 100644 internal/anomaly/event.go create mode 100644 internal/anomaly/go.mod create mode 100644 internal/anomaly/helpers_test.go create mode 100644 internal/anomaly/key.go create mode 100644 internal/anomaly/key_test.go create mode 100644 internal/anomaly/newstream.go create mode 100644 internal/anomaly/newstream_state_test.go create mode 100644 internal/anomaly/normalizer.go create mode 100644 internal/anomaly/normalizer_test.go create mode 100644 internal/anomaly/periodic.go create mode 100644 internal/anomaly/periodic_test.go create mode 100644 internal/anomaly/rate_test.go create mode 100644 internal/anomaly/state.go create mode 100644 internal/anomaly/types.go create mode 100644 internal/anomaly/window.go diff --git a/internal/anomaly/bounds_test.go b/internal/anomaly/bounds_test.go new file mode 100644 index 000000000..32cd7788a --- /dev/null +++ b/internal/anomaly/bounds_test.go @@ -0,0 +1,286 @@ +package anomaly + +import ( + "encoding/json" + "errors" + "fmt" + "math/rand" + "reflect" + "regexp" + "strings" + "testing" + "time" +) + +func TestZeroValueConfigIsRejected(t *testing.T) { + if _, err := New(Config{}); err == nil { + t.Fatal("zero-value Config accepted") + } + cfg := Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}} + if _, err := New(cfg); !errors.Is(err, errNoRules) { + t.Fatalf("a config without rules must be rejected explicitly, got %v", err) + } + cfg.Rate = []RateRule{{Name: "r", Scope: ScopeGlobal, Window: time.Minute}} + if _, err := New(cfg); err == nil { + t.Fatal("a rule without threshold/state accepted") + } + cfg.Rate = []RateRule{rateRule("r", ScopeGlobal, time.Minute, 5)} + cfg.Expected.Mode = 0 + if _, err := New(cfg); err == nil { + t.Fatal("unset Expected.Mode accepted") + } + if _, err := NewNormalizer(NormalizerConfig{}); err == nil { + t.Fatal("zero-value NormalizerConfig accepted") + } + // DedupHorizon must cover the longest window + cfg.Expected.Mode = ExpectedInclude + cfg.Limits.DedupHorizon = 30 * time.Second + if _, err := New(cfg); err == nil { + t.Fatal("dedup horizon shorter than the window accepted") + } +} + +func TestKeyCapacityIsBoundedAndReported(t *testing.T) { + lim := experimentalLimits() + lim.MaxKeysPerScope = 50 + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 3)}}) + // an attacker mints 1000 unique route keys + for i := 0; i < 1000; i++ { + d.Observe(relayEvent(t, fmt.Sprint(i), at(time.Duration(i)*time.Millisecond), byte(i), byte(i>>8), byte(i), 0x01)) + if n := len(d.scopes[ScopeStream].m); n > lim.MaxKeysPerScope { + t.Fatalf("keys %d > cap", n) + } + } + s := d.Snapshot() + if s.CapacityEvictions[ScopeStream] != 950 || !s.CoverageReduced() { + t.Fatalf("capacity evictions not reported: %+v", s.CapacityEvictions) + } + // candidates in a scope under capacity pressure carry the flag + cs := run(t, d, series(t, "hot", at(2*time.Second), 3, time.Second, nil, 0x99, 0x98)) + a := filter(cs, "r", StateActive) + if len(a) != 1 || a[0].Coverage&CoverageKeyEvictions == 0 { + t.Fatalf("capacity pressure not flagged: %+v", a) + } +} + +func TestEvictingAnOpenEpisodeClosesItExplicitly(t *testing.T) { + lim := experimentalLimits() + lim.MaxKeysPerScope = 1 + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 1)}}) + cs := run(t, d, []Event{relayEvent(t, "a", t0, 1, 2), relayEvent(t, "b", at(time.Second), 3, 4)}) + var ev []Candidate + for _, c := range cs { + if c.Reason == ReasonEvicted { + ev = append(ev, c) + } + } + if len(ev) != 1 || ev[0].From != StateActive || ev[0].To != StateEnded || ev[0].Summary == nil { + t.Fatalf("evicted open episode must emit active->ended(evicted):\n%s", describe(cs)) + } + if d.Snapshot().EpisodesEvicted != 1 { + t.Fatal("evicted episodes must be counted") + } +} + +func TestIdleKeysExpireByTTL(t *testing.T) { + lim := experimentalLimits() + lim.StateTTL = time.Hour + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 5)}}) + run(t, d, []Event{relayEvent(t, "a", t0, 1, 2), relayEvent(t, "b", at(2*time.Hour), 3, 4)}) + s := d.Snapshot() + if s.Keys[ScopeStream] != 1 || s.TTLEvictions[ScopeStream] != 1 { + t.Fatalf("TTL: %+v %+v", s.Keys, s.TTLEvictions) + } +} + +func TestReorderWithinDelayGivesTheSameOutput(t *testing.T) { + lim := experimentalLimits() + lim.ReorderDelay = 30 * time.Second + cfg := Config{Limits: lim, + Rate: []RateRule{rateRule("r", ScopeStream, 2*time.Minute, 6), rateRule("g", ScopeGlobal, time.Minute, 4)}, + Periodic: []PeriodicRule{experimentalPeriodic()}} + var evs []Event + evs = append(evs, series(t, "p", t0, 14, 5*time.Minute, nil, 0x11, 0x22)...) + for i := 0; i < 40; i++ { + evs = append(evs, relayEvent(t, fmt.Sprintf("n%02d", i), at(time.Duration(i)*17*time.Second), byte(i%3), 0x33)) + } + sortEvents(evs) + // DecidedAt records when the call sequence made a decision possible, so + // it legitimately depends on arrival order; everything else must not. + withoutDecidedAt := func(cs []Candidate) []Candidate { + for i := range cs { + if cs[i].DecidedAt.Before(cs[i].At) { + t.Fatal("DecidedAt before At") + } + cs[i].DecidedAt = time.Time{} + } + return cs + } + want := withoutDecidedAt(run(t, newDet(t, cfg), evs)) + if len(want) == 0 { + t.Fatal("fixture should produce candidates") + } + rng := rand.New(rand.NewSource(3)) + for trial := 0; trial < 20; trial++ { + shuf := append([]Event(nil), evs...) + // local shuffles that keep every event within 20 s of its sorted slot + for i := 0; i+1 < len(shuf); i++ { + if rng.Intn(2) == 0 && shuf[i+1].Time.Sub(shuf[i].Time) < 20*time.Second { + shuf[i], shuf[i+1] = shuf[i+1], shuf[i] + } + } + got := withoutDecidedAt(run(t, newDet(t, cfg), shuf)) + if !reflect.DeepEqual(got, want) { + t.Fatalf("trial %d: arrival order within ReorderDelay changed the output", trial) + } + } +} + +func TestLateEventsAreRejectedNotMisplaced(t *testing.T) { + d := newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 2)}}) + d.Observe(relayEvent(t, "a", at(time.Minute), 1, 2)) + r := d.Observe(relayEvent(t, "old", t0, 1, 2)) + if r.Status != StatusLate || d.Snapshot().Late != 1 { + t.Fatalf("event behind the frontier: %v", r.Status) + } + if cs := run(t, d, nil); len(cs) != 0 { + t.Fatal("a late event must not be counted") + } +} + +func TestReorderOverflowIsForcedAndReported(t *testing.T) { + lim := experimentalLimits() + lim.ReorderDelay = time.Hour + lim.MaxReorderBuffer = 3 + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 50)}}) + var reduced bool + for i := 0; i < 10; i++ { + r := d.Observe(relayEvent(t, fmt.Sprint(i), at(time.Duration(i)*time.Second), 1, 2)) + reduced = reduced || r.CoverageReduced + if d.reorder.Len() > lim.MaxReorderBuffer { + t.Fatal("reorder buffer exceeded its cap") + } + } + if !reduced || d.Snapshot().ReorderForced != 7 { + t.Fatalf("forced releases: %d", d.Snapshot().ReorderForced) + } +} + +func TestCandidateBufferIsBoundedAndDrainable(t *testing.T) { + lim := experimentalLimits() + lim.MaxCandidatesPerCall = 2 + lim.MaxPendingCandidates = 5 + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 1)}}) + var got []Candidate + for i := 0; i < 4; i++ { // 4 streams x 2 transitions = 8 candidates + r := d.Observe(relayEvent(t, fmt.Sprint(i), t0, byte(i), 0x01)) + if len(r.Candidates) > lim.MaxCandidatesPerCall { + t.Fatal("per-call cap exceeded") + } + got = append(got, r.Candidates...) + } + for { + r := d.Drain() + got = append(got, r.Candidates...) + if !r.More { + break + } + } + s := d.Snapshot() + if s.CandidatesEmitted != 8 || int(s.CandidatesDropped)+len(got) != 8 || s.PendingCandidates != 0 { + t.Fatalf("emitted %d dropped %d returned %d", s.CandidatesEmitted, s.CandidatesDropped, len(got)) + } + // generous pending cap: nothing is dropped, everything comes out in order + lim.MaxPendingCandidates = 100 + d = newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 1)}}) + got = got[:0] + for i := 0; i < 4; i++ { + got = append(got, d.Observe(relayEvent(t, fmt.Sprint(i), t0, byte(i), 0x01)).Candidates...) + } + for r := d.Drain(); ; r = d.Drain() { + got = append(got, r.Candidates...) + if !r.More { + break + } + } + if len(got) != 8 || d.Snapshot().CandidatesDropped != 0 { + t.Fatalf("drain returned %d", len(got)) + } +} + +func TestOutputHoldsOnlyCallerValues(t *testing.T) { + // marker bytes the caller supplied; nothing else identifying may appear + e := relayEvent(t, "caller-tx-0001", t0, 0xC3, 0x7A, 0x5B) + evs := []Event{e} + for i := 1; i < 3; i++ { + x := e + x.ID = TxID(fmt.Sprintf("caller-tx-%04d", i+1)) + x.Time = at(time.Duration(i) * time.Second) + evs = append(evs, x) + } + cs := run(t, newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 3)}}), evs) + b, err := json.Marshal(cs) + if err != nil { + t.Fatal(err) + } + out := string(b) + for _, forbidden := range []string{"enc_", "#ping", "#wardriving", "sender", "observer", "pubkey"} { + if strings.Contains(strings.ToLower(out), strings.ToLower(forbidden)) { + t.Fatalf("output contains %q", forbidden) + } + } + // every string value is an enum/rule name, a time, the stream key, the + // caller's route, a caller ID or a 32-hex EventID + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + t.Fatal(err) + } + callerIDs := map[string]bool{"caller-tx-0001": true, "caller-tx-0002": true, "caller-tx-0003": true} + eventID := regexp.MustCompile(`^[0-9a-f]{32}$`) + enums := map[string]bool{"r": true, "rate": true, "stream": true, "normal": true, "suspicious": true, "active": true, + "ended": true, "threshold_crossed": true, "confirmed": true, "quiet": true, "route_observed": true, "unclassified": true} + var walk func(x interface{}) + walk = func(x interface{}) { + switch y := x.(type) { + case map[string]interface{}: + for _, z := range y { + walk(z) + } + case []interface{}: + for _, z := range y { + walk(z) + } + case string: + if _, err := time.Parse(time.RFC3339Nano, y); err == nil { + return + } + switch { + case enums[y], callerIDs[y], eventID.MatchString(y), + y == StreamKeyOf(e).String(), y == "2:7a5b0000": + default: + t.Fatalf("unexpected string value %q in output", y) + } + } + } + walk(v) +} + +func TestStaleDeadlinesStayBounded(t *testing.T) { + lim := experimentalLimits() + lim.MaxKeysPerScope = 4 + st := StateParams{EndAfterQuiet: 24 * time.Hour} + d := newDet(t, Config{Limits: lim, Rate: []RateRule{{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 1, State: st}}}) + // every event opens an episode on a new key and evicts an old one, leaving + // its deadline stale + for i := 0; i < 5000; i++ { + d.Observe(relayEvent(t, fmt.Sprint(i), at(time.Duration(i)*time.Second), byte(i), byte(i>>8), byte(i), 0x07)) + for r := d.Drain(); r.More; r = d.Drain() { + } + if d.deadlines.Len() > 4*d.liveEpisodes()+1024+1 { + t.Fatalf("deadline heap %d grows without bound (live %d)", d.deadlines.Len(), d.liveEpisodes()) + } + } + if d.Snapshot().EpisodesEvicted < 4000 { + t.Fatal("fixture should evict open episodes") + } +} diff --git a/internal/anomaly/candidate.go b/internal/anomaly/candidate.go new file mode 100644 index 000000000..9b3f60583 --- /dev/null +++ b/internal/anomaly/candidate.go @@ -0,0 +1,249 @@ +package anomaly + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "strconv" + "time" +) + +// Kind is the detector family that produced a candidate. +type Kind uint8 + +const ( + KindRate Kind = iota + 1 + KindNewStream + KindPeriodic +) + +var kindNames = [...]string{"", "rate", "new_stream", "periodic"} + +func (k Kind) String() string { + if int(k) < len(kindNames) && k > 0 { + return kindNames[k] + } + return "kind(" + strconv.Itoa(int(k)) + ")" +} + +// State is an episode state. +type State uint8 + +const ( + StateNormal State = iota + StateSuspicious + StateActive + StateEnded +) + +var stateNames = [...]string{"normal", "suspicious", "active", "ended"} + +func (s State) String() string { + if int(s) < len(stateNames) { + return stateNames[s] + } + return "state(" + strconv.Itoa(int(s)) + ")" +} + +// Reason is a machine-readable transition reason. +type Reason uint8 + +const ( + // ReasonThresholdCrossed: normal -> suspicious, the rule became hot. + ReasonThresholdCrossed Reason = iota + 1 + // ReasonConfirmed: suspicious -> active after State.ConfirmAfter. + ReasonConfirmed + // ReasonNotConfirmed: suspicious -> normal, quiet before confirmation. + ReasonNotConfirmed + // ReasonQuiet: active -> ended after State.EndAfterQuiet without a hot + // signal. + ReasonQuiet + // ReasonReopened: ended -> active, hot again within State.Cooldown. + ReasonReopened + // ReasonEvicted: the key's state was evicted by a capacity limit while + // the episode was open; the episode is closed without a quiet period. + ReasonEvicted +) + +var reasonNames = [...]string{"", "threshold_crossed", "confirmed", "not_confirmed", "quiet", "reopened", "evicted"} + +func (r Reason) String() string { + if int(r) < len(reasonNames) && r > 0 { + return reasonNames[r] + } + return "reason(" + strconv.Itoa(int(r)) + ")" +} + +// legalTransition is the only place that defines which transitions exist. +func legalTransition(from, to State, r Reason) bool { + switch { + case from == StateNormal && to == StateSuspicious && r == ReasonThresholdCrossed: + case from == StateSuspicious && to == StateActive && r == ReasonConfirmed: + case from == StateSuspicious && to == StateNormal && (r == ReasonNotConfirmed || r == ReasonEvicted): + case from == StateActive && to == StateEnded && (r == ReasonQuiet || r == ReasonEvicted): + case from == StateEnded && to == StateActive && r == ReasonReopened: + default: + return false + } + return true +} + +// CoverageFlags mark evidence that capacity limits may have cut. +type CoverageFlags uint16 + +const ( + // CoverageWindowSaturated: the key's window merged entries (counts may + // be overstated). + CoverageWindowSaturated CoverageFlags = 1 << iota + // CoverageKeyEvictions: the scope evicted keys for capacity within the + // last StateTTL, so a "new" key may have existed before. + CoverageKeyEvictions + // CoverageDedupEvictedEarly: dedup IDs were evicted before the horizon + // within the last DedupHorizon; a repeat may have been counted twice. + CoverageDedupEvictedEarly + // CoverageReorderForced: the reorder buffer overflowed recently; late + // events may have been rejected. + CoverageReorderForced + // CoverageInputTruncated: an event within the last DedupHorizon had + // Truncated set (its normalization dropped observations for capacity), + // so its first hop may rest on incomplete route evidence. + CoverageInputTruncated +) + +// PeriodAlternative is another period that explains a comparable chain. +type PeriodAlternative struct { + Period time.Duration + Support int + Coverage float64 +} + +// RateEvidence is attached to rate candidates. +type RateEvidence struct { + Window time.Duration + Threshold int + Count uint64 // events in (t-Window, t], including expected ones + Expected uint64 + Monitored uint64 + Saturated bool +} + +// NewStreamEvidence is attached to new-stream candidates. +type NewStreamEvidence struct { + Window time.Duration + MinCount int + Count int + Age time.Duration // time from the episode's first event + Censored bool // first seen within CensorWindow of HistoryStart: may predate the data + Reactivated bool // seen before, silent for QuietPeriod, active again +} + +// PeriodicEvidence is attached to periodic candidates. +type PeriodicEvidence struct { + // Period is the hypothesis period. It is fixed when the hypothesis is + // formed (a recent gap divided by k, replaced by the phase estimate + // chain span / periods spanned when that explains at least as much) and + // kept while it explains every new gap within Tolerance; it is not + // re-fitted on every pulse. + Period time.Duration + // Tolerance is the per-gap tolerance max(JitterAbs, JitterRel*Period). + Tolerance time.Duration + // JitterMedian is the median |gap - k*Period| over the chain. It is a + // gap measure: independent per-pulse jitter shows up roughly doubled. + JitterMedian time.Duration + Support int // pulses in the chain + Events int // events merged into those pulses + Coverage float64 + // Chance is an upper bound (union bound) on the expected number of + // equally long chains under a Poisson null with the key's local pulse + // rate, over all pulses evaluated on the key so far and every candidate + // period the search may try; one gap is discounted for fitting Period. + Chance float64 + SufficientAt time.Time // first time the evidence met every criterion + Alternatives []PeriodAlternative +} + +// EpisodeSummary is attached to ended / not-confirmed / evicted candidates. +type EpisodeSummary struct { + ActiveSince time.Time // zero if never active + LastHot time.Time + HotSignals uint32 +} + +// Candidate is one state transition of one episode. Only transitions are +// emitted: repeated hot signals inside an episode are counted, not emitted. +type Candidate struct { + // EventID is stable across replays: it depends only on the rule name, + // the key encoding and the episode start time. + EventID string + Rule string + Kind Kind + Key Key + From, To State + Reason Reason + At time.Time // data time of the transition + // DecidedAt is the detector's decision clock when the transition was + // made: the latest FinalAt of every event offered so far, or the latest + // Advance time (and >= At). It is therefore >= the FinalAt of every + // event behind the decision. When events are offered in the order they + // became final (as Normalizer releases them), it is the time a live + // detector would have made the decision. + DecidedAt time.Time + EpisodeStart time.Time + Confidence Confidence + Traffic TrafficLabel + Suppressed bool + Coverage CoverageFlags + // TriggerID is the caller's ID of the event that caused the transition + // (empty for time-driven transitions). + TriggerID TxID + // TriggerRoute is the full route of that event, as separate evidence. + // For channel and global scope it only describes the event that crossed + // the threshold; it is not an attribution of the anomaly. + TriggerRoute Route + Rate *RateEvidence `json:",omitempty"` + NewStream *NewStreamEvidence `json:",omitempty"` + Periodic *PeriodicEvidence `json:",omitempty"` + Summary *EpisodeSummary `json:",omitempty"` +} + +func episodeID(rule string, k Key, start int64) string { + h := sha256.New() + h.Write([]byte(rule)) + h.Write([]byte{0}) + h.Write([]byte(k.String())) + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(start)) + h.Write([]byte{0}) + h.Write(b[:]) + return hex.EncodeToString(h.Sum(nil)[:16]) +} + +// ---- deterministic JSON ---- + +func (k Kind) MarshalJSON() ([]byte, error) { return json.Marshal(k.String()) } +func (s State) MarshalJSON() ([]byte, error) { return json.Marshal(s.String()) } +func (r Reason) MarshalJSON() ([]byte, error) { return json.Marshal(r.String()) } +func (s Scope) MarshalJSON() ([]byte, error) { return json.Marshal(s.String()) } +func (c Confidence) MarshalJSON() ([]byte, error) { return json.Marshal(c.String()) } +func (l TrafficLabel) MarshalJSON() ([]byte, error) { return json.Marshal(l.String()) } +func (k Key) MarshalJSON() ([]byte, error) { return json.Marshal(k.String()) } + +// MarshalJSON encodes a route as ":" ("-" when empty). +func (r Route) MarshalJSON() ([]byte, error) { + if r.w == 0 { + return json.Marshal("-") + } + return json.Marshal(strconv.Itoa(int(r.w)) + ":" + hex.EncodeToString(r.hops)) +} + +// MarshalJSON lists the set flags by name. +func (f CoverageFlags) MarshalJSON() ([]byte, error) { + names := []string{} + for i, n := range []string{"window_saturated", "key_evictions", "dedup_evicted_early", "reorder_forced", "input_truncated"} { + if f&(1< 0), except +// ReorderDelay, where 0 means "events must arrive in data-time order". +type Limits struct { + // MaxKeysPerScope caps tracked keys per scope. When full, the least + // recently seen key without an open episode is evicted (counted). + MaxKeysPerScope int + // StateTTL evicts keys idle this long (data time). + StateTTL time.Duration + // DedupHorizon: an ID is guaranteed to be counted at most once if its + // repeat arrives within this data-time horizon (and MaxDedupIDs was not + // exhausted). Beyond it, a repeat is counted again. Unbounded dedup of + // arbitrary IDs cannot be combined with bounded memory. + DedupHorizon time.Duration + // MaxDedupIDs caps remembered IDs. Evicting an ID before DedupHorizon is + // counted and flags reduced coverage. + MaxDedupIDs int + // ReorderDelay is how long events are held so that events arriving out + // of data-time order within this delay are still processed in order. + // Events older than the processed frontier are rejected as late. With a + // delay > 0, events with equal times are processed in ID order, and one + // arriving after its tie was already processed (by Advance, Flush or a + // later event) is late if its ID sorts first; with 0, every event is + // processed on arrival and equal times keep arrival order. + ReorderDelay time.Duration + // MaxReorderBuffer caps held events; overflow force-releases the oldest. + MaxReorderBuffer int + // MaxWindowEntries caps distinct timestamps kept per key for rate + // windows. Overflow merges the two oldest entries (counts may be + // overstated until they expire) and is flagged on candidates. + MaxWindowEntries int + // MaxCandidatesPerCall caps candidates returned by one call; the rest + // wait in a buffer returned by later calls or Drain. + MaxCandidatesPerCall int + // MaxPendingCandidates caps that buffer; overflow drops the oldest + // candidate (counted, flagged). + MaxPendingCandidates int +} + +// ExpectedMode says how caller-labelled expected traffic is reported. +type ExpectedMode uint8 + +const ( + expectedModeUnset ExpectedMode = iota + // ExpectedInclude: expected traffic counts toward thresholds; candidates + // carry a TrafficLabel. + ExpectedInclude + // ExpectedSuppress: as Include, but candidates whose label is + // LabelExpected are marked Suppressed. They are still returned and all + // counters still include them; nothing is deleted. + ExpectedSuppress +) + +// ExpectedPolicy must be set explicitly. +type ExpectedPolicy struct { + Mode ExpectedMode +} + +// StateParams drive the per-episode state machine +// normal -> suspicious -> active -> ended. +type StateParams struct { + // ConfirmAfter: the rule must stay hot this long after it first fires + // before the episode becomes active (minimum duration). 0 confirms on + // the first signal. + ConfirmAfter time.Duration + // EndAfterQuiet: an episode with no hot signal for this long ends (or, + // if never confirmed, returns to normal). + EndAfterQuiet time.Duration + // Cooldown: a hot signal within this long after the end reopens the + // same episode (same EventID) instead of starting a new one. + Cooldown time.Duration +} + +// RateRule fires when a key's count in a sliding window (t-Window, t] +// reaches Threshold. +type RateRule struct { + Name string + Scope Scope + Window time.Duration + Threshold int + PayloadTypes []PayloadType // empty = all payload types + State StateParams +} + +// NewStreamRule fires when a key that was not seen before (or was silent for +// QuietPeriod) reaches MinCount events within Window of its first event. +type NewStreamRule struct { + Name string + Scope Scope + Window time.Duration + MinCount int + CensorWindow time.Duration + QuietPeriod time.Duration + PayloadTypes []PayloadType + State StateParams +} + +// PeriodicRule detects a regular pulse train on a key. All evaluation is +// causal: a signal at time t uses only events at or before t. +type PeriodicRule struct { + Name string + Scope Scope + PayloadTypes []PayloadType + // MinPeriod and MaxPeriod bound the fundamental period searched. + MinPeriod, MaxPeriod time.Duration + // BurstGap: events within this gap of the previous event belong to the + // same pulse (the pulse time is its first event). Must be < MinPeriod/2. + BurstGap time.Duration + // Tolerance for one gap: max(JitterAbs, JitterRel*period). + JitterAbs time.Duration + JitterRel float64 + // MaxMissing is the number of consecutive missing pulses one gap may + // bridge (a gap may span up to MaxMissing+1 periods). + MaxMissing int + // MinPulses is the minimum number of pulses in the current chain. + MinPulses int + // HistoryLen is the pulse ring size (bounds memory and CPU). + HistoryLen int + // MinCoverage: explained gaps / periods spanned, in (0,1]. + MinCoverage float64 + // MaxJitterFraction: median residual must be <= this fraction of the + // tolerance. + MaxJitterFraction float64 + // MaxChance bounds PeriodicEvidence.Chance: a union bound on the expected + // number of equally long chains under a Poisson null with the key's + // recent (local) pulse rate, over the period search and every pulse + // evaluated on the key. + MaxChance float64 + State StateParams +} + +const ( + maxWindow = 24 * time.Hour + maxHistoryLen = 256 + minHistoryLen = 8 + maxMissingLimit = 8 + maxRuleNameLen = 64 + maxKeysLimit = 10_000_000 +) + +var errNoRules = errors.New("anomaly: config enables no rule") + +func validName(s string) bool { + if len(s) == 0 || len(s) > maxRuleNameLen { + return false + } + for _, c := range s { + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_' || c == '-' || c == '.') { + return false + } + } + return true +} + +func (p StateParams) validate(name string) error { + if p.ConfirmAfter < 0 || p.Cooldown < 0 { + return fmt.Errorf("anomaly: rule %s: ConfirmAfter and Cooldown must be >= 0", name) + } + if p.EndAfterQuiet <= 0 { + return fmt.Errorf("anomaly: rule %s: State.EndAfterQuiet must be > 0", name) + } + if p.ConfirmAfter > maxWindow || p.EndAfterQuiet > maxWindow || p.Cooldown > 7*maxWindow { + return fmt.Errorf("anomaly: rule %s: state durations out of range", name) + } + return nil +} + +func validPayloadTypes(ts []PayloadType) bool { + for _, t := range ts { + if t > MaxPayloadType { + return false + } + } + return true +} + +func finiteIn(v, lo, hi float64) bool { return !math.IsNaN(v) && v >= lo && v <= hi } + +// Validate checks the whole config. +func (c *Config) Validate() error { + l := c.Limits + switch { + case l.MaxKeysPerScope <= 0 || l.MaxKeysPerScope > maxKeysLimit: + return errors.New("anomaly: Limits.MaxKeysPerScope must be in 1..10000000") + case l.StateTTL <= 0: + return errors.New("anomaly: Limits.StateTTL must be > 0") + case l.DedupHorizon <= 0: + return errors.New("anomaly: Limits.DedupHorizon must be > 0") + case l.MaxDedupIDs <= 0: + return errors.New("anomaly: Limits.MaxDedupIDs must be > 0") + case l.ReorderDelay < 0 || l.ReorderDelay > time.Hour: + return errors.New("anomaly: Limits.ReorderDelay must be in 0..1h") + case l.MaxReorderBuffer <= 0: + return errors.New("anomaly: Limits.MaxReorderBuffer must be > 0") + case l.MaxWindowEntries < 2: + return errors.New("anomaly: Limits.MaxWindowEntries must be >= 2") + case l.MaxCandidatesPerCall <= 0: + return errors.New("anomaly: Limits.MaxCandidatesPerCall must be > 0") + case l.MaxPendingCandidates < l.MaxCandidatesPerCall: + return errors.New("anomaly: Limits.MaxPendingCandidates must be >= MaxCandidatesPerCall") + } + switch c.Expected.Mode { + case ExpectedInclude, ExpectedSuppress: + default: + return errors.New("anomaly: Expected.Mode must be set explicitly") + } + if len(c.Rate)+len(c.NewStream)+len(c.Periodic) == 0 { + return errNoRules + } + if !c.HistoryStart.IsZero() && !validTime(c.HistoryStart) { + return errors.New("anomaly: HistoryStart out of range") + } + names := map[string]bool{} + checkName := func(n string) error { + if !validName(n) { + return errors.New("anomaly: rule names must be 1-64 chars of [a-z0-9_.-]") + } + if names[n] { + return fmt.Errorf("anomaly: duplicate rule name %s", n) + } + names[n] = true + return nil + } + var needTTL, maxRateWindow time.Duration + need := func(d time.Duration) { + if d > needTTL { + needTTL = d + } + } + for _, r := range c.Rate { + if err := checkName(r.Name); err != nil { + return err + } + if !r.Scope.valid() { + return fmt.Errorf("anomaly: rule %s: invalid scope", r.Name) + } + if r.Window < time.Second || r.Window > maxWindow { + return fmt.Errorf("anomaly: rule %s: Window must be in 1s..24h", r.Name) + } + if r.Threshold < 1 { + return fmt.Errorf("anomaly: rule %s: Threshold must be >= 1", r.Name) + } + if !validPayloadTypes(r.PayloadTypes) { + return fmt.Errorf("anomaly: rule %s: invalid payload type", r.Name) + } + if err := r.State.validate(r.Name); err != nil { + return err + } + if r.Window > maxRateWindow { + maxRateWindow = r.Window + } + need(r.Window) + need(r.State.EndAfterQuiet + r.State.Cooldown) + } + for _, r := range c.NewStream { + if err := checkName(r.Name); err != nil { + return err + } + if r.Scope != ScopeStream && r.Scope != ScopeChannel && r.Scope != ScopeRouteGroup { + return fmt.Errorf("anomaly: rule %s: new-stream scope must be stream, channel or route", r.Name) + } + if r.Window <= 0 || r.Window > maxWindow || r.MinCount < 1 { + return fmt.Errorf("anomaly: rule %s: Window must be in (0,24h] and MinCount >= 1", r.Name) + } + if r.CensorWindow <= 0 || r.QuietPeriod <= 0 { + return fmt.Errorf("anomaly: rule %s: CensorWindow and QuietPeriod must be > 0", r.Name) + } + // A key silent for less than QuietPeriod before the history start + // would still be a continuing stream; censoring must cover that. + if r.CensorWindow < r.QuietPeriod { + return fmt.Errorf("anomaly: rule %s: CensorWindow must be >= QuietPeriod", r.Name) + } + // The rule signals once per episode, so it could never be confirmed. + if r.State.ConfirmAfter != 0 { + return fmt.Errorf("anomaly: rule %s: new-stream rules signal once per episode; State.ConfirmAfter must be 0", r.Name) + } + if !validPayloadTypes(r.PayloadTypes) { + return fmt.Errorf("anomaly: rule %s: invalid payload type", r.Name) + } + if err := r.State.validate(r.Name); err != nil { + return err + } + need(r.Window) + need(r.QuietPeriod) + need(r.State.EndAfterQuiet + r.State.Cooldown) + } + for _, r := range c.Periodic { + if err := checkName(r.Name); err != nil { + return err + } + if r.Scope != ScopeStream && r.Scope != ScopeChannel { + return fmt.Errorf("anomaly: rule %s: periodic scope must be stream or channel", r.Name) + } + switch { + case r.MinPeriod <= 0 || r.MaxPeriod <= r.MinPeriod || r.MaxPeriod > maxWindow: + return fmt.Errorf("anomaly: rule %s: need 0 < MinPeriod < MaxPeriod <= 24h", r.Name) + case r.BurstGap < 0 || 2*r.BurstGap >= r.MinPeriod: + return fmt.Errorf("anomaly: rule %s: BurstGap must be in [0, MinPeriod/2)", r.Name) + case r.JitterAbs < 0 || !finiteIn(r.JitterRel, 0, 0.25) || (r.JitterAbs == 0 && r.JitterRel == 0): + return fmt.Errorf("anomaly: rule %s: need JitterAbs >= 0, JitterRel in [0,0.25], not both 0", r.Name) + case r.MaxMissing < 0 || r.MaxMissing > maxMissingLimit: + return fmt.Errorf("anomaly: rule %s: MaxMissing must be in 0..%d", r.Name, maxMissingLimit) + case r.HistoryLen < minHistoryLen || r.HistoryLen > maxHistoryLen: + return fmt.Errorf("anomaly: rule %s: HistoryLen must be in %d..%d", r.Name, minHistoryLen, maxHistoryLen) + case r.MinPulses < 4 || r.MinPulses > r.HistoryLen: + return fmt.Errorf("anomaly: rule %s: MinPulses must be in 4..HistoryLen", r.Name) + case !finiteIn(r.MinCoverage, 0, 1) || r.MinCoverage == 0: + return fmt.Errorf("anomaly: rule %s: MinCoverage must be in (0,1]", r.Name) + case !finiteIn(r.MaxJitterFraction, 0, 1) || r.MaxJitterFraction == 0: + return fmt.Errorf("anomaly: rule %s: MaxJitterFraction must be in (0,1]", r.Name) + case !finiteIn(r.MaxChance, 0, 1) || r.MaxChance == 0: + return fmt.Errorf("anomaly: rule %s: MaxChance must be in (0,1]", r.Name) + } + // A tolerance at or above half the period would let one gap match two + // multiples; refuse it. + if tolFor(r, r.MinPeriod) >= r.MinPeriod/2 { + return fmt.Errorf("anomaly: rule %s: tolerance must stay below MinPeriod/2", r.Name) + } + if !validPayloadTypes(r.PayloadTypes) { + return fmt.Errorf("anomaly: rule %s: invalid payload type", r.Name) + } + if err := r.State.validate(r.Name); err != nil { + return err + } + need(time.Duration(r.MaxMissing+1)*r.MaxPeriod + r.State.EndAfterQuiet) + need(r.State.EndAfterQuiet + r.State.Cooldown) + } + if l.StateTTL < needTTL { + return fmt.Errorf("anomaly: Limits.StateTTL must be >= %s (longest window, quiet period or episode memory)", needTTL) + } + if l.DedupHorizon < maxRateWindow+l.ReorderDelay { + return errors.New("anomaly: Limits.DedupHorizon must cover the longest rate window plus ReorderDelay") + } + return nil +} + +func tolFor(r PeriodicRule, period time.Duration) time.Duration { + rel := time.Duration(r.JitterRel * float64(period)) + if r.JitterAbs > rel { + return r.JitterAbs + } + return rel +} diff --git a/internal/anomaly/contract_test.go b/internal/anomaly/contract_test.go new file mode 100644 index 000000000..57254e5dd --- /dev/null +++ b/internal/anomaly/contract_test.go @@ -0,0 +1,446 @@ +package anomaly + +import ( + "math" + "testing" + "time" +) + +// Regression tests for the documented contracts: decision time, labels on +// every transition, route-group membership, tie order, config validation +// and normalization. + +// An event counted in a window that became final later than the trigger +// must hold the decision back: DecidedAt covers every event behind it. +func TestDecidedAtCoversNonTriggerEvidence(t *testing.T) { + lim := experimentalLimits() + lim.ReorderDelay = time.Minute + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeRouteGroup, time.Minute, 2)}}) + late := relayEvent(t, "a", t0, 1, 0x5c) + late.FinalAt = at(5 * time.Minute) // its first observation arrived late + trig := relayEvent(t, "b", at(time.Second), 1, 0x5c) + // offered in finalization order: b first, a later (within ReorderDelay) + var cs []Candidate + cs = append(cs, d.Observe(trig).Candidates...) + cs = append(cs, d.Observe(late).Candidates...) + cs = append(cs, d.Flush().Candidates...) + a := filter(cs, "r", StateActive) + if len(a) != 1 || a[0].TriggerID != "b" { + t.Fatalf("want one activation triggered by b:\n%s", describe(cs)) + } + if a[0].DecidedAt.Before(late.FinalAt) { + t.Fatalf("DecidedAt %v is before the FinalAt %v of evidence it used", a[0].DecidedAt.Sub(t0), late.FinalAt.Sub(t0)) + } +} + +// A time-driven end is never decided before the transitions that opened the +// episode, and never before deadline + ReorderDelay (the earliest time an +// Advance could release it). +func TestQuietEndIsNotDecidedBeforeItsEpisodeOrItsDeadline(t *testing.T) { + lim := experimentalLimits() + lim.ReorderDelay = 20 * time.Second + r := RateRule{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 1, State: StateParams{EndAfterQuiet: 10 * time.Second}} + d := newDet(t, Config{Limits: lim, Rate: []RateRule{r}}) + e := relayEvent(t, "a", t0, 1, 2) + e.FinalAt = at(60 * time.Second) + var cs []Candidate + cs = append(cs, d.Observe(e).Candidates...) + cs = append(cs, d.Advance(at(2*time.Hour)).Candidates...) + if len(cs) != 3 || cs[2].To != StateEnded { + t.Fatalf("want open + active + ended:\n%s", describe(cs)) + } + end := cs[2] + if !end.At.Equal(at(10 * time.Second)) { + t.Fatalf("quiet end At %v", end.At.Sub(t0)) + } + if end.DecidedAt.Before(cs[1].DecidedAt) || end.DecidedAt.Before(end.At.Add(lim.ReorderDelay)) { + t.Fatalf("quiet end DecidedAt %v, opened %v", end.DecidedAt.Sub(t0), cs[1].DecidedAt.Sub(t0)) + } + // Advance models continuously passing time: not stamped with the final t + if !end.DecidedAt.Equal(at(60 * time.Second)) { + t.Fatalf("quiet end DecidedAt %v, want max(FinalAt 60s, deadline+delay 30s)", end.DecidedAt.Sub(t0)) + } + // without a late FinalAt the deadline + ReorderDelay is what counts + d = newDet(t, Config{Limits: lim, Rate: []RateRule{r}}) + d.Observe(relayEvent(t, "a", t0, 1, 2)) + ends := filter(d.Advance(at(2*time.Hour)).Candidates, "r", StateEnded) + if len(ends) != 1 || !ends[0].DecidedAt.Equal(at(30*time.Second)) { + t.Fatalf("quiet end %+v, want DecidedAt = deadline 10s + ReorderDelay 20s", ends) + } +} + +func TestSuppressModeMarksEveryTransitionOfAnExpectedEpisode(t *testing.T) { + r := RateRule{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 2, + State: StateParams{ConfirmAfter: time.Minute, EndAfterQuiet: 5 * time.Minute}} + check := func(name string, cs []Candidate, wantTransitions int) { + t.Helper() + if len(cs) < wantTransitions { + t.Fatalf("%s: too few transitions:\n%s", name, describe(cs)) + } + for _, c := range cs { + if !c.Suppressed || c.Traffic != LabelExpected { + t.Fatalf("%s: %s->%s (%s) Suppressed=%v Traffic=%v", name, c.From, c.To, c.Reason, c.Suppressed, c.Traffic) + } + } + } + expected := func(evs []Event) []Event { + for i := range evs { + evs[i].Traffic = TrafficExpected + } + return evs + } + // opened, confirmed, ended by quiet + d := newDet(t, Config{Rate: []RateRule{r}, Expected: ExpectedPolicy{Mode: ExpectedSuppress}}) + cs := run(t, d, expected(series(t, "e", t0, 12, 10*time.Second, nil, 1, 2))) + cs = append(cs, d.Advance(at(time.Hour)).Candidates...) + check("quiet", cs, 3) + // opened, not confirmed + d = newDet(t, Config{Rate: []RateRule{r}, Expected: ExpectedPolicy{Mode: ExpectedSuppress}}) + cs = run(t, d, expected(series(t, "n", t0, 2, 10*time.Second, nil, 1, 2))) + cs = append(cs, d.Advance(at(time.Hour)).Candidates...) + check("not_confirmed", cs, 2) + // opened, evicted for capacity: also flagged as a key eviction + lim := experimentalLimits() + lim.MaxKeysPerScope = 1 + d = newDet(t, Config{Limits: lim, Rate: []RateRule{r}, Expected: ExpectedPolicy{Mode: ExpectedSuppress}}) + cs = run(t, d, expected(append(series(t, "x", t0, 2, time.Second, nil, 1, 2), relayEvent(t, "other", at(3*time.Second), 1, 3)))) + var ev []Candidate + for _, c := range cs { + if c.Reason == ReasonEvicted { + ev = append(ev, c) + } + } + if len(ev) != 1 || ev[0].Coverage&CoverageKeyEvictions == 0 { + t.Fatalf("capacity eviction must carry key_evictions:\n%s", describe(cs)) + } + check("evicted", cs, 2) +} + +func TestTimeDrivenTransitionsKeepTheEpisodeCoverage(t *testing.T) { + lim := experimentalLimits() + lim.MaxDedupIDs = 1 // every new ID evicts the previous one early + r := RateRule{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 2, State: StateParams{EndAfterQuiet: 5 * time.Minute}} + d := newDet(t, Config{Limits: lim, Rate: []RateRule{r}}) + cs := run(t, d, series(t, "c", t0, 3, 10*time.Second, nil, 1, 2)) + cs = append(cs, d.Advance(at(time.Hour)).Candidates...) + end := filter(cs, "r", StateEnded) + if len(end) != 1 || end[0].Coverage&CoverageDedupEvictedEarly == 0 { + t.Fatalf("ended candidate lost the episode's coverage flags: %+v", end) + } +} + +func TestMixedAndNonFloodRelayEventsNeverFormARouteGroup(t *testing.T) { + e, _, err := NormalizeTransmission([]Observation{ + obs(t, "tx", "a", t0, RouteClassFlood, 1, 0x11), + obs(t, "tx", "b", at(time.Second), RouteClassTransportDirect, 0), + }, 30*time.Second) + if err != nil { + t.Fatal(err) + } + if e.RouteClass != RouteClassMixed || e.RouteKind != RouteKindRelay { + t.Fatalf("fixture: class %v kind %v", e.RouteClass, e.RouteKind) + } + if _, ok := RouteGroupKeyOf(e); ok { + t.Fatal("a mixed transmission formed a route group") + } + cs := run(t, newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeRouteGroup, time.Minute, 1)}}), []Event{e}) + if len(cs) != 0 { + t.Fatalf("route-group candidate from a mixed transmission:\n%s", describe(cs)) + } + bad := relayEvent(t, "d", t0, 1, 0x5c) + for _, rc := range []RouteClass{RouteClassDirect, RouteClassTransportDirect, RouteClassUnknown} { + bad.RouteClass = rc + if err := bad.Validate(); err == nil { + t.Fatalf("relay kind with route class %v accepted", rc) + } + } +} + +// Ties at the frontier are ordered as if every event had waited in the +// reorder buffer, whatever Advance calls came in between. +func TestTieOrderDoesNotDependOnAdvanceCalls(t *testing.T) { + lim := experimentalLimits() + lim.ReorderDelay = 10 * time.Second + cfg := Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 2)}} + x, y := relayEvent(t, "b", t0, 1, 2), relayEvent(t, "a", t0, 1, 2) + + buffered := run(t, newDet(t, cfg), []Event{x, y}) + d := newDet(t, cfg) + var cs []Candidate + cs = append(cs, d.Observe(x).Candidates...) + cs = append(cs, d.Advance(at(10*time.Second)).Candidates...) + r := d.Observe(y) + cs = append(cs, r.Candidates...) + cs = append(cs, d.Flush().Candidates...) + if r.Status != StatusLate { + // accepting y after x was processed would reorder the tie + t.Fatalf("tied event with a smaller ID after its tie was processed: %v", r.Status) + } + if len(buffered) == 0 || buffered[len(buffered)-1].TriggerID != "b" { + t.Fatalf("buffered run:\n%s", describe(buffered)) + } + // a tied event that sorts after the processed one is still accepted + d = newDet(t, cfg) + d.Observe(y) + d.Advance(at(10 * time.Second)) + if st := d.Observe(x).Status; st != StatusAccepted { + t.Fatalf("tied event that sorts later: %v", st) + } +} + +func TestConfigRejectsNewStreamRulesThatCannotWork(t *testing.T) { + r := nsRule(5) + r.CensorWindow = r.QuietPeriod - time.Minute + if _, err := New(Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}, NewStream: []NewStreamRule{r}}); err == nil { + t.Fatal("CensorWindow < QuietPeriod accepted: a replay could call a continuing stream new") + } + r = nsRule(5) + r.State.ConfirmAfter = time.Minute + if _, err := New(Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}, NewStream: []NewStreamRule{r}}); err == nil { + t.Fatal("new-stream rule with ConfirmAfter accepted: it could never be confirmed") + } +} + +// A stream that was active shortly before the history start and reappears +// after the censor window is not reported as certainly new. +func TestNewStreamCensoringCoversTheQuietPeriod(t *testing.T) { + histStart := t0 + evs := series(t, "old", histStart.Add(nsRule(30).CensorWindow-time.Minute), 30, 30*time.Second, nil, 0x11, 0x22) + cs := run(t, newDet(t, Config{NewStream: []NewStreamRule{nsRule(30)}, HistoryStart: histStart}), evs) + for _, c := range filter(cs, "ns", StateActive) { + if !c.NewStream.Censored || c.Confidence != ConfidenceLow { + t.Fatalf("stream within QuietPeriod of the history start reported as certainly new: %+v", c.NewStream) + } + } +} + +func TestEpochZeroIsNotAValidTime(t *testing.T) { + e := relayEvent(t, "z", time.Unix(0, 0), 1, 2) + if err := e.Validate(); err == nil { + t.Fatal("Unix epoch accepted as an event time") + } + e.Time = time.Unix(0, 1) + if err := e.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestNormalizerPayloadConflictsAreCheckedAgainstEveryKnownSize(t *testing.T) { + a := obs(t, "tx", "a", t0, RouteClassFlood, 1, 0x11) + b := obs(t, "tx", "b", at(time.Second), RouteClassFlood, 1, 0x12) + b.PayloadKnown, b.PayloadBytes = true, 10 + c := obs(t, "tx", "c", at(2*time.Second), RouteClassFlood, 1, 0x13) + c.PayloadKnown, c.PayloadBytes = true, 20 + n, _ := NewNormalizer(normCfg()) + n.Add(a) + n.Add(b) + if st := n.Add(c).Status; st != ObsConflict { + t.Fatalf("size 20 after a held size 10: %v", st) + } + if _, _, err := NormalizeTransmission([]Observation{a, b, c}, 30*time.Second); err == nil { + t.Fatal("batch merged conflicting payload sizes") + } +} + +// The batch path excludes a late observation before any conflict check, +// exactly like the streaming Normalizer. +func TestBatchExcludesLateConflictsLikeStreaming(t *testing.T) { + a := obs(t, "tx", "a", t0, RouteClassFlood, 1, 0x11) + a.PayloadKnown, a.PayloadBytes = true, 30 + b := obs(t, "tx", "b", t0, RouteClassFlood, 1, 0x12) + b.ArrivedAt = at(2 * time.Minute) + b.PayloadKnown, b.PayloadBytes = true, 31 + e, rep, err := NormalizeTransmission([]Observation{a, b}, time.Minute) + if err != nil || rep.ExcludedLate != 1 || rep.Included != 1 || e.PayloadBytes != 30 { + t.Fatalf("batch: %v %+v %+v", err, rep, e) + } + n, _ := NewNormalizer(NormalizerConfig{SettleWindow: time.Minute, MaxPendingTransmissions: 10, + MaxObservationsPerTransmission: 10, FinalizedHorizon: time.Hour, MaxFinalizedIDs: 10}) + n.Add(a) + if st := n.Add(b).Status; st != ObsLate { + t.Fatalf("streaming: %v", st) + } +} + +func TestNormalizerTruncationReachesTheEventAndTheCandidates(t *testing.T) { + cfg := normCfg() + cfg.MaxObservationsPerTransmission = 1 + n, _ := NewNormalizer(cfg) + n.Add(obs(t, "tx", "a", at(2*time.Second), RouteClassFlood, 1, 0x11)) + b := obs(t, "tx", "b", t0, RouteClassFlood, 1, 0x22) // earlier data time, arrives later + b.ArrivedAt = at(3 * time.Second) + if st := n.Add(b).Status; st != ObsDroppedFanout { + t.Fatalf("fixture: %v", st) + } + evs := n.Flush().Events + if len(evs) != 1 || !evs[0].Truncated || evs[0].ExcludedObservations != 1 { + t.Fatalf("truncated event: %+v", evs) + } + d := newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 1)}}) + r := d.Observe(evs[0]) + if !r.CoverageReduced || len(r.Candidates) == 0 || r.Candidates[0].Coverage&CoverageInputTruncated == 0 { + t.Fatalf("candidate on truncated input not flagged: %+v", r) + } + if s := d.Snapshot(); s.TruncatedInputs != 1 || !s.CoverageReduced() { + t.Fatalf("stats: %+v", s) + } + // forced early finalization is truncation too + cfg = normCfg() + cfg.MaxPendingTransmissions = 1 + n, _ = NewNormalizer(cfg) + n.Add(obs(t, "t1", "a", t0, RouteClassFlood, 1, 0x11)) + res := n.Add(obs(t, "t2", "a", at(time.Second), RouteClassFlood, 1, 0x11)) + if len(res.Events) != 1 || !res.Events[0].Truncated { + t.Fatalf("forced finalization: %+v", res.Events) + } +} + +func TestNormalizerRejectsUnboundedDurations(t *testing.T) { + c := normCfg() + c.SettleWindow, c.FinalizedHorizon = math.MaxInt64, math.MaxInt64 + if _, err := NewNormalizer(c); err == nil { + t.Fatal("overflowing SettleWindow accepted") + } + c = normCfg() + c.SettleWindow, c.FinalizedHorizon = 25*time.Hour, 25*time.Hour + if _, err := NewNormalizer(c); err == nil { + t.Fatal("SettleWindow above 24h accepted") + } + c = normCfg() + c.FinalizedHorizon = 365 * 24 * time.Hour + if _, err := NewNormalizer(c); err == nil { + t.Fatal("FinalizedHorizon of a year accepted") + } +} + +func TestAdvanceOutsideTheValidRangeIsCounted(t *testing.T) { + d := newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 1)}}) + d.Advance(time.Date(2300, 1, 1, 0, 0, 0, 0, time.UTC)) + if d.Snapshot().InvalidAdvance != 1 { + t.Fatal("invalid Advance not counted") + } +} + +// Flush (end of input, as used by Replay) stamps released items no earlier +// than an Advance could have released them. +func TestFlushDecidesNoEarlierThanALiveDetector(t *testing.T) { + lim := experimentalLimits() + lim.ReorderDelay = 20 * time.Second + r := RateRule{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 1, State: StateParams{EndAfterQuiet: 10 * time.Second}} + d := newDet(t, Config{Limits: lim, Rate: []RateRule{r}}) + d.Observe(relayEvent(t, "a", t0, 1, 2)) + d.Observe(relayEvent(t, "b", at(15*time.Second), 1, 3)) + cs := d.Flush().Candidates + var endX *Candidate + for i := range cs { + c := &cs[i] + if c.DecidedAt.Before(c.At.Add(lim.ReorderDelay)) { + t.Fatalf("%s->%s at %v decided at %v, before At+ReorderDelay", c.From, c.To, c.At.Sub(t0), c.DecidedAt.Sub(t0)) + } + if c.To == StateEnded { + endX = c + } + } + if endX == nil || !endX.DecidedAt.Equal(at(30*time.Second)) { + t.Fatalf("quiet end %+v, want DecidedAt = deadline 10s + ReorderDelay 20s", endX) + } +} + +// With ReorderDelay 0 every event is processed on arrival, so same-time +// events keep arrival order and none is rejected. +func TestZeroReorderDelayKeepsTiesInArrivalOrder(t *testing.T) { + d := newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 2)}}) + if st := d.Observe(relayEvent(t, "b", t0, 1, 2)).Status; st != StatusAccepted { + t.Fatal(st) + } + if st := d.Observe(relayEvent(t, "a", t0, 1, 2)).Status; st != StatusAccepted { + t.Fatalf("same-time event with a smaller ID rejected with ReorderDelay 0: %v", st) + } +} + +func TestObservationPayloadMustFitEveryFrame(t *testing.T) { + o := obs(t, "tx", "a", t0, RouteClassFlood, 1, 0x11) + o.WireBytes, o.PayloadKnown, o.PayloadBytes = 10, true, 100 + if o.Validate() == nil { + t.Fatal("payload larger than its own frame accepted") + } + // the small frame is not the first observation, so only the held-frame + // check can see the mismatch + first := obs(t, "tx", "u", t0, RouteClassFlood, 1, 0x10) + first.WireBytes = 0 + small := obs(t, "tx", "s", t0, RouteClassFlood, 1, 0x11) + small.WireBytes = 20 + big := obs(t, "tx", "b", at(time.Second), RouteClassFlood, 1, 0x12) + big.WireBytes, big.PayloadKnown, big.PayloadBytes = 120, true, 100 + n, _ := NewNormalizer(normCfg()) + n.Add(first) + n.Add(small) + if st := n.Add(big).Status; st != ObsConflict { + t.Fatalf("payload that cannot fit a held frame: %v", st) + } + for _, e := range n.Flush().Events { + if err := e.Validate(); err != nil { + t.Fatalf("Normalizer emitted an invalid event: %v", err) + } + } +} + +func TestParseKeyAppliesTheKindClassContract(t *testing.T) { + if _, err := ParseKey("v1|s=stream|pt=5|ch=1:ab|rc=direct|rk=relay|hop=1:5c"); err == nil { + t.Fatal("stream key with relay kind on a direct class accepted") + } +} + +// Coverage flags look back from the newest capacity event: an older event +// accepted out of order must not move that point backwards. +func TestCoverageFlagTimesOnlyMoveForward(t *testing.T) { + lim := experimentalLimits() + lim.ReorderDelay, lim.DedupHorizon = time.Hour, 2*time.Hour + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 1)}}) + a := relayEvent(t, "a", at(59*time.Minute), 1, 2) + a.Truncated = true + b := relayEvent(t, "b", t0, 1, 3) + b.Truncated = true + d.Observe(a) + d.Observe(b) // accepted: within ReorderDelay + c := relayEvent(t, "c", at(150*time.Minute), 1, 4) + d.Observe(c) + var got []Candidate + for r := d.Flush(); ; r = d.Drain() { + got = append(got, r.Candidates...) + if !r.More { + break + } + } + for _, x := range got { + if x.TriggerID == "c" && x.Coverage&CoverageInputTruncated == 0 { + t.Fatal("candidate 91 minutes after truncated input lost CoverageInputTruncated") + } + } +} + +// A time-driven end run early by a forced reorder release is flagged. +func TestForcedReleaseFlagsTimeDrivenEnds(t *testing.T) { + lim := experimentalLimits() + lim.ReorderDelay, lim.MaxReorderBuffer = 10*time.Second, 2 + r := RateRule{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 1, State: StateParams{EndAfterQuiet: 5 * time.Second}} + d := newDet(t, Config{Limits: lim, Rate: []RateRule{r}}) + var cs []Candidate + for _, e := range []Event{ + relayEvent(t, "a", at(100*time.Second), 1, 2), // released normally by d + relayEvent(t, "d", at(111*time.Second), 1, 3), + relayEvent(t, "e", at(112*time.Second), 1, 4), + relayEvent(t, "f", at(113*time.Second), 1, 5), // overflow: forces d out, a's end runs first + } { + cs = append(cs, d.Observe(e).Candidates...) + } + for _, c := range cs { + if c.To == StateEnded && c.Key.FirstHop == mustHop(t, 2) { + if c.Coverage&CoverageReorderForced == 0 { + t.Fatalf("early end without reorder_forced: %+v", c) + } + return + } + } + t.Fatalf("no end for a:\n%s", describe(cs)) +} diff --git a/internal/anomaly/detector.go b/internal/anomaly/detector.go new file mode 100644 index 000000000..b9a732d08 --- /dev/null +++ b/internal/anomaly/detector.go @@ -0,0 +1,912 @@ +package anomaly + +import ( + "container/heap" + "time" +) + +func nsTime(ns int64) time.Time { return time.Unix(0, ns).UTC() } + +// Status is the outcome of Detector.Observe. +type Status uint8 + +const ( + StatusAccepted Status = iota + 1 // accepted (it may still wait in the reorder buffer) + StatusDuplicate // ID already counted within the dedup horizon + StatusLate // older than the processed frontier + StatusInvalid // failed Event.Validate +) + +var statusNames = [...]string{"", "accepted", "duplicate", "late", "invalid"} + +func (s Status) String() string { + if int(s) < len(statusNames) && s > 0 { + return statusNames[s] + } + return "status(?)" +} + +// Result is returned by Observe, Advance, Flush and Drain. +type Result struct { + Status Status // zero for Advance, Flush and Drain + Err error // set for StatusInvalid + // Candidates are transitions in the order they happened in data time. + Candidates []Candidate + // More is true when further candidates wait in the buffer (Drain). + More bool + // CoverageReduced is true when a capacity limit cut evidence or dropped + // candidates during this call. + CoverageReduced bool +} + +// Stats are cumulative counters and current sizes. +type Stats struct { + Accepted, Duplicates, DuplicateConflicts, Late, Invalid uint64 + Processed uint64 + // TruncatedInputs counts accepted events with Event.Truncated set. + TruncatedInputs uint64 + // InvalidAdvance counts Advance calls with a time outside the valid range. + InvalidAdvance uint64 + Buffered int + DedupIDs int + DedupEvictedEarly uint64 + // EffectiveDedupHorizon is the data-time span the dedup set currently + // covers; below Limits.DedupHorizon when capacity forced evictions. + EffectiveDedupHorizon time.Duration + Keys [numScopes]int + CapacityEvictions [numScopes]uint64 + TTLEvictions [numScopes]uint64 + EpisodesEvicted uint64 + WindowMerges uint64 + ReorderForced uint64 + CandidatesEmitted uint64 + CandidatesDropped uint64 + PendingCandidates int + MaxCandidateBurst int + Frontier time.Time +} + +// CoverageReduced reports whether any capacity limit has cut evidence. +func (s Stats) CoverageReduced() bool { + if s.DedupEvictedEarly > 0 || s.EpisodesEvicted > 0 || s.WindowMerges > 0 || s.ReorderForced > 0 || s.CandidatesDropped > 0 || + s.TruncatedInputs > 0 { + return true + } + for _, n := range s.CapacityEvictions { + if n > 0 { + return true + } + } + return false +} + +// Detector is a deterministic, single-goroutine anomaly detector. Separate +// Detectors share nothing and may run on separate goroutines. It holds no +// timers and never reads the wall clock: time moves only with event times +// and with Advance. +type Detector struct { + cfg Config + scopes [numScopes]*keyTable + historySet bool + history int64 + + maxSeen int64 + frontier int64 // latest data time processed (events and deadlines) + started bool + // clock is the decision clock: the latest FinalAt of any valid event + // offered, or the latest Advance time. Candidates are stamped with it. + clock int64 + // the last processed event, so a tie at the frontier is ordered exactly + // as if both events had waited in the reorder buffer + lastEvT int64 + lastEvID TxID + evDone bool + + dedup map[TxID]int64 + dedupLog []dedupRec + dedupHead int + + reorder eventHeap + deadlines deadlineHeap + + pending []Candidate + pendingHead int + + lastDedupEarly, lastReorderForced, lastTruncated int64 + stats Stats + trBuf []transition + perScratch periodicScratch + callReduced bool // a capacity limit cut evidence during the current call +} + +type dedupRec struct { + id TxID + t int64 +} + +// New validates cfg and returns a Detector. The config is copied. +func New(cfg Config) (*Detector, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + c := cfg + c.Rate = append([]RateRule(nil), cfg.Rate...) + c.NewStream = append([]NewStreamRule(nil), cfg.NewStream...) + c.Periodic = append([]PeriodicRule(nil), cfg.Periodic...) + d := &Detector{cfg: c, dedup: make(map[TxID]int64)} + for i := range c.Rate { + d.table(c.Rate[i].Scope).addRate(i, c.Rate[i].Window) + } + for i := range c.NewStream { + d.table(c.NewStream[i].Scope).rules.newStream = append(d.table(c.NewStream[i].Scope).rules.newStream, i) + } + for i := range c.Periodic { + d.table(c.Periodic[i].Scope).rules.periodic = append(d.table(c.Periodic[i].Scope).rules.periodic, i) + } + if !c.HistoryStart.IsZero() { + d.history, d.historySet = c.HistoryStart.UnixNano(), true + } + return d, nil +} + +func (d *Detector) table(s Scope) *keyTable { + if d.scopes[s] == nil { + d.scopes[s] = newKeyTable(s) + } + return d.scopes[s] +} + +// Observe offers one final event. Events must arrive in data-time order, or +// out of order by at most Limits.ReorderDelay; anything older than the +// processed frontier is rejected as late. Candidates may be returned by a +// later call because of the reorder delay and the per-call cap. +func (d *Detector) Observe(e Event) Result { + var res Result + if err := e.Validate(); err != nil { + d.stats.Invalid++ + res.Status, res.Err = StatusInvalid, err + return res + } + t := e.Time.UnixNano() + if f := e.finalAt().UnixNano(); f > d.clock { + d.clock = f + } + if prev, ok := d.dedup[e.ID]; ok { + d.stats.Duplicates++ + if prev != t { + d.stats.DuplicateConflicts++ + } + res.Status = StatusDuplicate + return d.finish(res) + } + // With ReorderDelay > 0, a tie at the frontier that was already processed + // (e.g. by Advance) would be reordered by accepting a smaller ID; with 0 + // every event is processed on arrival, so ties keep arrival order. + tieLate := d.cfg.Limits.ReorderDelay > 0 && d.evDone && t == d.lastEvT && e.ID <= d.lastEvID + if d.started && (t < d.frontier || tieLate) { + d.stats.Late++ + res.Status = StatusLate + return d.finish(res) + } + d.started = true + if !d.historySet { + d.history, d.historySet = t, true + } + d.rememberID(e.ID, t) + d.stats.Accepted++ + if e.Truncated { + d.stats.TruncatedInputs++ + d.lastTruncated = maxInt64(d.lastTruncated, t) + d.callReduced = true + } + res.Status = StatusAccepted + // the event may wait in the reorder buffer: never alias caller memory + e.FullRoute = Route{w: e.FullRoute.w, hops: append([]byte(nil), e.FullRoute.hops...)} + d.reorder.push(e) + if t > d.maxSeen { + d.maxSeen = t + } + if d.reorder.Len() > d.cfg.Limits.MaxReorderBuffer { + // force the oldest out; it is still processed in order + d.stats.ReorderForced++ + d.lastReorderForced = maxInt64(d.lastReorderForced, t) + d.callReduced = true + d.process(d.reorder[0].Time.UnixNano(), true, false) + } + d.process(d.maxSeen-int64(d.cfg.Limits.ReorderDelay), false, false) + return d.finish(res) +} + +// Advance tells the detector that data time has reached t: events earlier +// than t-ReorderDelay will not arrive any more, so held events and due +// deadlines up to that point are processed. Time is taken to have passed +// continuously: an item at data time x is decided at the earliest time an +// Advance could have released it, max(clock, x+ReorderDelay), and the +// decision clock ends at t. t earlier than already known is ignored; t +// outside the valid time range is ignored and counted in +// Stats.InvalidAdvance. +func (d *Detector) Advance(t time.Time) Result { + var res Result + if !validTime(t) { + d.stats.InvalidAdvance++ + } else { + ns := t.UnixNano() + if ns > d.maxSeen { + d.maxSeen = ns + } + if d.started { + d.process(d.maxSeen-int64(d.cfg.Limits.ReorderDelay), false, true) + } + if ns > d.clock { + d.clock = ns + } + } + return d.finish(res) +} + +// Flush processes every held event (end of input) and the deadlines up to +// the latest event time. It does not invent time beyond the data. Like +// Advance, it stamps each released item no earlier than its data time + +// ReorderDelay, the earliest time a live detector could have released it. +func (d *Detector) Flush() Result { + var res Result + if d.started { + d.process(d.maxSeen, false, true) + } + return d.finish(res) +} + +// Drain returns buffered candidates without processing anything. +func (d *Detector) Drain() Result { return d.finish(Result{}) } + +// Snapshot returns counters and current sizes. +func (d *Detector) Snapshot() Stats { + s := d.stats + s.Buffered = d.reorder.Len() + s.DedupIDs = len(d.dedup) + if d.dedupHead < len(d.dedupLog) { + s.EffectiveDedupHorizon = time.Duration(d.maxSeen - d.dedupLog[d.dedupHead].t) + } + for sc, tb := range d.scopes { + if tb != nil { + s.Keys[sc] = len(tb.m) + s.CapacityEvictions[sc] = tb.capacityEvictions + s.TTLEvictions[sc] = tb.ttlEvictions + } + } + s.PendingCandidates = len(d.pending) - d.pendingHead + if d.started { + s.Frontier = nsTime(d.frontier) + } + return s +} + +func (d *Detector) finish(res Result) Result { + res.CoverageReduced = res.CoverageReduced || d.callReduced + d.callReduced = false + n := len(d.pending) - d.pendingHead + if n > d.cfg.Limits.MaxCandidatesPerCall { + n = d.cfg.Limits.MaxCandidatesPerCall + } + if n > 0 { + res.Candidates = append(res.Candidates, d.pending[d.pendingHead:d.pendingHead+n]...) + for i := d.pendingHead; i < d.pendingHead+n; i++ { + d.pending[i] = Candidate{} + } + d.pendingHead += n + } + res.More = d.pendingHead < len(d.pending) + if d.pendingHead == len(d.pending) { + d.pending, d.pendingHead = d.pending[:0], 0 + } + if len(res.Candidates) > d.stats.MaxCandidateBurst { + d.stats.MaxCandidateBurst = len(res.Candidates) + } + return res +} + +func (d *Detector) rememberID(id TxID, t int64) { + lim := &d.cfg.Limits + // expire by horizon + lo := d.maxSeen - int64(lim.DedupHorizon) + if t > d.maxSeen { + lo = t - int64(lim.DedupHorizon) + } + for d.dedupHead < len(d.dedupLog) && d.dedupLog[d.dedupHead].t < lo { + d.dropDedupHead() + } + d.dedup[id] = t + d.dedupLog = append(d.dedupLog, dedupRec{id: id, t: t}) + for len(d.dedup) > lim.MaxDedupIDs { + d.stats.DedupEvictedEarly++ + d.callReduced = true + d.lastDedupEarly = maxInt64(d.lastDedupEarly, t) + d.dropDedupHead() + } + if d.dedupHead >= 1024 && d.dedupHead*2 >= len(d.dedupLog) { + d.dedupLog = append([]dedupRec(nil), d.dedupLog[d.dedupHead:]...) + d.dedupHead = 0 + } +} + +func (d *Detector) dropDedupHead() { + r := d.dedupLog[d.dedupHead] + d.dedupLog[d.dedupHead] = dedupRec{} + d.dedupHead++ + if t, ok := d.dedup[r.id]; ok && t == r.t { + delete(d.dedup, r.id) + } +} + +// process handles held events and deadlines in data-time order up to +// horizon. Deadlines at or before an event's time run before the event. +// With forceOne it releases exactly the oldest held event (and the +// deadlines at or before it), whatever the horizon. With tick (Advance and +// Flush) the decision clock moves with the items: to at least +// x+ReorderDelay for an item at data time x. +func (d *Detector) process(horizon int64, forceOne, tick bool) { + for { + d.dropStaleDeadlines() + hasEv, hasDl := d.reorder.Len() > 0, d.deadlines.Len() > 0 + var evT, dlT int64 + if hasEv { + evT = d.reorder[0].Time.UnixNano() + if forceOne { + horizon = evT + } + } + if hasDl { + dlT = d.deadlines[0].at + } + if hasDl && dlT <= horizon && (!hasEv || dlT <= evT) { + dl := heap.Pop(&d.deadlines).(deadline) + if tick { + d.tickTo(dl.at) + } + d.advanceFrontier(dl.at) + d.runDeadline(dl) + continue + } + if hasEv && evT <= horizon { + e := d.reorder.pop() + if tick { + d.tickTo(evT) + } + d.advanceFrontier(evT) + d.handle(&e) + if forceOne { + return + } + continue + } + return + } +} + +// tickTo moves the decision clock to the earliest time an Advance could +// have released an item at data time x. +func (d *Detector) tickTo(x int64) { + if c := x + int64(d.cfg.Limits.ReorderDelay); c > d.clock { + d.clock = c + } +} + +func (d *Detector) advanceFrontier(t int64) { + if t > d.frontier { + d.frontier = t + } + d.expireKeys(d.frontier) +} + +// ---- key tables ---- + +type scopeRules struct { + rate []int // indices into Config.Rate + widthOf []int // per rate rule: index into widths + widths []int64 // distinct window widths (ns) + newStream []int + periodic []int +} + +func (r *scopeRules) slots() int { return len(r.rate) + len(r.newStream) + len(r.periodic) } + +type keyState struct { + key Key + lastSeen int64 + prev, next *keyState + win *windowSet + ns []newStreamState + per []*periodicState + eps []episode +} + +func (k *keyState) openEpisodes() bool { + for i := range k.eps { + if k.eps[i].open() { + return true + } + } + return false +} + +type keyTable struct { + scope Scope + rules scopeRules + m map[Key]*keyState + root keyState // LRU sentinel: root.next = most recent + capacityEvictions uint64 + ttlEvictions uint64 + lastCapacityEviction int64 + everCapacityEvicted bool +} + +func newKeyTable(s Scope) *keyTable { + t := &keyTable{scope: s, m: make(map[Key]*keyState)} + t.root.next, t.root.prev = &t.root, &t.root + return t +} + +func (t *keyTable) addRate(ruleIdx int, w time.Duration) { + wi := -1 + for i, x := range t.rules.widths { + if x == int64(w) { + wi = i + } + } + if wi < 0 { + wi = len(t.rules.widths) + t.rules.widths = append(t.rules.widths, int64(w)) + } + t.rules.rate = append(t.rules.rate, ruleIdx) + t.rules.widthOf = append(t.rules.widthOf, wi) +} + +func (t *keyTable) unlink(k *keyState) { k.prev.next, k.next.prev = k.next, k.prev } + +func (t *keyTable) pushFront(k *keyState) { + k.next, k.prev = t.root.next, &t.root + t.root.next.prev = k + t.root.next = k +} + +// expireKeys drops keys idle for StateTTL (from the LRU tail). +func (d *Detector) expireKeys(now int64) { + ttl := int64(d.cfg.Limits.StateTTL) + for _, t := range d.scopes { + if t == nil { + continue + } + // strict: a key idle exactly StateTTL is kept, so a quiet deadline at + // that same instant still ends its episode normally + for tail := t.root.prev; tail != &t.root && tail.lastSeen < now-ttl; tail = t.root.prev { + d.dropKey(t, tail, now, false) + t.ttlEvictions++ + } + } +} + +func (d *Detector) dropKey(t *keyTable, k *keyState, now int64, capacity bool) { + if k.openEpisodes() { + for slot := range k.eps { + if !k.eps[slot].open() { + continue + } + d.trBuf = k.eps[slot].evict(now, d.trBuf[:0]) + d.stats.EpisodesEvicted++ + var cov CoverageFlags + if capacity { + cov = CoverageKeyEvictions + } + d.emit(t, k, slot, d.trBuf, emitCtx{at: now, coverage: cov}) + } + } + t.unlink(k) + delete(t.m, k.key) + if capacity { + d.callReduced = true + t.capacityEvictions++ + t.lastCapacityEviction = now + t.everCapacityEvicted = true + } +} + +func (d *Detector) getKey(t *keyTable, key Key, now int64) *keyState { + if k, ok := t.m[key]; ok { + t.unlink(k) + t.pushFront(k) + return k + } + if len(t.m) >= d.cfg.Limits.MaxKeysPerScope { + // prefer the least recent key without an open episode (bounded scan) + victim := t.root.prev + for i, c := 0, t.root.prev; i < 8 && c != &t.root; i, c = i+1, c.prev { + if !c.openEpisodes() { + victim = c + break + } + } + d.dropKey(t, victim, now, true) + } + k := &keyState{key: key} + r := &t.rules + if len(r.rate) > 0 { + k.win = newWindowSet(len(r.widths)) + } + if len(r.newStream) > 0 { + k.ns = make([]newStreamState, len(r.newStream)) + } + if len(r.periodic) > 0 { + k.per = make([]*periodicState, len(r.periodic)) + for i, ri := range r.periodic { + k.per[i] = newPeriodicState(d.cfg.Periodic[ri].HistoryLen) + } + } + k.eps = make([]episode, r.slots()) + t.m[key] = k + t.pushFront(k) + return k +} + +func appliesTo(types []PayloadType, pt PayloadType) bool { + if len(types) == 0 { + return true + } + for _, x := range types { + if x == pt { + return true + } + } + return false +} + +// ---- event handling ---- + +type emitCtx struct { + at int64 + trigger *Event + label TrafficLabel + rate *RateEvidence + ns *NewStreamEvidence + per *PeriodicEvidence + coverage CoverageFlags +} + +func (d *Detector) handle(e *Event) { + d.stats.Processed++ + t := e.Time.UnixNano() + d.lastEvT, d.lastEvID, d.evDone = t, e.ID, true + for s := Scope(1); s < numScopes; s++ { + tb := d.scopes[s] + if tb == nil { + continue + } + key, ok := keyOf(s, *e) + if !ok { + continue + } + r := &tb.rules + applies := false + for _, i := range r.rate { + applies = applies || appliesTo(d.cfg.Rate[i].PayloadTypes, e.PayloadType) + } + for _, i := range r.newStream { + applies = applies || appliesTo(d.cfg.NewStream[i].PayloadTypes, e.PayloadType) + } + for _, i := range r.periodic { + applies = applies || appliesTo(d.cfg.Periodic[i].PayloadTypes, e.PayloadType) + } + if !applies { + continue + } + k := d.getKey(tb, key, t) + k.lastSeen = t + cov := d.coverageFor(tb, k, t) + slot := 0 + if k.win != nil { + if k.win.add(t, e.Traffic, r.widths, d.cfg.Limits.MaxWindowEntries) { + d.stats.WindowMerges++ + d.callReduced = true + cov |= CoverageWindowSaturated + } + for ri, i := range r.rate { + rule := &d.cfg.Rate[i] + if appliesTo(rule.PayloadTypes, e.PayloadType) { + sum := k.win.sums[r.widthOf[ri]] + if sum.n >= uint64(rule.Threshold) { + ev := &RateEvidence{Window: rule.Window, Threshold: rule.Threshold, Count: sum.n, + Expected: sum.exp, Monitored: sum.mon, + Saturated: k.win.lastMerge != 0 && k.win.lastMerge > t-int64(rule.Window)} + c := cov + if ev.Saturated { + c |= CoverageWindowSaturated + } + d.hot(tb, k, slot, &rule.State, emitCtx{at: t, trigger: e, + label: labelFromCounts(sum.n, sum.exp, sum.mon), rate: ev, coverage: c}) + } + } + slot++ + } + } + for ni, i := range r.newStream { + rule := &d.cfg.NewStream[i] + if appliesTo(rule.PayloadTypes, e.PayloadType) { + if ev, lbl, ok := k.ns[ni].observe(t, e.Traffic, d.history, rule); ok { + d.hot(tb, k, slot, &rule.State, emitCtx{at: t, trigger: e, label: lbl, ns: ev, coverage: cov}) + } + } + slot++ + } + for pi, i := range r.periodic { + rule := &d.cfg.Periodic[i] + if appliesTo(rule.PayloadTypes, e.PayloadType) { + if sig, ok := k.per[pi].observe(t, e.Traffic, rule, &d.perScratch); ok { + ev := sig.ev + d.hot(tb, k, slot, &rule.State, emitCtx{at: t, trigger: e, label: sig.label, per: &ev, coverage: cov}) + } + } + slot++ + } + } +} + +func (d *Detector) coverageFor(tb *keyTable, k *keyState, t int64) CoverageFlags { + var c CoverageFlags + lim := &d.cfg.Limits + if tb.everCapacityEvicted && t-tb.lastCapacityEviction < int64(lim.StateTTL) { + c |= CoverageKeyEvictions + } + if d.lastDedupEarly != 0 && t-d.lastDedupEarly < int64(lim.DedupHorizon) { + c |= CoverageDedupEvictedEarly + } + if d.lastReorderForced != 0 && t-d.lastReorderForced < int64(lim.DedupHorizon) { + c |= CoverageReorderForced + } + if d.lastTruncated != 0 && t-d.lastTruncated < int64(lim.DedupHorizon) { + c |= CoverageInputTruncated + } + return c +} + +func (d *Detector) params(tb *keyTable, slot int) (*StateParams, string, Kind) { + r := &tb.rules + if slot < len(r.rate) { + x := &d.cfg.Rate[r.rate[slot]] + return &x.State, x.Name, KindRate + } + slot -= len(r.rate) + if slot < len(r.newStream) { + x := &d.cfg.NewStream[r.newStream[slot]] + return &x.State, x.Name, KindNewStream + } + slot -= len(r.newStream) + x := &d.cfg.Periodic[r.periodic[slot]] + return &x.State, x.Name, KindPeriodic +} + +func (d *Detector) hot(tb *keyTable, k *keyState, slot int, p *StateParams, ctx emitCtx) { + ep := &k.eps[slot] + d.trBuf = ep.onHot(ctx.at, p, d.trBuf[:0]) + ep.note(ctx.label, ctx.coverage) + d.emit(tb, k, slot, d.trBuf, ctx) + d.schedule(tb, k, slot, p) +} + +func (d *Detector) schedule(tb *keyTable, k *keyState, slot int, p *StateParams) { + ep := &k.eps[slot] + if at := ep.deadline(p); at != 0 { + heap.Push(&d.deadlines, deadline{at: at, scope: tb.scope, key: k.key, slot: slot, version: ep.version}) + } +} + +func (d *Detector) runDeadline(dl deadline) { + tb := d.scopes[dl.scope] + k, ok := tb.m[dl.key] + if !ok || k.eps[dl.slot].version != dl.version { + return + } + p, _, _ := d.params(tb, dl.slot) + ep := &k.eps[dl.slot] + d.trBuf = ep.onTime(dl.at, p, d.trBuf[:0]) + // capacity events near the deadline (e.g. a forced reorder release that + // ran it early) are flagged on top of the episode's own coverage + d.emit(tb, k, dl.slot, d.trBuf, emitCtx{at: dl.at, coverage: d.coverageFor(tb, k, dl.at)}) + d.schedule(tb, k, dl.slot, p) +} + +func (d *Detector) emit(tb *keyTable, k *keyState, slot int, trs []transition, ctx emitCtx) { + if len(trs) == 0 { + return + } + _, name, kind := d.params(tb, slot) + for _, tr := range trs { + c := Candidate{ + EventID: episodeID(name, k.key, tr.start), + Rule: name, + Kind: kind, + Key: k.key, + From: tr.from, + To: tr.to, + Reason: tr.reason, + At: nsTime(tr.at), + DecidedAt: nsTime(maxInt64(tr.at, d.clock)), + EpisodeStart: nsTime(tr.start), + Confidence: k.key.confidence(), + Traffic: ctx.label, + Coverage: ctx.coverage, + Summary: tr.summary, + } + if ctx.trigger == nil { + // time-driven or eviction: the episode's own label and coverage + c.Traffic = tr.label + c.Coverage = tr.cov | ctx.coverage + } else { + c.TriggerID = ctx.trigger.ID + c.TriggerRoute = Route{w: ctx.trigger.FullRoute.w, hops: append([]byte(nil), ctx.trigger.FullRoute.hops...)} + if ctx.rate != nil { + ev := *ctx.rate + c.Rate = &ev + } + if ctx.ns != nil { + ev := *ctx.ns + c.NewStream = &ev + if ev.Censored { + c.Confidence = ConfidenceLow + } + } + if ctx.per != nil { + ev := *ctx.per + ev.Alternatives = append([]PeriodAlternative(nil), ctx.per.Alternatives...) + c.Periodic = &ev + } + } + c.Suppressed = d.cfg.Expected.Mode == ExpectedSuppress && c.Traffic == LabelExpected + d.push(c) + } +} + +func (d *Detector) push(c Candidate) { + d.stats.CandidatesEmitted++ + if len(d.pending)-d.pendingHead >= d.cfg.Limits.MaxPendingCandidates { + d.pending[d.pendingHead] = Candidate{} + d.pendingHead++ + d.stats.CandidatesDropped++ + d.callReduced = true + } + d.pending = append(d.pending, c) + if d.pendingHead >= 256 && d.pendingHead*2 >= len(d.pending) { + d.pending = append([]Candidate(nil), d.pending[d.pendingHead:]...) + d.pendingHead = 0 + } +} + +func maxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} + +// ---- heaps ---- + +// eventHeap is a typed min-heap on (Time, ID). It avoids container/heap so +// events are not boxed into interfaces (no allocation per event). +type eventHeap []Event + +func (h eventHeap) Len() int { return len(h) } +func (h eventHeap) less(i, j int) bool { + if !h[i].Time.Equal(h[j].Time) { + return h[i].Time.Before(h[j].Time) + } + return h[i].ID < h[j].ID +} + +func (h *eventHeap) push(e Event) { + *h = append(*h, e) + a := *h + for i := len(a) - 1; i > 0; { + p := (i - 1) / 2 + if !a.less(i, p) { + break + } + a[i], a[p] = a[p], a[i] + i = p + } +} + +func (h *eventHeap) pop() Event { + a := *h + top := a[0] + last := len(a) - 1 + a[0] = a[last] + a[last] = Event{} + a = a[:last] + for i := 0; ; { + l, r, m := 2*i+1, 2*i+2, i + if l < len(a) && a.less(l, m) { + m = l + } + if r < len(a) && a.less(r, m) { + m = r + } + if m == i { + break + } + a[i], a[m] = a[m], a[i] + i = m + } + *h = a + return top +} + +type deadline struct { + at int64 + scope Scope + key Key + slot int + version uint32 +} + +type deadlineHeap []deadline + +func (h deadlineHeap) Len() int { return len(h) } +func (h deadlineHeap) Less(i, j int) bool { + a, b := h[i], h[j] + if a.at != b.at { + return a.at < b.at + } + if a.scope != b.scope { + return a.scope < b.scope + } + if a.key != b.key { + return a.key.less(b.key) + } + if a.slot != b.slot { + return a.slot < b.slot + } + return a.version < b.version +} +func (h deadlineHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *deadlineHeap) Push(x interface{}) { *h = append(*h, x.(deadline)) } +func (h *deadlineHeap) Pop() interface{} { + old := *h + x := old[len(old)-1] + *h = old[:len(old)-1] + return x +} + +// dropStaleDeadlines pops stale entries at the top and compacts the heap +// when stale entries dominate, so it stays bounded by live episodes. +func (d *Detector) dropStaleDeadlines() { + for d.deadlines.Len() > 0 && !d.deadlineLive(d.deadlines[0]) { + heap.Pop(&d.deadlines) + } + if d.deadlines.Len() > 1024 && d.deadlines.Len() > 4*d.liveEpisodes()+1024 { + kept := d.deadlines[:0] + for _, x := range d.deadlines { + if d.deadlineLive(x) { + kept = append(kept, x) + } + } + d.deadlines = kept + heap.Init(&d.deadlines) + } +} + +func (d *Detector) deadlineLive(x deadline) bool { + tb := d.scopes[x.scope] + if tb == nil { + return false + } + k, ok := tb.m[x.key] + return ok && k.eps[x.slot].version == x.version +} + +func (d *Detector) liveEpisodes() int { + n := 0 + for _, tb := range d.scopes { + if tb != nil { + n += len(tb.m) * tb.rules.slots() + } + } + return n +} diff --git a/internal/anomaly/doc.go b/internal/anomaly/doc.go new file mode 100644 index 000000000..aa9850cf2 --- /dev/null +++ b/internal/anomaly/doc.go @@ -0,0 +1,119 @@ +// Package anomaly is an EXPERIMENTAL, OFFLINE library for +// encryption-independent traffic anomaly detection on MeshCore transmissions. +// +// # Status +// +// - Experimental and offline only. Nothing in CoreScope calls this package: +// no ingestor, server, API, UI, database, config, webhook or notification +// wiring exists. Its only callers are its own tests and benchmarks. +// - There are NO default thresholds. Every limit and threshold must be set +// explicitly in Config; New rejects the zero value and rejects a config +// that enables no rule. Values used in tests are named experimental +// fixtures, not recommendations. +// - Phase 1 of the study (a 30-day staging snapshot) validated only three +// incidents. No rule here is validated. A blind validation on a fresh +// snapshot starting no earlier than 2026-10-08 is required before any +// threshold is proposed for production. +// +// # What "encryption-independent" means +// +// Detection uses only on-wire, key-free facts supplied by the caller: +// payload type, an opaque channel hash, route class, the first relay hop +// with its byte width, the full route (as separate evidence), frame sizes +// and data timestamps. It does not need to decrypt anything and it does not +// depend on names such as "enc_XX" or on whether the caller's instance could +// decrypt a channel (DecryptStatus is metadata only). +// +// Encryption-independent does not mean identity-independent, and it is not +// attribution: +// +// - A stream key is an observed route group: channel hash + route class + +// first hop. It is not a sender. Mobile senders spread over many route +// groups; one busy relay carries many senders. +// - The first hop is the first relay after the originator as seen on the +// wire. 1-byte hops are globally ambiguous (many nodes share a prefix); +// 2- and 3-byte hops much less so. A 1-byte hop and the prefix of a wider +// hop are different values. +// - The observer is never assumed to be the sender or the first relay. +// No-path, direct, mixed and unknown routes are a separate low-confidence +// class and never form a route group; Event.Validate rejects a relay or +// no-path kind without a flood (or mixed) route class. +// - A global (payload-type) candidate is never attributed to a stream, +// channel or route. +// +// # Data contract +// +// The Detector consumes FINAL Events, one per logical transmission, and +// counts each Event.ID at most once within Limits.DedupHorizon (bounded by +// Limits.MaxDedupIDs; early evictions are counted and flagged). Observations +// are evidence of receivers and routes only and never multiply a rate. +// Callers either finish normalization themselves or use Normalizer, whose +// finality policy is: an event is final at first arrival + SettleWindow, +// and observations arriving after that are rejected as late, so a replay +// never uses route evidence that arrived after the decision. When the +// Normalizer's own caps cut a transmission's evidence, the Event is marked +// Truncated and candidates nearby carry CoverageInputTruncated. +// +// Candidates carry DecidedAt from a monotone decision clock: the latest +// FinalAt of every event offered so far, and within Advance the earliest +// time an Advance could have released the item (data time + +// ReorderDelay; Flush likewise). DecidedAt is therefore >= the FinalAt of +// every event behind a decision (not only the trigger), a time-driven end +// is never decided before the transitions that opened its episode nor, +// except after a forced release on reorder-buffer overflow (flagged +// CoverageReorderForced), before its deadline + ReorderDelay, and events +// offered in finalization order give the decision times of a live +// detector. +// +// Upstream deduplication by content hash (as in CoreScope's database) merges +// byte-identical re-broadcasts into one transmission before this library +// sees them; such repeats cannot be counted here. +// +// # Time +// +// Only data time is used. Valid times lie after the Unix epoch and before +// 2200. Events must arrive in data-time order or within +// Limits.ReorderDelay of it; older events are rejected as late (counted), +// never silently misplaced. With ReorderDelay > 0, events with equal times +// are processed in ID order and one that arrives after its tie was +// processed and would sort first is late too; with ReorderDelay 0 equal +// times keep arrival order. Deadlines (quiet ends, cooldowns) are +// processed in data-time order together with events, so the output depends +// only on the input and the config, not on when Advance or Flush were +// called; the one exception is DecidedAt, which records when the call +// sequence made each decision possible. +// +// # Building blocks implemented +// +// - RateRule: sliding windows (t-W, t] at stream, channel, route-group or +// global scope, any width from 1 s to 24 h. +// - NewStreamRule: causal new-stream volume with left censoring (the +// censor window must cover QuietPeriod) and reactivation after silence. +// - PeriodicRule: causal periodicity with explicit burst, jitter, missing +// pulse and harmonic handling, recovery after stray pulses, and a +// local-rate Poisson null with a union bound over the period search. +// - A per-episode state machine normal -> suspicious -> active -> ended +// with confirmation time, quiet end, cooldown, stable EventIDs and +// reason codes. +// +// # Not implemented +// +// Deliberately NOT implemented: hash-rotation, baseline-deviation and +// large-packet rules. Phase 1 did not validate them and they would widen +// this change; route-group and global rate windows cover the multi-channel +// storm seen in phase 1. +// +// # Memory and coverage +// +// Memory is bounded by Limits (keys per scope, dedup IDs, window entries per +// key, reorder buffer, candidate buffer) and by the fixed pulse ring per +// periodic rule. When a limit cuts evidence, Stats counts it, the Result +// that saw it has CoverageReduced set and affected candidates carry +// CoverageFlags; time-driven and eviction transitions keep the traffic +// label and coverage flags of the episode they close. +// +// # Concurrency +// +// A Detector or Normalizer is not safe for concurrent use. +// Separate instances share no state. +package anomaly diff --git a/internal/anomaly/event.go b/internal/anomaly/event.go new file mode 100644 index 000000000..e081169f7 --- /dev/null +++ b/internal/anomaly/event.go @@ -0,0 +1,340 @@ +package anomaly + +import ( + "errors" + "fmt" + "sort" + "time" +) + +// TxID identifies one logical transmission (for CoreScope: the content +// hash). The library only compares it for equality and never interprets it. +type TxID string + +// MaxIDLen bounds TxID and observation IDs, so a caller cannot make the +// dedup set hold arbitrarily large strings. +const MaxIDLen = 256 + +// MaxPayloadBytes is the MeshCore payload limit (MAX_PACKET_PAYLOAD). +const MaxPayloadBytes = 184 + +// Valid event times: [minTime, maxTime), i.e. after the Unix epoch and +// before 2200. Times outside this range are rejected so nanosecond +// arithmetic can never overflow; the epoch itself (a typical unset-clock +// value) is rejected so zero can mean "never" internally. +var ( + minTime = time.Unix(0, 1).UTC() + maxTime = time.Date(2200, 1, 1, 0, 0, 0, 0, time.UTC) +) + +func validTime(t time.Time) bool { return !t.Before(minTime) && t.Before(maxTime) } + +// Event is one FINAL, normalized logical transmission. The detector counts +// each Event.ID at most once (within the dedup horizon) and never changes an +// event after Observe. Callers either build events themselves, having already +// finished normalization, or use Normalizer, which finalizes events by an +// explicit settle policy. +type Event struct { + ID TxID + // Time is the data timestamp of the transmission: the earliest included + // observation time. The detector never reads the wall clock. + Time time.Time + // FinalAt is when the event's evidence was complete (for Normalizer: + // first arrival + settle window). The detector's decision clock never + // runs behind the FinalAt of any event it was offered, so every + // candidate that uses this event reports DecidedAt >= FinalAt and a + // replay never claims an alarm earlier than the route evidence behind + // it existed. Zero means FinalAt == Time. + FinalAt time.Time + PayloadType PayloadType + // Channel is opaque; zero for payload types without a channel. + Channel ChannelHash + // Decryption is metadata only; it never affects keys or counting. + Decryption DecryptStatus + RouteClass RouteClass + RouteKind RouteKind + // FirstHop is set exactly when RouteKind == RouteKindRelay. + FirstHop Hop + // FullRoute is optional separate evidence; if set on a relay event its + // first hop must equal FirstHop. + FullRoute Route + // WireBytes is the size of the reference frame in bytes (not hex + // characters). 0 means unknown. + WireBytes int + // PayloadBytes is valid only when PayloadKnown is true. + PayloadBytes int + PayloadKnown bool + Traffic TrafficClass + // Normalization evidence (0 for caller-built events). + Observations int + ExcludedObservations int + // Truncated is set when normalization cut this transmission's evidence + // for capacity (fan-out cap or forced early finalization); its first hop + // may then rest on incomplete route evidence. + Truncated bool +} + +func (e *Event) finalAt() time.Time { + if e.FinalAt.IsZero() { + return e.Time + } + return e.FinalAt +} + +var errInvalidEvent = errors.New("anomaly: invalid event") + +// kindFitsClass: a relay or no-path kind needs a flood class, a direct kind +// a direct class; RouteClassMixed (observations disagree) allows any kind. +func kindFitsClass(k RouteKind, c RouteClass) bool { + switch k { + case RouteKindRelay, RouteKindNoPath: + return c.isFlood() || c == RouteClassMixed + case RouteKindDirect: + return c == RouteClassDirect || c == RouteClassTransportDirect || c == RouteClassMixed + } + return true +} + +// Validate checks the event contract. Error messages name the violated +// field only; they never echo IDs, hashes or routes. +func (e *Event) Validate() error { + fail := func(what string) error { return fmt.Errorf("%w: %s", errInvalidEvent, what) } + if len(e.ID) == 0 || len(e.ID) > MaxIDLen { + return fail("ID length") + } + if !validTime(e.Time) { + return fail("Time out of range") + } + if !e.FinalAt.IsZero() && (!validTime(e.FinalAt) || e.FinalAt.Before(e.Time)) { + return fail("FinalAt before Time or out of range") + } + if e.PayloadType > MaxPayloadType { + return fail("PayloadType") + } + if !e.RouteClass.valid() || !e.RouteKind.valid() || !e.Decryption.valid() || !e.Traffic.valid() { + return fail("enum field out of range") + } + if e.Channel.canonical() != nil || e.FirstHop.canonical() != nil { + return fail("non-canonical channel or hop") + } + if (e.RouteKind == RouteKindRelay) == e.FirstHop.IsZero() { + return fail("FirstHop must be set exactly for RouteKindRelay") + } + if e.RouteKind == RouteKindRelay && e.PayloadType == PayloadTypeTrace { + return fail("TRACE paths are not route evidence") + } + if !kindFitsClass(e.RouteKind, e.RouteClass) { + return fail("RouteKind inconsistent with RouteClass") + } + if e.FullRoute.Width() != 0 { + if e.RouteKind != RouteKindRelay || e.FullRoute.First() != e.FirstHop { + return fail("FullRoute inconsistent with FirstHop") + } + } + if e.WireBytes < 0 || e.WireBytes > MaxWireBytes { + return fail("WireBytes") + } + if e.PayloadKnown { + if e.PayloadBytes < 0 || e.PayloadBytes > MaxPayloadBytes { + return fail("PayloadBytes") + } + if e.WireBytes > 0 && e.PayloadBytes > e.WireBytes-2 { + return fail("PayloadBytes larger than the frame allows") + } + } else if e.PayloadBytes != 0 { + return fail("PayloadBytes set without PayloadKnown") + } + if e.Observations < 0 || e.ExcludedObservations < 0 { + return fail("observation counts") + } + return nil +} + +// Observation is one reception of a transmission by one observer. The +// observer itself is deliberately absent: it is never evidence of the +// sender or of the first relay. +type Observation struct { + TxID TxID + // ObsID is a stable identifier unique within the transmission (for + // CoreScope: the observation row id). It breaks timestamp ties. + ObsID string + // Time is the observation's data timestamp. + Time time.Time + // ArrivedAt is when the caller received the observation. Arrival order + // drives finalization; it may differ from Time. Zero means ArrivedAt == + // Time. + ArrivedAt time.Time + PayloadType PayloadType + Channel ChannelHash + Decryption DecryptStatus + RouteClass RouteClass + Path Route + WireBytes int + PayloadBytes int + PayloadKnown bool + Traffic TrafficClass +} + +func (o *Observation) arrived() time.Time { + if o.ArrivedAt.IsZero() { + return o.Time + } + return o.ArrivedAt +} + +// kind classifies the observation's route evidence. +func (o *Observation) kind() RouteKind { + if o.PayloadType == PayloadTypeTrace { + return RouteKindUnknown + } + switch { + case o.RouteClass.isFlood() && o.Path.Hops() > 0: + return RouteKindRelay + case o.RouteClass.isFlood(): + return RouteKindNoPath + case o.RouteClass == RouteClassDirect || o.RouteClass == RouteClassTransportDirect: + return RouteKindDirect + } + return RouteKindUnknown +} + +var errInvalidObservation = errors.New("anomaly: invalid observation") + +// Validate checks one observation. Messages never echo field values. +func (o *Observation) Validate() error { + fail := func(what string) error { return fmt.Errorf("%w: %s", errInvalidObservation, what) } + if len(o.TxID) == 0 || len(o.TxID) > MaxIDLen || len(o.ObsID) == 0 || len(o.ObsID) > MaxIDLen { + return fail("ID length") + } + if !validTime(o.Time) || (!o.ArrivedAt.IsZero() && !validTime(o.ArrivedAt)) { + return fail("time out of range") + } + if o.PayloadType > MaxPayloadType || !o.RouteClass.valid() || o.RouteClass == RouteClassMixed || + !o.Decryption.valid() || !o.Traffic.valid() { + return fail("enum field out of range") + } + if o.Channel.canonical() != nil { + return fail("non-canonical channel") + } + if o.WireBytes < 0 || o.WireBytes > MaxWireBytes { + return fail("WireBytes") + } + if o.PayloadKnown && (o.PayloadBytes < 0 || o.PayloadBytes > MaxPayloadBytes) { + return fail("PayloadBytes") + } + if !o.PayloadKnown && o.PayloadBytes != 0 { + return fail("PayloadBytes set without PayloadKnown") + } + if !fitsFrame(o, o.WireBytes) { + return fail("PayloadBytes larger than the frame allows") + } + if o.Path.w > MaxHopWidth || (o.Path.w == 0) != (len(o.Path.hops) == 0) || len(o.Path.hops) > MaxPathBytes { + return fail("path") + } + return nil +} + +// compatible reports whether two observations can belong to one +// transmission: a content hash covers the payload, so payload type, channel +// and (when both know it) payload size must agree. +func compatible(a, b *Observation) bool { + return a.PayloadType == b.PayloadType && a.Channel == b.Channel && + (!a.PayloadKnown || !b.PayloadKnown || a.PayloadBytes == b.PayloadBytes) && + fitsFrame(a, b.WireBytes) && fitsFrame(b, a.WireBytes) +} + +// fitsFrame reports whether o's known payload fits a frame of w bytes +// (header and path length bytes at least; w == 0 is unknown). +func fitsFrame(o *Observation, w int) bool { + return !o.PayloadKnown || w == 0 || o.PayloadBytes <= w-2 +} + +func sameObservation(a, b *Observation) bool { + return a.TxID == b.TxID && a.ObsID == b.ObsID && a.Time.Equal(b.Time) && + a.arrived().Equal(b.arrived()) && a.PayloadType == b.PayloadType && a.Channel == b.Channel && + a.PayloadKnown == b.PayloadKnown && a.PayloadBytes == b.PayloadBytes && + a.Decryption == b.Decryption && a.RouteClass == b.RouteClass && + a.Path.equal(b.Path) && a.WireBytes == b.WireBytes && a.Traffic == b.Traffic +} + +// obsLess orders observations by data time, then by ObsID. This is the +// order used to pick the reference observation. +func obsLess(a, b *Observation) bool { + if !a.Time.Equal(b.Time) { + return a.Time.Before(b.Time) + } + return a.ObsID < b.ObsID +} + +// buildEvent turns the included observations of one transmission into a +// final Event. obs must be non-empty, validated, share TxID and payload +// fields, and hold no duplicate ObsID. It does not modify obs. +func buildEvent(obs []*Observation, finalAt time.Time, excluded int) Event { + sorted := make([]*Observation, len(obs)) + copy(sorted, obs) + sort.Slice(sorted, func(i, j int) bool { return obsLess(sorted[i], sorted[j]) }) + + e := Event{ + ID: sorted[0].TxID, + Time: sorted[0].Time, + PayloadType: sorted[0].PayloadType, + Channel: sorted[0].Channel, + Observations: len(sorted), + ExcludedObservations: excluded, + } + // Reference observation: the earliest (Time, ObsID) with a relay path; + // else the earliest no-path flood; else the earliest direct; else the + // earliest overall. Hop width is kept as received. + var ref *Observation + for _, want := range []RouteKind{RouteKindRelay, RouteKindNoPath, RouteKindDirect} { + for _, o := range sorted { + if o.kind() == want { + ref = o + break + } + } + if ref != nil { + break + } + } + if ref == nil { + ref = sorted[0] + } + e.RouteKind = ref.kind() + e.RouteClass = ref.RouteClass + e.WireBytes = ref.WireBytes + if e.RouteKind == RouteKindRelay { + e.FirstHop = ref.Path.First() + e.FullRoute = Route{w: ref.Path.w, hops: append([]byte(nil), ref.Path.hops...)} + } + var classes uint8 + for _, o := range sorted { + if o.RouteClass != RouteClassUnknown { + classes |= 1 << o.RouteClass + } + switch o.Decryption { + case DecryptDecrypted: + e.Decryption = DecryptDecrypted + case DecryptNotDecrypted: + if e.Decryption != DecryptDecrypted { + e.Decryption = DecryptNotDecrypted + } + } + if o.Traffic > e.Traffic { + e.Traffic = o.Traffic + } + if o.PayloadKnown && !e.PayloadKnown { + e.PayloadBytes, e.PayloadKnown = o.PayloadBytes, true + } + } + if classes&(classes-1) != 0 { + // Observations disagree on the route class. Keep the reference + // hop as evidence but mark the class mixed (low confidence). + e.RouteClass = RouteClassMixed + } + e.FinalAt = finalAt + if e.FinalAt.Before(e.Time) { + e.FinalAt = e.Time + } + return e +} diff --git a/internal/anomaly/go.mod b/internal/anomaly/go.mod new file mode 100644 index 000000000..e001f2a7b --- /dev/null +++ b/internal/anomaly/go.mod @@ -0,0 +1,3 @@ +module github.com/meshcore-analyzer/anomaly + +go 1.22 diff --git a/internal/anomaly/helpers_test.go b/internal/anomaly/helpers_test.go new file mode 100644 index 000000000..e18213ca0 --- /dev/null +++ b/internal/anomaly/helpers_test.go @@ -0,0 +1,142 @@ +package anomaly + +import ( + "fmt" + "testing" + "time" +) + +// All fixtures are synthetic: an arbitrary base time in 2030, made-up +// channel and hop bytes, and thresholds that are named experimental test +// values, not recommendations. + +var t0 = time.Date(2030, 1, 7, 12, 0, 0, 0, time.UTC) + +func at(d time.Duration) time.Time { return t0.Add(d) } + +func mustHop(t testing.TB, b ...byte) Hop { + t.Helper() + h, err := NewHop(b) + if err != nil { + t.Fatal(err) + } + return h +} + +func mustChan(t testing.TB, b ...byte) ChannelHash { + t.Helper() + c, err := NewChannelHash(b) + if err != nil { + t.Fatal(err) + } + return c +} + +func mustRoute(t testing.TB, w int, b ...byte) Route { + t.Helper() + r, err := NewRoute(w, b) + if err != nil { + t.Fatal(err) + } + return r +} + +// relayEvent builds a final flood event on channel ch via first hop hop. +func relayEvent(t testing.TB, id string, when time.Time, ch byte, hop ...byte) Event { + t.Helper() + return Event{ID: TxID(id), Time: when, PayloadType: 5, Channel: mustChan(t, ch), + RouteClass: RouteClassFlood, RouteKind: RouteKindRelay, FirstHop: mustHop(t, hop...), + FullRoute: mustRoute(t, len(hop), append(append([]byte(nil), hop...), make([]byte, len(hop))...)...), + WireBytes: 60} +} + +// experimentalLimits are generous test limits (not recommendations). +func experimentalLimits() Limits { + return Limits{ + MaxKeysPerScope: 10_000, + StateTTL: 48 * time.Hour, + DedupHorizon: 6 * time.Hour, + MaxDedupIDs: 1_000_000, + ReorderDelay: 0, + MaxReorderBuffer: 1024, + MaxWindowEntries: 100_000, + MaxCandidatesPerCall: 100_000, + MaxPendingCandidates: 1_000_000, + } +} + +func quickState() StateParams { + return StateParams{ConfirmAfter: 0, EndAfterQuiet: 10 * time.Minute, Cooldown: 0} +} + +func rateRule(name string, s Scope, w time.Duration, threshold int) RateRule { + return RateRule{Name: name, Scope: s, Window: w, Threshold: threshold, State: quickState()} +} + +func newDet(t testing.TB, cfg Config) *Detector { + t.Helper() + if cfg.Limits == (Limits{}) { + cfg.Limits = experimentalLimits() + } + if cfg.Expected.Mode == 0 { + cfg.Expected.Mode = ExpectedInclude + } + d, err := New(cfg) + if err != nil { + t.Fatalf("New: %v", err) + } + return d +} + +// run feeds events in order, then flushes, and returns every candidate. +func run(t testing.TB, d *Detector, evs []Event) []Candidate { + t.Helper() + var out []Candidate + for i, e := range evs { + r := d.Observe(e) + if r.Status != StatusAccepted && r.Status != StatusDuplicate { + t.Fatalf("event %d: status %v err %v", i, r.Status, r.Err) + } + out = append(out, r.Candidates...) + } + for { + r := d.Flush() + out = append(out, r.Candidates...) + if !r.More { + break + } + } + return out +} + +func filter(cs []Candidate, rule string, to State) []Candidate { + var out []Candidate + for _, c := range cs { + if c.Rule == rule && c.To == to { + out = append(out, c) + } + } + return out +} + +func describe(cs []Candidate) string { + s := "" + for _, c := range cs { + s += fmt.Sprintf(" %s %s %s->%s %s at=%s\n", c.Rule, c.Key.Scope, c.From, c.To, c.Reason, c.At.Sub(t0)) + } + return s +} + +// series builds n events on one stream at start + i*step (+ jitter[i]). +func series(t testing.TB, prefix string, start time.Time, n int, step time.Duration, jitter func(i int) time.Duration, ch byte, hop ...byte) []Event { + t.Helper() + evs := make([]Event, 0, n) + for i := 0; i < n; i++ { + j := time.Duration(0) + if jitter != nil { + j = jitter(i) + } + evs = append(evs, relayEvent(t, fmt.Sprintf("%s-%04d", prefix, i), start.Add(time.Duration(i)*step+j), ch, hop...)) + } + return evs +} diff --git a/internal/anomaly/key.go b/internal/anomaly/key.go new file mode 100644 index 000000000..ad0c08559 --- /dev/null +++ b/internal/anomaly/key.go @@ -0,0 +1,367 @@ +package anomaly + +import ( + "encoding/hex" + "errors" + "strconv" + "strings" +) + +// Scope is the aggregation level a key and a rule work on. +type Scope uint8 + +const ( + scopeInvalid Scope = iota + // ScopeStream: payload type + channel hash + route class + route kind + + // first hop (with width). An observed route group, not an originator. + ScopeStream + // ScopeChannel: payload type + channel hash, all routes merged. + ScopeChannel + // ScopeRouteGroup: payload type + first hop (with width), all channels and + // flood route classes merged. Only transmissions with RouteKindRelay and + // a flood route class (not mixed or unknown) belong to a route group. + ScopeRouteGroup + // ScopeGlobal: payload type only. A global candidate is never attributed + // to a stream, channel or route. + ScopeGlobal + numScopes +) + +var scopeNames = [...]string{"invalid", "stream", "channel", "route", "global"} + +func (s Scope) String() string { + if int(s) < len(scopeNames) { + return scopeNames[s] + } + return "scope(" + strconv.Itoa(int(s)) + ")" +} + +func (s Scope) valid() bool { return s > scopeInvalid && s < numScopes } + +// Key identifies one series at one scope. Key is comparable and usable as a +// map key. Fields that do not belong to the key's scope are always zero, so +// two keys are equal exactly when their canonical encodings are equal. +type Key struct { + Scope Scope + PayloadType PayloadType + Channel ChannelHash + RouteClass RouteClass + RouteKind RouteKind + FirstHop Hop +} + +// StreamKeyOf returns the stream key of a finalized event. +func StreamKeyOf(e Event) Key { + k := Key{Scope: ScopeStream, PayloadType: e.PayloadType, Channel: e.Channel, + RouteClass: e.RouteClass, RouteKind: e.RouteKind} + if e.RouteKind == RouteKindRelay { + k.FirstHop = e.FirstHop + } + return k +} + +// ChannelKeyOf returns the channel key of a finalized event. +func ChannelKeyOf(e Event) Key { + return Key{Scope: ScopeChannel, PayloadType: e.PayloadType, Channel: e.Channel} +} + +// RouteGroupKeyOf returns the route-group key of a finalized event. ok is +// false unless the event has a relay first hop from a flood route class: +// no-path, direct, mixed and unknown transmissions never belong to a route +// group. +func RouteGroupKeyOf(e Event) (k Key, ok bool) { + if e.RouteKind != RouteKindRelay || e.FirstHop.IsZero() || !e.RouteClass.isFlood() { + return Key{}, false + } + return Key{Scope: ScopeRouteGroup, PayloadType: e.PayloadType, RouteKind: RouteKindRelay, FirstHop: e.FirstHop}, true +} + +// GlobalKeyOf returns the global key (payload type only). +func GlobalKeyOf(e Event) Key { + return Key{Scope: ScopeGlobal, PayloadType: e.PayloadType} +} + +// keyOf maps an event to the key of the given scope. +func keyOf(s Scope, e Event) (Key, bool) { + switch s { + case ScopeStream: + return StreamKeyOf(e), true + case ScopeChannel: + return ChannelKeyOf(e), true + case ScopeRouteGroup: + return RouteGroupKeyOf(e) + case ScopeGlobal: + return GlobalKeyOf(e), true + } + return Key{}, false +} + +// Confidence says how much a candidate's key says about routing. +type Confidence uint8 + +const ( + // ConfidenceAggregate: channel or global scope; no route claim at all. + ConfidenceAggregate Confidence = iota + 1 + // ConfidenceRouteObserved: a relay first hop was observed. It is still + // only observation evidence and may be globally ambiguous. + ConfidenceRouteObserved + // ConfidenceLow: no path, direct, mixed or unknown route; or the + // evidence itself is incomplete (for example a left-censored stream). + ConfidenceLow +) + +var confidenceNames = [...]string{"", "aggregate", "route_observed", "low"} + +func (c Confidence) String() string { + if int(c) < len(confidenceNames) && c > 0 { + return confidenceNames[c] + } + return "confidence(" + strconv.Itoa(int(c)) + ")" +} + +func (k Key) confidence() Confidence { + switch k.Scope { + case ScopeChannel, ScopeGlobal: + return ConfidenceAggregate + case ScopeRouteGroup: + return ConfidenceRouteObserved + } + if k.RouteKind == RouteKindRelay && k.RouteClass != RouteClassMixed && k.RouteClass != RouteClassUnknown { + return ConfidenceRouteObserved + } + return ConfidenceLow +} + +// Validate checks that k is canonical for its scope. +func (k Key) Validate() error { + if !k.Scope.valid() { + return errors.New("anomaly: key has invalid scope") + } + if k.PayloadType > MaxPayloadType { + return errors.New("anomaly: key payload type out of range") + } + if !k.RouteClass.valid() || !k.RouteKind.valid() { + return errors.New("anomaly: key route class or kind out of range") + } + if err := k.Channel.canonical(); err != nil { + return err + } + if err := k.FirstHop.canonical(); err != nil { + return err + } + switch k.Scope { + case ScopeStream: + if (k.RouteKind == RouteKindRelay) != !k.FirstHop.IsZero() { + return errors.New("anomaly: stream key must have a first hop exactly when its route kind is relay") + } + if !kindFitsClass(k.RouteKind, k.RouteClass) { + return errors.New("anomaly: stream key route kind inconsistent with its route class") + } + case ScopeChannel: + if k.RouteClass != 0 || k.RouteKind != 0 || !k.FirstHop.IsZero() { + return errors.New("anomaly: channel key must not carry route fields") + } + case ScopeRouteGroup: + if !k.Channel.IsZero() || k.RouteClass != 0 || k.RouteKind != RouteKindRelay || k.FirstHop.IsZero() { + return errors.New("anomaly: route key must carry only a relay first hop") + } + case ScopeGlobal: + if !k.Channel.IsZero() || k.RouteClass != 0 || k.RouteKind != 0 || !k.FirstHop.IsZero() { + return errors.New("anomaly: global key must carry only a payload type") + } + } + return nil +} + +func (h Hop) canonical() error { + if h.w > MaxHopWidth { + return errors.New("anomaly: hop width out of range") + } + for i := int(h.w); i < MaxHopWidth; i++ { + if h.b[i] != 0 { + return errors.New("anomaly: hop has bytes beyond its width") + } + } + return nil +} + +func (c ChannelHash) canonical() error { + if c.n > MaxChannelHashLen { + return errors.New("anomaly: channel hash length out of range") + } + for i := int(c.n); i < MaxChannelHashLen; i++ { + if c.b[i] != 0 { + return errors.New("anomaly: channel hash has bytes beyond its length") + } + } + return nil +} + +// String returns the canonical encoding, for example +// +// v1|s=stream|pt=5|ch=1:ab|rc=flood|rk=relay|hop=1:5c +// +// Every field is always present, lengths are explicit and hex is lower case, +// so the encoding is injective: two distinct valid keys never encode to the +// same string, and a 1-byte hop can never be read as the prefix of a wider +// one. The encoding holds only values the caller supplied. +func (k Key) String() string { + var b strings.Builder + b.Grow(48) + b.WriteString("v1|s=") + b.WriteString(k.Scope.String()) + b.WriteString("|pt=") + b.WriteString(strconv.Itoa(int(k.PayloadType))) + b.WriteString("|ch=") + b.WriteString(k.Channel.encode()) + b.WriteString("|rc=") + b.WriteString(k.RouteClass.String()) + b.WriteString("|rk=") + b.WriteString(k.RouteKind.String()) + b.WriteString("|hop=") + b.WriteString(k.FirstHop.encode()) + return b.String() +} + +var errBadKeyEncoding = errors.New("anomaly: not a canonical key encoding") + +// ParseKey parses the canonical encoding produced by Key.String. It rejects +// every non-canonical form, so ParseKey(s) succeeds only if +// ParseKey(s).String() == s. +func ParseKey(s string) (Key, error) { + parts := strings.Split(s, "|") + if len(parts) != 7 || parts[0] != "v1" { + return Key{}, errBadKeyEncoding + } + field := func(i int, name string) (string, bool) { + p := parts[i] + if !strings.HasPrefix(p, name+"=") { + return "", false + } + return p[len(name)+1:], true + } + var k Key + sv, ok := field(1, "s") + if !ok { + return Key{}, errBadKeyEncoding + } + for i, n := range scopeNames { + if i > 0 && n == sv { + k.Scope = Scope(i) + } + } + pv, ok := field(2, "pt") + if !ok { + return Key{}, errBadKeyEncoding + } + pt, err := strconv.ParseUint(pv, 10, 8) + if err != nil || strconv.FormatUint(pt, 10) != pv || pt > uint64(MaxPayloadType) { + return Key{}, errBadKeyEncoding + } + k.PayloadType = PayloadType(pt) + cv, ok := field(3, "ch") + if !ok { + return Key{}, errBadKeyEncoding + } + cb, err := decodeLenHex(cv, MaxChannelHashLen) + if err != nil { + return Key{}, errBadKeyEncoding + } + if cb != nil { + k.Channel, _ = NewChannelHash(cb) + } + rv, ok := field(4, "rc") + if !ok { + return Key{}, errBadKeyEncoding + } + found := false + for i, n := range routeClassNames { + if n == rv { + k.RouteClass, found = RouteClass(i), true + } + } + if !found { + return Key{}, errBadKeyEncoding + } + kv, ok := field(5, "rk") + if !ok { + return Key{}, errBadKeyEncoding + } + found = false + for i, n := range routeKindNames { + if n == kv { + k.RouteKind, found = RouteKind(i), true + } + } + if !found { + return Key{}, errBadKeyEncoding + } + hv, ok := field(6, "hop") + if !ok { + return Key{}, errBadKeyEncoding + } + hb, err := decodeLenHex(hv, MaxHopWidth) + if err != nil { + return Key{}, errBadKeyEncoding + } + if hb != nil { + k.FirstHop, _ = NewHop(hb) + } + if err := k.Validate(); err != nil { + return Key{}, errBadKeyEncoding + } + if k.String() != s { + return Key{}, errBadKeyEncoding + } + return k, nil +} + +// decodeLenHex decodes "-" (nil) or ":<2n lower-case hex>". +func decodeLenHex(v string, max int) ([]byte, error) { + if v == "-" { + return nil, nil + } + i := strings.IndexByte(v, ':') + if i != 1 { + return nil, errBadKeyEncoding + } + n := int(v[0] - '0') + if n < 1 || n > max { + return nil, errBadKeyEncoding + } + h := v[2:] + if len(h) != 2*n || strings.ToLower(h) != h { + return nil, errBadKeyEncoding + } + b, err := hex.DecodeString(h) + if err != nil { + return nil, errBadKeyEncoding + } + return b, nil +} + +// less orders keys deterministically (by canonical encoding fields). +func (k Key) less(o Key) bool { + if k.Scope != o.Scope { + return k.Scope < o.Scope + } + if k.PayloadType != o.PayloadType { + return k.PayloadType < o.PayloadType + } + if k.Channel.n != o.Channel.n { + return k.Channel.n < o.Channel.n + } + if k.Channel.b != o.Channel.b { + return string(k.Channel.b[:]) < string(o.Channel.b[:]) + } + if k.RouteClass != o.RouteClass { + return k.RouteClass < o.RouteClass + } + if k.RouteKind != o.RouteKind { + return k.RouteKind < o.RouteKind + } + if k.FirstHop.w != o.FirstHop.w { + return k.FirstHop.w < o.FirstHop.w + } + return string(k.FirstHop.b[:]) < string(o.FirstHop.b[:]) +} diff --git a/internal/anomaly/key_test.go b/internal/anomaly/key_test.go new file mode 100644 index 000000000..c66348085 --- /dev/null +++ b/internal/anomaly/key_test.go @@ -0,0 +1,146 @@ +package anomaly + +import ( + "strings" + "testing" +) + +func TestHopWidthIsPartOfTheValue(t *testing.T) { + one := mustHop(t, 0x5C) + two := mustHop(t, 0x5C, 0x12) + three := mustHop(t, 0x5C, 0x12, 0x34) + if one == two || two == three || one == three { + t.Fatal("hops of different width must differ") + } + e1 := relayEvent(t, "a", t0, 0x11, 0x5C) + e2 := relayEvent(t, "b", t0, 0x11, 0x5C, 0x12) + e3 := relayEvent(t, "c", t0, 0x11, 0x5C, 0x12, 0x34) + k1, k2, k3 := StreamKeyOf(e1), StreamKeyOf(e2), StreamKeyOf(e3) + if k1 == k2 || k2 == k3 || k1 == k3 { + t.Fatal("1-byte hop must not share a stream key with a wider hop that starts with the same byte") + } + for _, k := range []Key{k1, k2, k3} { + if strings.Count(k.String(), "|") != 6 { + t.Fatalf("encoding shape: %s", k) + } + } + if k1.String() == k2.String() || strings.HasPrefix(k2.String(), k1.String()) { + t.Fatalf("encodings collide or prefix: %q %q", k1, k2) + } + r1, _ := RouteGroupKeyOf(e1) + r2, _ := RouteGroupKeyOf(e2) + if r1 == r2 { + t.Fatal("route-group keys must keep the hop width") + } +} + +func TestKeyEncodingRoundTrip(t *testing.T) { + e := relayEvent(t, "a", t0, 0xAB, 0x01, 0x02) + nopath := Event{ID: "n", Time: t0, PayloadType: 5, Channel: mustChan(t, 0xAB), RouteClass: RouteClassFlood, RouteKind: RouteKindNoPath} + direct := Event{ID: "d", Time: t0, PayloadType: 2, RouteClass: RouteClassDirect, RouteKind: RouteKindDirect} + rg, _ := RouteGroupKeyOf(e) + keys := []Key{StreamKeyOf(e), ChannelKeyOf(e), rg, GlobalKeyOf(e), StreamKeyOf(nopath), StreamKeyOf(direct), ChannelKeyOf(direct)} + seen := map[string]bool{} + for _, k := range keys { + if err := k.Validate(); err != nil { + t.Fatalf("%s: %v", k, err) + } + s := k.String() + if seen[s] { + t.Fatalf("duplicate encoding %s", s) + } + seen[s] = true + back, err := ParseKey(s) + if err != nil || back != k { + t.Fatalf("round trip %s: %v", s, err) + } + } + want := "v1|s=stream|pt=5|ch=1:ab|rc=flood|rk=relay|hop=2:0102" + if got := StreamKeyOf(e).String(); got != want { + t.Fatalf("encoding changed: %s", got) + } +} + +func TestParseKeyRejectsNonCanonical(t *testing.T) { + good := "v1|s=stream|pt=5|ch=1:ab|rc=flood|rk=relay|hop=1:5c" + if _, err := ParseKey(good); err != nil { + t.Fatal(err) + } + for _, bad := range []string{ + "v1|s=stream|pt=5|ch=1:AB|rc=flood|rk=relay|hop=1:5c", // upper-case hex + "v1|s=stream|pt=05|ch=1:ab|rc=flood|rk=relay|hop=1:5c", // leading zero + "v1|s=stream|pt=5|ch=2:ab|rc=flood|rk=relay|hop=1:5c", // length mismatch + "v1|s=stream|pt=5|ch=1:ab|rc=flood|rk=relay|hop=-", // relay without hop + "v1|s=stream|pt=5|ch=1:ab|rc=flood|rk=nopath|hop=1:5c", // hop on no-path + "v1|s=global|pt=5|ch=1:ab|rc=unknown|rk=unknown|hop=-", // global with channel + "v1|s=stream|pt=16|ch=1:ab|rc=flood|rk=relay|hop=1:5c", // pt out of range + "v1|s=stream|pt=5|ch=1:ab|rk=relay|rc=flood|hop=1:5c", // field order + "v2|s=stream|pt=5|ch=1:ab|rc=flood|rk=relay|hop=1:5c", + "v1|s=stream|pt=5|ch=1:ab|rc=flood|rk=relay|hop=4:5c000000", + "", + } { + if _, err := ParseKey(bad); err == nil { + t.Errorf("accepted non-canonical %q", bad) + } + } +} + +func TestFullRouteDoesNotFragmentStreamKey(t *testing.T) { + a := relayEvent(t, "a", t0, 0x11, 0x22) + b := a + b.ID = "b" + b.FullRoute = mustRoute(t, 1, 0x22, 0x33, 0x44, 0x55) + if StreamKeyOf(a) != StreamKeyOf(b) { + t.Fatal("different full routes with the same first hop must share the stream key") + } + if a.FullRoute.equal(b.FullRoute) { + t.Fatal("fixture: full routes should differ") + } +} + +func TestNoPathAndDirectAreLowConfidence(t *testing.T) { + nopath := Event{ID: "n", Time: t0, PayloadType: 5, Channel: mustChan(t, 0x11), RouteClass: RouteClassFlood, RouteKind: RouteKindNoPath} + direct := Event{ID: "d", Time: t0, PayloadType: 5, Channel: mustChan(t, 0x11), RouteClass: RouteClassDirect, RouteKind: RouteKindDirect} + for _, e := range []Event{nopath, direct} { + if err := e.Validate(); err != nil { + t.Fatal(err) + } + if c := StreamKeyOf(e).confidence(); c != ConfidenceLow { + t.Fatalf("%v: confidence %v, want low", e.RouteKind, c) + } + if _, ok := RouteGroupKeyOf(e); ok { + t.Fatalf("%v must not form a route group", e.RouteKind) + } + } + if c := StreamKeyOf(relayEvent(t, "r", t0, 0x11, 0x22)).confidence(); c != ConfidenceRouteObserved { + t.Fatalf("relay confidence %v", c) + } + mixed := relayEvent(t, "m", t0, 0x11, 0x22) + mixed.RouteClass = RouteClassMixed + if c := StreamKeyOf(mixed).confidence(); c != ConfidenceLow { + t.Fatalf("mixed route class must be low confidence, got %v", c) + } +} + +func TestGlobalKeyCarriesNoAttribution(t *testing.T) { + k := GlobalKeyOf(relayEvent(t, "a", t0, 0x11, 0x22)) + if !k.Channel.IsZero() || !k.FirstHop.IsZero() || k.RouteKind != 0 || k.RouteClass != 0 { + t.Fatalf("global key carries attribution: %s", k) + } + if k.confidence() != ConfidenceAggregate { + t.Fatal("global key must be aggregate confidence") + } +} + +func TestChannelHashIsOpaqueAndNoChannelDiffersFromZeroByte(t *testing.T) { + zero := mustChan(t, 0x00) + if zero.IsZero() { + t.Fatal("a 1-byte hash 0x00 is a channel, not 'no channel'") + } + e := Event{ID: "x", Time: t0, PayloadType: 2, RouteClass: RouteClassDirect, RouteKind: RouteKindDirect} + f := e + f.ID, f.Channel = "y", zero + if ChannelKeyOf(e) == ChannelKeyOf(f) { + t.Fatal("no-channel and channel 0x00 must differ") + } +} diff --git a/internal/anomaly/newstream.go b/internal/anomaly/newstream.go new file mode 100644 index 000000000..c15c380c7 --- /dev/null +++ b/internal/anomaly/newstream.go @@ -0,0 +1,60 @@ +package anomaly + +import "time" + +// New-stream volume. +// +// An episode of a key starts at its first observed event, or at the first +// event after QuietPeriod of silence (reactivation). The rule counts events +// causally in [start, start+Window) and fires once, at the event that brings +// the count to MinCount. +// +// Left censoring: a key first seen within CensorWindow of the history start +// may have been active before the data begins, so it is reported with +// Censored=true and low confidence, never as a certain new stream. This is +// what keeps a restart or a replay from calling old streams new. +// +// Limitation: a key evicted by StateTTL (idle longer than StateTTL >= +// QuietPeriod) and seen again later is indistinguishable from a new key; it +// is reported as new, not reactivated. Capacity evictions are flagged on +// candidates via CoverageKeyEvictions. +type newStreamState struct { + start, lastSeen int64 + count, exp, mon uint32 + begun bool + censored, reactivate bool + fired bool +} + +func (s *newStreamState) observe(t int64, cls TrafficClass, history int64, r *NewStreamRule) (*NewStreamEvidence, TrafficLabel, bool) { + switch { + case !s.begun: + *s = newStreamState{begun: true, start: t, censored: t-history < int64(r.CensorWindow)} + case t-s.lastSeen >= int64(r.QuietPeriod): + *s = newStreamState{begun: true, start: t, reactivate: true} + } + s.lastSeen = t + if t-s.start >= int64(r.Window) { + return nil, 0, false + } + s.count++ + switch cls { + case TrafficExpected: + s.exp++ + case TrafficMonitored: + s.mon++ + } + if s.fired || s.count < uint32(r.MinCount) { + return nil, 0, false + } + s.fired = true + ev := &NewStreamEvidence{ + Window: r.Window, + MinCount: r.MinCount, + Count: int(s.count), + Age: time.Duration(t - s.start), + Censored: s.censored, + Reactivated: s.reactivate, + } + return ev, labelFromCounts(uint64(s.count), uint64(s.exp), uint64(s.mon)), true +} diff --git a/internal/anomaly/newstream_state_test.go b/internal/anomaly/newstream_state_test.go new file mode 100644 index 000000000..93eb5ce8a --- /dev/null +++ b/internal/anomaly/newstream_state_test.go @@ -0,0 +1,271 @@ +package anomaly + +import ( + "fmt" + "reflect" + "testing" + "time" +) + +func nsRule(minCount int) NewStreamRule { + return NewStreamRule{Name: "ns", Scope: ScopeStream, Window: time.Hour, MinCount: minCount, + CensorWindow: 12 * time.Hour, QuietPeriod: 12 * time.Hour, State: quickState()} +} + +// historyThen returns a config whose history starts a day before t0, so +// streams appearing at t0 are not censored. +func historyThen(r NewStreamRule) Config { + return Config{NewStream: []NewStreamRule{r}, HistoryStart: t0.Add(-24 * time.Hour)} +} + +func TestNewStreamGenuinelyNew(t *testing.T) { + evs := series(t, "n", t0, 30, 30*time.Second, nil, 0x11, 0x22) + cs := run(t, newDet(t, historyThen(nsRule(30))), evs) + a := filter(cs, "ns", StateActive) + if len(a) != 1 { + t.Fatalf("new stream must fire once:\n%s", describe(cs)) + } + ev := a[0].NewStream + if ev.Censored || ev.Reactivated || ev.Count != 30 || ev.Age != 29*30*time.Second || !a[0].At.Equal(evs[29].Time) { + t.Fatalf("evidence %+v at %v", ev, a[0].At) + } + if a[0].Confidence != ConfidenceRouteObserved { + t.Fatalf("confidence %v", a[0].Confidence) + } +} + +func TestNewStreamLeftCensoredIsNeverCertainlyNew(t *testing.T) { + // history starts at t0: the stream may predate the data + cfg := Config{NewStream: []NewStreamRule{nsRule(30)}, HistoryStart: t0} + evs := series(t, "n", t0.Add(time.Minute), 30, 30*time.Second, nil, 0x11, 0x22) + cs := run(t, newDet(t, cfg), evs) + a := filter(cs, "ns", StateActive) + if len(a) != 1 || !a[0].NewStream.Censored || a[0].Confidence != ConfidenceLow { + t.Fatalf("left-censored stream must be marked censored / low confidence:\n%s", describe(cs)) + } + // zero HistoryStart = first processed event: the same holds + cs = run(t, newDet(t, Config{NewStream: []NewStreamRule{nsRule(30)}}), evs) + if a := filter(cs, "ns", StateActive); len(a) != 1 || !a[0].NewStream.Censored { + t.Fatal("stream at the start of the data must be censored") + } +} + +func TestNewStreamRestartReplayDoesNotCallOldStreamsNew(t *testing.T) { + // A stream active for days; a restarted detector replays history from a + // start point in the middle of it. It must be censored, not new. + replayStart := t0.Add(48 * time.Hour) + evs := series(t, "old", replayStart, 40, 20*time.Second, nil, 0x11, 0x22) + cs := run(t, newDet(t, Config{NewStream: []NewStreamRule{nsRule(30)}, HistoryStart: replayStart}), evs) + for _, c := range filter(cs, "ns", StateActive) { + if !c.NewStream.Censored { + t.Fatalf("restart replay called an old stream new: %+v", c.NewStream) + } + } +} + +func TestNewStream29Versus30(t *testing.T) { + for _, n := range []int{29, 30} { + cs := run(t, newDet(t, historyThen(nsRule(30))), series(t, "n", t0, n, time.Minute, nil, 1, 2)) + if got := len(filter(cs, "ns", StateActive)); (n == 30) != (got == 1) { + t.Fatalf("%d events in the first hour: %d activations", n, got) + } + } +} + +func TestNewStreamOnlyCountsTheFirstWindow(t *testing.T) { + // 29 events in the first hour, the 30th just after it: not new-volume + evs := series(t, "n", t0, 29, 2*time.Minute, nil, 1, 2) // last at 56 min + evs = append(evs, relayEvent(t, "late", t0.Add(time.Hour), 1, 2)) + cs := run(t, newDet(t, historyThen(nsRule(30))), evs) + if len(cs) != 0 { + t.Fatalf("an event at exactly start+Window is outside [start, start+Window):\n%s", describe(cs)) + } +} + +func TestNewStreamReactivationAfterSilenceAndNoRepeats(t *testing.T) { + r := nsRule(5) + var evs []Event + evs = append(evs, series(t, "a", t0, 10, time.Minute, nil, 1, 2)...) // fires once at #5 + evs = append(evs, series(t, "b", t0.Add(13*time.Hour), 10, time.Minute, nil, 1, 2)...) + cs := run(t, newDet(t, historyThen(r)), evs) + act := filter(cs, "ns", StateActive) + if len(act) != 2 { + t.Fatalf("one activation per episode expected:\n%s", describe(cs)) + } + if act[0].NewStream.Reactivated || !act[1].NewStream.Reactivated { + t.Fatalf("reactivation flags: %+v %+v", act[0].NewStream, act[1].NewStream) + } + if act[0].EventID == act[1].EventID { + t.Fatal("a reactivation after the cooldown is a new episode with a new EventID") + } + // no reactivation for a gap shorter than QuietPeriod + evs = append(series(t, "a", t0, 10, time.Minute, nil, 1, 2), series(t, "c", t0.Add(6*time.Hour), 10, time.Minute, nil, 1, 2)...) + cs = run(t, newDet(t, historyThen(r)), evs) + if len(filter(cs, "ns", StateActive)) != 1 { + t.Fatalf("gap below QuietPeriod must not reactivate:\n%s", describe(cs)) + } +} + +// ---- state machine ---- + +func TestLegalTransitions(t *testing.T) { + legal := map[[3]int]bool{ + {int(StateNormal), int(StateSuspicious), int(ReasonThresholdCrossed)}: true, + {int(StateSuspicious), int(StateActive), int(ReasonConfirmed)}: true, + {int(StateSuspicious), int(StateNormal), int(ReasonNotConfirmed)}: true, + {int(StateSuspicious), int(StateNormal), int(ReasonEvicted)}: true, + {int(StateActive), int(StateEnded), int(ReasonQuiet)}: true, + {int(StateActive), int(StateEnded), int(ReasonEvicted)}: true, + {int(StateEnded), int(StateActive), int(ReasonReopened)}: true, + } + for from := StateNormal; from <= StateEnded; from++ { + for to := StateNormal; to <= StateEnded; to++ { + for r := Reason(0); r <= ReasonEvicted+1; r++ { + want := legal[[3]int{int(from), int(to), int(r)}] + if got := legalTransition(from, to, r); got != want { + t.Fatalf("%v->%v (%v): legal=%v want %v", from, to, r, got, want) + } + } + } + } +} + +func TestStateMachineFullCycle(t *testing.T) { + p := StateParams{ConfirmAfter: 2 * time.Minute, EndAfterQuiet: 10 * time.Minute, Cooldown: 30 * time.Minute} + r := RateRule{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 1, State: p} + var evs []Event + add := func(id string, d time.Duration) { evs = append(evs, relayEvent(t, id, at(d), 1, 2)) } + add("a", 0) // suspicious + add("b", time.Minute) // still suspicious (1 min < confirm 2 min) + add("c", 3*time.Minute) // active + add("d", 20*time.Minute) // ended at 13m (quiet), reopened at 20m (within cooldown) + add("e", 100*time.Minute) // ended at 30m; cooldown over at 60m; new episode + add("f", 100*time.Minute+30) // same second-ish, same episode + d := newDet(t, Config{Rate: []RateRule{r}}) + cs := run(t, d, evs) + cs = append(cs, d.Advance(at(10*time.Hour)).Candidates...) + type tr struct { + from, to State + reason Reason + at time.Duration + } + var got []tr + for _, c := range cs { + got = append(got, tr{c.From, c.To, c.Reason, c.At.Sub(t0)}) + } + want := []tr{ + {StateNormal, StateSuspicious, ReasonThresholdCrossed, 0}, + {StateSuspicious, StateActive, ReasonConfirmed, 3 * time.Minute}, + {StateActive, StateEnded, ReasonQuiet, 13 * time.Minute}, + {StateEnded, StateActive, ReasonReopened, 20 * time.Minute}, + {StateActive, StateEnded, ReasonQuiet, 30 * time.Minute}, + {StateNormal, StateSuspicious, ReasonThresholdCrossed, 100 * time.Minute}, + } + want = append(want, tr{StateSuspicious, StateNormal, ReasonNotConfirmed, 110*time.Minute + 30}) + if !reflect.DeepEqual(got, want) { + t.Fatalf("transitions:\n%s", describe(cs)) + } + if cs[0].EventID != cs[3].EventID || cs[3].EventID == cs[5].EventID { + t.Fatal("reopen within cooldown keeps the EventID; a new episode after cooldown gets a new one") + } + if cs[2].Summary == nil || cs[2].Summary.HotSignals != 3 { + t.Fatalf("ended candidate summary %+v", cs[2].Summary) + } + // a quiet end is decided no earlier than its own data time and never + // before the transitions that opened the episode + if cs[2].DecidedAt.Before(cs[2].At) || cs[2].DecidedAt.Before(cs[1].DecidedAt) { + t.Fatalf("quiet end DecidedAt %v (At %v, opened %v)", cs[2].DecidedAt, cs[2].At, cs[1].DecidedAt) + } +} + +func TestSuspiciousWithoutConfirmationReturnsToNormal(t *testing.T) { + p := StateParams{ConfirmAfter: 5 * time.Minute, EndAfterQuiet: 2 * time.Minute} + r := RateRule{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 1, State: p} + cs := run(t, newDet(t, Config{Rate: []RateRule{r}}), []Event{ + relayEvent(t, "a", t0, 1, 2), relayEvent(t, "b", at(10*time.Minute), 1, 2)}) + if len(cs) < 2 || cs[1].From != StateSuspicious || cs[1].To != StateNormal || cs[1].Reason != ReasonNotConfirmed || + !cs[1].At.Equal(at(2*time.Minute)) { + t.Fatalf("unconfirmed episode:\n%s", describe(cs)) + } +} + +func TestCooldownSuppressesRepeatedAlerts(t *testing.T) { + p := StateParams{EndAfterQuiet: time.Minute, Cooldown: time.Hour} + r := RateRule{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 1, State: p} + var evs []Event + for i := 0; i < 5; i++ { // bursts every 10 minutes + evs = append(evs, relayEvent(t, fmt.Sprint(i), at(time.Duration(i)*10*time.Minute), 1, 2)) + } + cs := run(t, newDet(t, Config{Rate: []RateRule{r}}), evs) + if n := len(filter(cs, "r", StateSuspicious)); n != 1 { + t.Fatalf("cooldown must keep one episode, got %d openings:\n%s", n, describe(cs)) + } + ids := map[string]bool{} + for _, c := range cs { + ids[c.EventID] = true + } + if len(ids) != 1 { + t.Fatalf("one stable EventID expected, got %d", len(ids)) + } +} + +func TestEventIDsAreStableAcrossReplays(t *testing.T) { + evs := series(t, "s", t0, 40, 10*time.Second, nil, 1, 2) + cfg := Config{Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 5)}} + a := run(t, newDet(t, cfg), evs) + b := run(t, newDet(t, cfg), evs) + if len(a) == 0 || !reflect.DeepEqual(a, b) { + t.Fatal("identical input and config must give identical candidates") + } +} + +func TestObserveDoesNotAliasCallerData(t *testing.T) { + e := relayEvent(t, "a", t0, 1, 2) + hops := e.FullRoute.hops + d := newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 1)}}) + cs := run(t, d, []Event{e}) + hops[0] ^= 0xFF // caller mutates its own buffer afterwards + if cs[0].TriggerRoute.hops[0] == hops[0] { + t.Fatal("candidate aliases the caller's route buffer") + } +} + +func TestEpisodeTimeTransitionsUseTheirOwnDataTime(t *testing.T) { + // the transition time must not depend on when time was advanced + p := StateParams{EndAfterQuiet: 10 * time.Minute, Cooldown: 5 * time.Minute} + var ep episode + start := t0.UnixNano() + ep.onHot(start, &p, nil) + trs := ep.onTime(start+int64(3*time.Hour), &p, nil) + if len(trs) != 1 || trs[0].reason != ReasonQuiet || trs[0].at != start+int64(10*time.Minute) { + t.Fatalf("quiet end stamped %v, want lastHot+quiet", trs) + } + if ep.state != StateNormal { + t.Fatalf("cooldown also expired: state %v", ep.state) + } +} + +func TestQuietEndAtExactlyStateTTLIsNotAnEviction(t *testing.T) { + lim := experimentalLimits() + lim.StateTTL = 10 * time.Minute + r := RateRule{Name: "r", Scope: ScopeStream, Window: time.Minute, Threshold: 1, State: StateParams{EndAfterQuiet: 10 * time.Minute}} + cs := run(t, newDet(t, Config{Limits: lim, Rate: []RateRule{r}}), + []Event{relayEvent(t, "a", t0, 1, 2), relayEvent(t, "b", at(10*time.Minute), 3, 4)}) + e := filter(cs, "r", StateEnded) + if len(e) == 0 || e[0].Reason != ReasonQuiet { + t.Fatalf("an episode whose quiet end coincides with StateTTL must end as quiet:\n%s", describe(cs)) + } +} + +func TestNewStreamSignalsOncePerEpisode(t *testing.T) { + // 20 events a minute apart: the rule is hot once (at the 5th); later + // events in the same episode must not keep the episode alive + evs := series(t, "n", t0, 20, time.Minute, nil, 1, 2) + d := newDet(t, historyThen(nsRule(5))) + cs := run(t, d, evs) + cs = append(cs, d.Advance(at(3*time.Hour)).Candidates...) + e := filter(cs, "ns", StateEnded) + if len(e) != 1 || !e[0].At.Equal(evs[4].Time.Add(10*time.Minute)) || e[0].Summary.HotSignals != 1 { + t.Fatalf("new-stream episode must end quiet after its single signal:\n%s", describe(cs)) + } +} diff --git a/internal/anomaly/normalizer.go b/internal/anomaly/normalizer.go new file mode 100644 index 000000000..7dd059aa9 --- /dev/null +++ b/internal/anomaly/normalizer.go @@ -0,0 +1,412 @@ +package anomaly + +import ( + "container/heap" + "errors" + "fmt" + "sort" + "time" +) + +// NormalizerConfig bounds the Normalizer. Every field is required. +// +// Finality policy: a transmission is final at (first arrival + SettleWindow). +// Observations that arrive at or before that deadline are included, even if +// their data timestamp is earlier than observations already seen (which can +// change the chosen first hop while the event is still pending). The event +// is released only once the arrival watermark passes the deadline. An +// observation of a transmission that is already final is rejected as late +// and never changes the released event, so a replay can never use route +// evidence that arrived after the event was decided. +type NormalizerConfig struct { + // SettleWindow is how long after its first observation arrives a + // transmission may still collect observations. + SettleWindow time.Duration + // MaxPendingTransmissions caps transmissions waiting for their deadline. + // When full, the pending transmission with the earliest deadline is + // finalized early (counted as a forced finalization). + MaxPendingTransmissions int + // MaxObservationsPerTransmission caps the observation fan-out kept per + // transmission. Further observations are dropped and counted. + MaxObservationsPerTransmission int + // FinalizedHorizon is how long (in arrival time) finalized IDs are + // remembered to reject late observations. After it, a late observation + // would start a new pending transmission with the same ID, which the + // Detector's own dedup must then catch. + FinalizedHorizon time.Duration + // MaxFinalizedIDs caps the remembered finalized IDs. Evicting one before + // its horizon is counted as reduced coverage. + MaxFinalizedIDs int +} + +// Bounds on the Normalizer's durations (they also keep nanosecond +// arithmetic far from overflow). +const ( + maxSettleWindow = 24 * time.Hour + maxFinalizedHorizon = 30 * 24 * time.Hour +) + +// Validate checks that every limit is set explicitly. +func (c NormalizerConfig) Validate() error { + switch { + case c.SettleWindow <= 0 || c.SettleWindow > maxSettleWindow: + return errors.New("anomaly: NormalizerConfig.SettleWindow must be in (0, 24h]") + case c.FinalizedHorizon > maxFinalizedHorizon: + return errors.New("anomaly: NormalizerConfig.FinalizedHorizon must be <= 30 days") + case c.MaxPendingTransmissions <= 0: + return errors.New("anomaly: NormalizerConfig.MaxPendingTransmissions must be > 0") + case c.MaxObservationsPerTransmission <= 0: + return errors.New("anomaly: NormalizerConfig.MaxObservationsPerTransmission must be > 0") + case c.FinalizedHorizon < c.SettleWindow: + return errors.New("anomaly: NormalizerConfig.FinalizedHorizon must be >= SettleWindow") + case c.MaxFinalizedIDs <= 0: + return errors.New("anomaly: NormalizerConfig.MaxFinalizedIDs must be > 0") + } + return nil +} + +// ObsStatus is the outcome of Normalizer.Add. +type ObsStatus uint8 + +const ( + ObsAccepted ObsStatus = iota + 1 + ObsDuplicate // identical observation already held + ObsConflict // same TxID/ObsID with different content, or payload mismatch with any held observation + ObsLate // the transmission is already final + ObsInvalid // failed Observation.Validate + ObsOutOfArrivalOrder // ArrivedAt earlier than the arrival watermark + ObsDroppedFanout // MaxObservationsPerTransmission reached +) + +var obsStatusNames = [...]string{"", "accepted", "duplicate", "conflict", "late", "invalid", "out_of_arrival_order", "dropped_fanout"} + +func (s ObsStatus) String() string { + if int(s) < len(obsStatusNames) && s > 0 { + return obsStatusNames[s] + } + return fmt.Sprintf("obsstatus(%d)", uint8(s)) +} + +// NormalizeResult is returned by Normalizer.Add, Advance and Flush. +type NormalizeResult struct { + Status ObsStatus // zero for Advance/Flush + Err error // set for ObsInvalid + // Events are the transmissions finalized by this call, in deadline order. + Events []Event + // CoverageReduced is true when a capacity limit dropped or truncated + // evidence during this call. + CoverageReduced bool +} + +// NormalizerStats are cumulative counters. +type NormalizerStats struct { + Accepted, Duplicates, Conflicts, Late, Invalid, OutOfArrivalOrder uint64 + DroppedFanout uint64 + ForcedFinalizations uint64 + FinalizedIDsEvictedEarly uint64 + EventsFinalized uint64 + Pending, FinalizedIDs int +} + +// CoverageReduced reports whether any capacity limit has cut evidence. +func (s NormalizerStats) CoverageReduced() bool { + return s.DroppedFanout > 0 || s.ForcedFinalizations > 0 || s.FinalizedIDsEvictedEarly > 0 +} + +type pendingTx struct { + id TxID + firstArrival int64 + deadline int64 + obs []*Observation + idx int // heap index + // payload is the first observation with a known payload size; every + // later known size must agree with it + payload *Observation + // minWire is the smallest known frame size held; a known payload must + // fit every held frame + minWire int + dropped int // observations dropped by the fan-out cap + truncated bool // fan-out drop or forced early finalization +} + +func (p *pendingTx) compatible(o *Observation) bool { + if !compatible(p.obs[0], o) || !fitsFrame(o, p.minWire) { + return false + } + return p.payload == nil || compatible(p.payload, o) +} + +func (p *pendingTx) add(o *Observation) { + p.obs = append(p.obs, o) + if p.payload == nil && o.PayloadKnown { + p.payload = o + } + if o.WireBytes > 0 && (p.minWire == 0 || o.WireBytes < p.minWire) { + p.minWire = o.WireBytes + } +} + +type pendingHeap []*pendingTx + +func (h pendingHeap) Len() int { return len(h) } +func (h pendingHeap) Less(i, j int) bool { + if h[i].deadline != h[j].deadline { + return h[i].deadline < h[j].deadline + } + return h[i].id < h[j].id +} +func (h pendingHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i]; h[i].idx = i; h[j].idx = j } +func (h *pendingHeap) Push(x interface{}) { p := x.(*pendingTx); p.idx = len(*h); *h = append(*h, p) } +func (h *pendingHeap) Pop() interface{} { + old := *h + p := old[len(old)-1] + old[len(old)-1] = nil + *h = old[:len(old)-1] + return p +} + +type finalizedRec struct { + id TxID + at int64 +} + +// Normalizer turns observations, fed in arrival order, into final Events. It +// is deterministic, holds no goroutines or timers and never reads the wall +// clock. It is not safe for concurrent use; use one Normalizer per goroutine. +type Normalizer struct { + cfg NormalizerConfig + watermark int64 + started bool + pending map[TxID]*pendingTx + deadlines pendingHeap + finalized map[TxID]int64 + finalizedLog []finalizedRec // FIFO by finalization time; head at finalizedHead + finalizedHd int + stats NormalizerStats +} + +// NewNormalizer validates cfg and returns an empty Normalizer. +func NewNormalizer(cfg NormalizerConfig) (*Normalizer, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + return &Normalizer{cfg: cfg, pending: make(map[TxID]*pendingTx), finalized: make(map[TxID]int64)}, nil +} + +// Add ingests one observation. ArrivedAt (or Time when ArrivedAt is zero) +// must not be earlier than the latest arrival already seen. +func (n *Normalizer) Add(o Observation) NormalizeResult { + var res NormalizeResult + if err := o.Validate(); err != nil { + n.stats.Invalid++ + res.Status, res.Err = ObsInvalid, err + return res + } + a := o.arrived().UnixNano() + if n.started && a < n.watermark { + n.stats.OutOfArrivalOrder++ + res.Status = ObsOutOfArrivalOrder + return res + } + n.advanceTo(a, &res) + if _, done := n.finalized[o.TxID]; done { + n.stats.Late++ + res.Status = ObsLate + return res + } + p := n.pending[o.TxID] + if p == nil { + if len(n.pending) >= n.cfg.MaxPendingTransmissions { + oldest := heap.Pop(&n.deadlines).(*pendingTx) + oldest.truncated = true + n.finalize(oldest, n.watermark, &res) + n.stats.ForcedFinalizations++ + res.CoverageReduced = true + } + p = &pendingTx{id: o.TxID, firstArrival: a, deadline: a + int64(n.cfg.SettleWindow)} + n.pending[o.TxID] = p + heap.Push(&n.deadlines, p) + } else { + if !p.compatible(&o) { + n.stats.Conflicts++ + res.Status = ObsConflict + return res + } + for _, h := range p.obs { + if h.ObsID == o.ObsID { + if sameObservation(h, &o) { + n.stats.Duplicates++ + res.Status = ObsDuplicate + } else { + n.stats.Conflicts++ + res.Status = ObsConflict + } + return res + } + } + if len(p.obs) >= n.cfg.MaxObservationsPerTransmission { + p.dropped++ + p.truncated = true + n.stats.DroppedFanout++ + res.Status = ObsDroppedFanout + res.CoverageReduced = true + return res + } + } + oc := o + oc.Path = Route{w: o.Path.w, hops: append([]byte(nil), o.Path.hops...)} + p.add(&oc) + n.stats.Accepted++ + res.Status = ObsAccepted + return res +} + +// Advance moves the arrival watermark to t (if later) and releases every +// transmission whose deadline has passed. +func (n *Normalizer) Advance(t time.Time) NormalizeResult { + var res NormalizeResult + if !validTime(t) { + return res + } + a := t.UnixNano() + if n.started && a < n.watermark { + return res + } + n.advanceTo(a, &res) + return res +} + +// Flush finalizes every pending transmission at its deadline (end of input). +func (n *Normalizer) Flush() NormalizeResult { + var res NormalizeResult + for n.deadlines.Len() > 0 { + p := heap.Pop(&n.deadlines).(*pendingTx) + n.finalize(p, p.deadline, &res) + } + return res +} + +// Stats returns cumulative counters. +func (n *Normalizer) Stats() NormalizerStats { + s := n.stats + s.Pending = len(n.pending) + s.FinalizedIDs = len(n.finalized) + return s +} + +func (n *Normalizer) advanceTo(a int64, res *NormalizeResult) { + n.started = true + if a > n.watermark { + n.watermark = a + } + for n.deadlines.Len() > 0 && n.deadlines[0].deadline < n.watermark { + p := heap.Pop(&n.deadlines).(*pendingTx) + n.finalize(p, p.deadline, res) + } + horizon := n.watermark - int64(n.cfg.FinalizedHorizon) + for n.finalizedHd < len(n.finalizedLog) && n.finalizedLog[n.finalizedHd].at < horizon { + n.dropFinalizedHead() + } + n.compactFinalized() +} + +func (n *Normalizer) finalize(p *pendingTx, at int64, res *NormalizeResult) { + delete(n.pending, p.id) + ev := buildEvent(p.obs, time.Unix(0, at).UTC(), p.dropped) + ev.Truncated = p.truncated + res.Events = append(res.Events, ev) + n.stats.EventsFinalized++ + n.finalized[p.id] = at + n.finalizedLog = append(n.finalizedLog, finalizedRec{id: p.id, at: at}) + for len(n.finalized) > n.cfg.MaxFinalizedIDs { + if n.finalizedLog[n.finalizedHd].at >= n.watermark-int64(n.cfg.FinalizedHorizon) { + n.stats.FinalizedIDsEvictedEarly++ + res.CoverageReduced = true + } + n.dropFinalizedHead() + } +} + +func (n *Normalizer) dropFinalizedHead() { + r := n.finalizedLog[n.finalizedHd] + n.finalizedLog[n.finalizedHd] = finalizedRec{} + n.finalizedHd++ + if at, ok := n.finalized[r.id]; ok && at == r.at { + delete(n.finalized, r.id) + } +} + +func (n *Normalizer) compactFinalized() { + if n.finalizedHd > 1024 && n.finalizedHd*2 > len(n.finalizedLog) { + n.finalizedLog = append([]finalizedRec(nil), n.finalizedLog[n.finalizedHd:]...) + n.finalizedHd = 0 + } +} + +// BatchReport describes what NormalizeTransmission included. +type BatchReport struct { + Included, ExcludedLate, Duplicates int +} + +// NormalizeTransmission applies the Normalizer's finality policy to all +// observations of ONE transmission at once (offline use). Observations are +// taken in arrival order (ArrivedAt, then ObsID); those arriving after +// (first arrival + settle) are excluded, exactly as the streaming Normalizer +// would reject them as late, before any conflict check. It returns an error +// if the observations belong to different transmissions, or if included +// observations conflict (where the streaming Normalizer reports +// ObsConflict). +func NormalizeTransmission(obs []Observation, settle time.Duration) (Event, BatchReport, error) { + var rep BatchReport + if len(obs) == 0 { + return Event{}, rep, errors.New("anomaly: no observations") + } + if settle <= 0 { + return Event{}, rep, errors.New("anomaly: settle window must be > 0") + } + ptrs := make([]*Observation, len(obs)) + for i := range obs { + if err := obs[i].Validate(); err != nil { + return Event{}, rep, fmt.Errorf("observation %d: %w", i, err) + } + ptrs[i] = &obs[i] + } + sort.Slice(ptrs, func(i, j int) bool { + ai, aj := ptrs[i].arrived(), ptrs[j].arrived() + if !ai.Equal(aj) { + return ai.Before(aj) + } + return ptrs[i].ObsID < ptrs[j].ObsID + }) + first := ptrs[0] + deadline := first.arrived().Add(settle) + p := &pendingTx{id: first.TxID} + for i, o := range ptrs { + if o.TxID != first.TxID { + return Event{}, rep, fmt.Errorf("anomaly: observation %d belongs to another transmission", i) + } + if o.arrived().After(deadline) { + rep.ExcludedLate++ + continue + } + if len(p.obs) > 0 && !p.compatible(o) { + return Event{}, rep, fmt.Errorf("anomaly: observation %d conflicts", i) + } + dup := false + for _, h := range p.obs { + if h.ObsID == o.ObsID { + if !sameObservation(h, o) { + return Event{}, rep, fmt.Errorf("anomaly: observation %d reuses an ObsID with different content", i) + } + dup = true + } + } + if dup { + rep.Duplicates++ + continue + } + p.add(o) + } + rep.Included = len(p.obs) + return buildEvent(p.obs, deadline, rep.ExcludedLate), rep, nil +} diff --git a/internal/anomaly/normalizer_test.go b/internal/anomaly/normalizer_test.go new file mode 100644 index 000000000..4cb438ec7 --- /dev/null +++ b/internal/anomaly/normalizer_test.go @@ -0,0 +1,306 @@ +package anomaly + +import ( + "errors" + "fmt" + "math/rand" + "reflect" + "strings" + "testing" + "time" +) + +func normCfg() NormalizerConfig { + return NormalizerConfig{SettleWindow: 30 * time.Second, MaxPendingTransmissions: 1000, + MaxObservationsPerTransmission: 64, FinalizedHorizon: time.Hour, MaxFinalizedIDs: 100_000} +} + +func obs(t testing.TB, tx, id string, when time.Time, class RouteClass, width int, hops ...byte) Observation { + t.Helper() + o := Observation{TxID: TxID(tx), ObsID: id, Time: when, PayloadType: 5, Channel: mustChan(t, 0x42), + RouteClass: class, WireBytes: 40 + len(hops)} + if len(hops) > 0 { + o.Path = mustRoute(t, width, hops...) + } + return o +} + +func TestNormalizeFanoutIsOneTransmission(t *testing.T) { + var os []Observation + for i := 0; i < 5; i++ { + os = append(os, obs(t, "tx", fmt.Sprintf("o%d", i), at(time.Duration(i)*time.Second), RouteClassFlood, 1, byte(0x10+i), 0x99)) + } + e, rep, err := NormalizeTransmission(os, 30*time.Second) + if err != nil { + t.Fatal(err) + } + if e.Observations != 5 || rep.Included != 5 { + t.Fatalf("fan-out: %d observations, %d included", e.Observations, rep.Included) + } + // feeding the event to a detector counts it once + d := newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeGlobal, time.Minute, 2)}}) + cs := run(t, d, []Event{e}) + if len(cs) != 0 { + t.Fatalf("one transmission with five observations must count once:\n%s", describe(cs)) + } +} + +func TestNormalizeFirstHopIsEarliestPathKeepingWidth(t *testing.T) { + os := []Observation{ + obs(t, "tx", "b", at(2*time.Second), RouteClassFlood, 2, 0xAA, 0x01, 0xBB, 0x02), // later, path + obs(t, "tx", "a", at(1*time.Second), RouteClassFlood, 0), // earliest, no path + obs(t, "tx", "c", at(3*time.Second), RouteClassFlood, 2, 0xCC, 0x03), + } + e, _, err := NormalizeTransmission(os, time.Minute) + if err != nil { + t.Fatal(err) + } + if e.RouteKind != RouteKindRelay || e.FirstHop != mustHop(t, 0xAA, 0x01) { + t.Fatalf("first hop %x (kind %v), want 2-byte aa01 from the earliest observation with a path", e.FirstHop.Bytes(), e.RouteKind) + } + if e.FullRoute.Hops() != 2 || e.FullRoute.Width() != 2 { + t.Fatalf("full route %d hops width %d", e.FullRoute.Hops(), e.FullRoute.Width()) + } + if !e.Time.Equal(at(time.Second)) { + t.Fatalf("event time %v, want earliest observation", e.Time) + } + // tie on time: ObsID breaks it + tie := []Observation{ + obs(t, "tx2", "z", at(0), RouteClassFlood, 1, 0x02), + obs(t, "tx2", "y", at(0), RouteClassFlood, 1, 0x01), + } + e2, _, _ := NormalizeTransmission(tie, time.Minute) + if e2.FirstHop != mustHop(t, 0x01) { + t.Fatalf("tie must break on ObsID, got %x", e2.FirstHop.Bytes()) + } +} + +func TestNormalizePermutationInvariant(t *testing.T) { + base := []Observation{ + obs(t, "tx", "o1", at(3*time.Second), RouteClassFlood, 1, 0x05, 0x06), + obs(t, "tx", "o2", at(1*time.Second), RouteClassFlood, 1, 0x07), + obs(t, "tx", "o3", at(1*time.Second), RouteClassFlood, 0), + obs(t, "tx", "o4", at(2*time.Second), RouteClassTransportFlood, 1, 0x08), + obs(t, "tx", "o5", at(9*time.Second), RouteClassFlood, 1, 0x09), + } + want, _, err := NormalizeTransmission(base, 30*time.Second) + if err != nil { + t.Fatal(err) + } + rng := rand.New(rand.NewSource(1)) + for i := 0; i < 50; i++ { + p := append([]Observation(nil), base...) + rng.Shuffle(len(p), func(a, b int) { p[a], p[b] = p[b], p[a] }) + got, _, err := NormalizeTransmission(p, 30*time.Second) + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("permutation %d changed the event: %+v vs %+v (%v)", i, got, want, err) + } + } +} + +func TestNormalizerLateObservationNeverChangesFinalEvent(t *testing.T) { + n, _ := NewNormalizer(normCfg()) + a := obs(t, "tx", "a", at(10*time.Second), RouteClassFlood, 1, 0x11) + if r := n.Add(a); r.Status != ObsAccepted { + t.Fatal(r.Status) + } + // a later-arriving observation with an EARLIER timestamp, after the deadline + late := obs(t, "tx", "b", at(0), RouteClassFlood, 1, 0x22) + late.ArrivedAt = at(10*time.Second + 31*time.Second) + r := n.Add(late) + if len(r.Events) != 1 { + t.Fatalf("event should finalize when the watermark passes the deadline, got %d", len(r.Events)) + } + if r.Status != ObsLate { + t.Fatalf("status %v, want late", r.Status) + } + e := r.Events[0] + if e.FirstHop != mustHop(t, 0x11) || !e.Time.Equal(at(10*time.Second)) { + t.Fatalf("late evidence leaked into the final event: hop %x time %v", e.FirstHop.Bytes(), e.Time) + } + if !e.FinalAt.Equal(at(40 * time.Second)) { + t.Fatalf("FinalAt %v, want first arrival + settle", e.FinalAt) + } + if n.Stats().Late != 1 { + t.Fatal("late observation must be counted") + } +} + +func TestNormalizerEarlierTimestampWithinSettleIsIncluded(t *testing.T) { + n, _ := NewNormalizer(normCfg()) + n.Add(obs(t, "tx", "a", at(10*time.Second), RouteClassFlood, 1, 0x11)) + b := obs(t, "tx", "b", at(0), RouteClassFlood, 1, 0x22) + b.ArrivedAt = at(20 * time.Second) // within the settle window + if r := n.Add(b); r.Status != ObsAccepted { + t.Fatal(r.Status) + } + evs := n.Flush().Events + if len(evs) != 1 || evs[0].FirstHop != mustHop(t, 0x22) || !evs[0].Time.Equal(at(0)) { + t.Fatalf("pending event must use the earliest observation seen before finality: %+v", evs) + } +} + +func TestNormalizerStreamingEqualsBatch(t *testing.T) { + rng := rand.New(rand.NewSource(7)) + for trial := 0; trial < 40; trial++ { + var os []Observation + for i := 0; i < 1+rng.Intn(8); i++ { + o := obs(t, "tx", fmt.Sprintf("o%02d", i), at(time.Duration(rng.Intn(20))*time.Second), RouteClassFlood, 1, byte(rng.Intn(4))) + if rng.Intn(3) == 0 { + o.Path = Route{} + } + o.ArrivedAt = at(time.Duration(rng.Intn(60)) * time.Second) + os = append(os, o) + } + batch, _, err := NormalizeTransmission(os, 30*time.Second) + if err != nil { + t.Fatal(err) + } + n, _ := NewNormalizer(normCfg()) + sorted := append([]Observation(nil), os...) + // arrival order + for i := 1; i < len(sorted); i++ { + for j := i; j > 0 && (sorted[j].ArrivedAt.Before(sorted[j-1].ArrivedAt) || + sorted[j].ArrivedAt.Equal(sorted[j-1].ArrivedAt) && sorted[j].ObsID < sorted[j-1].ObsID); j-- { + sorted[j], sorted[j-1] = sorted[j-1], sorted[j] + } + } + var evs []Event + for _, o := range sorted { + evs = append(evs, n.Add(o).Events...) + } + evs = append(evs, n.Flush().Events...) + if len(evs) != 1 { + t.Fatalf("trial %d: %d events", trial, len(evs)) + } + got := evs[0] + got.ExcludedObservations = batch.ExcludedObservations // streaming cannot know future late arrivals + if !reflect.DeepEqual(got, batch) { + t.Fatalf("trial %d: streaming %+v != batch %+v", trial, got, batch) + } + } +} + +func TestNormalizerDuplicateAndConflict(t *testing.T) { + n, _ := NewNormalizer(normCfg()) + a := obs(t, "tx", "a", at(0), RouteClassFlood, 1, 0x11) + n.Add(a) + if r := n.Add(a); r.Status != ObsDuplicate { + t.Fatalf("identical observation: %v", r.Status) + } + b := a + b.WireBytes++ + if r := n.Add(b); r.Status != ObsConflict { + t.Fatalf("same ObsID, different content: %v", r.Status) + } + c := obs(t, "tx", "c", at(0), RouteClassFlood, 1, 0x11) + c.Channel = mustChan(t, 0x43) + if r := n.Add(c); r.Status != ObsConflict { + t.Fatalf("different channel for the same TxID: %v", r.Status) + } + _, _, err := NormalizeTransmission([]Observation{a, c}, time.Minute) + if err == nil { + t.Fatal("batch must reject conflicting observations") + } +} + +func TestNormalizeRouteClassesAndTrace(t *testing.T) { + mixed := []Observation{ + obs(t, "tx", "a", at(0), RouteClassFlood, 1, 0x11), + obs(t, "tx", "b", at(time.Second), RouteClassTransportDirect, 0), + } + e, _, _ := NormalizeTransmission(mixed, time.Minute) + if e.RouteClass != RouteClassMixed || StreamKeyOf(e).confidence() != ConfidenceLow { + t.Fatalf("disagreeing route classes must be mixed / low confidence: %v", e.RouteClass) + } + direct := []Observation{obs(t, "d", "a", at(0), RouteClassDirect, 1, 0x33, 0x44)} + e, _, _ = NormalizeTransmission(direct, time.Minute) + if e.RouteKind != RouteKindDirect || !e.FirstHop.IsZero() { + t.Fatalf("a direct path's next hop must not become a first hop: %v %x", e.RouteKind, e.FirstHop.Bytes()) + } + tr := obs(t, "tr", "a", at(0), RouteClassFlood, 1, 0x55) + tr.PayloadType = PayloadTypeTrace + e, _, _ = NormalizeTransmission([]Observation{tr}, time.Minute) + if e.RouteKind != RouteKindUnknown || !e.FirstHop.IsZero() { + t.Fatal("TRACE paths hold SNR bytes and must not become route evidence") + } + if err := e.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestNormalizerBoundsAreCountedAsReducedCoverage(t *testing.T) { + cfg := normCfg() + cfg.MaxPendingTransmissions = 2 + cfg.MaxObservationsPerTransmission = 2 + cfg.MaxFinalizedIDs = 2 + n, _ := NewNormalizer(cfg) + for i := 0; i < 3; i++ { + r := n.Add(obs(t, "fan", fmt.Sprintf("o%d", i), at(0), RouteClassFlood, 1, 0x01)) + if i == 2 && (r.Status != ObsDroppedFanout || !r.CoverageReduced) { + t.Fatalf("fan-out cap: %v", r.Status) + } + } + n.Add(obs(t, "p2", "o", at(time.Second), RouteClassFlood, 1, 0x01)) + r := n.Add(obs(t, "p3", "o", at(2*time.Second), RouteClassFlood, 1, 0x01)) + if len(r.Events) != 1 || !r.CoverageReduced { + t.Fatalf("pending cap must force-finalize the earliest deadline: %d events", len(r.Events)) + } + n.Flush() + st := n.Stats() + if st.DroppedFanout != 1 || st.ForcedFinalizations != 1 || st.FinalizedIDsEvictedEarly == 0 || !st.CoverageReduced() { + t.Fatalf("stats %+v", st) + } + if st.FinalizedIDs > 2 || st.Pending != 0 { + t.Fatalf("bounded sets exceeded: %+v", st) + } +} + +func TestNormalizerRejectsOutOfArrivalOrder(t *testing.T) { + n, _ := NewNormalizer(normCfg()) + n.Add(obs(t, "a", "o", at(10*time.Second), RouteClassFlood, 1, 0x01)) + if r := n.Add(obs(t, "b", "o", at(5*time.Second), RouteClassFlood, 1, 0x01)); r.Status != ObsOutOfArrivalOrder { + t.Fatalf("status %v", r.Status) + } +} + +func TestValidationErrorsDoNotEchoPrivateValues(t *testing.T) { + secretID := "SECRET-TX-5f3a" + o := obs(t, secretID, "OBS-SECRET-9c1d", at(0), RouteClassFlood, 1, 0xDE) + o.WireBytes = 999 + err := o.Validate() + if err == nil || !errors.Is(err, errInvalidObservation) { + t.Fatal("expected invalid observation") + } + e := relayEvent(t, secretID, t0, 0xAB, 0xDE) + e.WireBytes = 999 + err2 := e.Validate() + for _, msg := range []string{err.Error(), err2.Error()} { + for _, secret := range []string{secretID, "OBS-SECRET", "de", "ab", "DE", "AB"} { + if strings.Contains(strings.ToLower(msg), strings.ToLower(secret)) && secret != "de" && secret != "ab" { + t.Fatalf("error leaks %q: %s", secret, msg) + } + } + if strings.Contains(msg, "5f3a") || strings.Contains(msg, "9c1d") { + t.Fatalf("error leaks id: %s", msg) + } + } +} + +func TestNormalizerAdvanceReleasesDueTransmissions(t *testing.T) { + n, _ := NewNormalizer(normCfg()) + n.Add(obs(t, "tx", "a", at(0), RouteClassFlood, 1, 0x11)) + if r := n.Advance(at(29 * time.Second)); len(r.Events) != 0 { + t.Fatal("released before the deadline") + } + if r := n.Advance(at(30 * time.Second)); len(r.Events) != 0 { + t.Fatal("released at the deadline: an observation may still arrive at that instant") + } + r := n.Advance(at(31 * time.Second)) + if len(r.Events) != 1 || !r.Events[0].FinalAt.Equal(at(30*time.Second)) { + t.Fatalf("advance must release at first arrival + settle: %+v", r.Events) + } + if r := n.Advance(at(10 * time.Second)); len(r.Events) != 0 { + t.Fatal("going back in time must be ignored") + } +} diff --git a/internal/anomaly/periodic.go b/internal/anomaly/periodic.go new file mode 100644 index 000000000..2b3b4782c --- /dev/null +++ b/internal/anomaly/periodic.go @@ -0,0 +1,411 @@ +package anomaly + +import ( + "math" + "slices" + "time" +) + +// Periodicity detection. +// +// Events of a key are first grouped into pulses: an event within BurstGap of +// the previous event joins the current pulse, whose time is its first event. +// Each new pulse is evaluated immediately, using only pulses at or before it +// (causal). The detector explains the chain of consecutive gaps that ends at +// the newest pulse with one period P: a gap g is explained if +// |g - k*P| <= tol(P) for some k in 1..MaxMissing+1 (k-1 missing pulses). +// +// Candidate periods come from recent gaps divided by k, refined with the +// phase estimator (chain span / periods spanned). They are ranked by +// (meets MinPulses/MinCoverage/MaxJitterFraction first, then chain length +// desc, coverage desc, median residual asc, period asc). A multiple of the +// true period cannot explain the gaps of the true period, so its chain +// breaks; a sub-multiple explains the same chain only with lower coverage. +// A hypothesis is kept while it explains each new gap; when it no longer +// meets the criteria, the search runs again and a hypothesis that meets +// them wins over a longer one that does not. While it meets them, its +// multiples 2P..(MaxMissing+1)P are checked too, and one that meets the +// criteria on its own with higher coverage replaces it (a sub-multiple +// explains the true train only with lower coverage). So an extra (stray) +// pulse, which breaks the true period's chain and briefly favours a +// sub-multiple, costs at most MinPulses clean pulses before the true period +// is detected again, whatever MinCoverage is. Near-ties are reported as +// alternatives rather than hidden. +// +// The null model is a Poisson process with the key's local pulse rate over +// the pulse ring (causal): chance = evaluated * cells * p_gap^(chainGaps-1), +// where p_gap is the probability that one exponential gap falls within tol +// of some k*P. The exponent drops one gap because P is fitted to the chain +// (the gap it was derived from matches by construction). cells is the +// number of candidate periods the search may try per pulse and evaluated is +// the number of pulses tested on the key so far: a union bound over the +// period search and over every pulse that was another chance to fire. + +// recentGaps is how many of the newest gaps seed the period search. +const recentGaps = 16 + +type pulseFlags uint8 + +const ( + pulseAllExpected pulseFlags = 1 << iota + pulseAnyExpected + pulseAnyMonitored +) + +type periodicState struct { + times []int64 // ring of pulse start times + flags []pulseFlags + counts []uint16 // events merged into each pulse (saturating) + head, n int + lastEvent int64 + started bool + evaluated uint64 // pulses evaluated on this key so far (multiple-testing count) + // current hypothesis for the chain ending at the newest pulse + period int64 + sufficientAt int64 // first pulse time at which the current chain met every criterion (0 = not yet) + alts []PeriodAlternative +} + +func newPeriodicState(historyLen int) *periodicState { + return &periodicState{times: make([]int64, historyLen), flags: make([]pulseFlags, historyLen), counts: make([]uint16, historyLen)} +} + +func (p *periodicState) at(i int) int64 { // i-th oldest pulse + return p.times[(p.head+i)%len(p.times)] +} + +func (p *periodicState) flagAt(i int) pulseFlags { return p.flags[(p.head+i)%len(p.flags)] } + +func (p *periodicState) countAt(i int) uint16 { return p.counts[(p.head+i)%len(p.counts)] } + +func classFlags(c TrafficClass) pulseFlags { + switch c { + case TrafficExpected: + return pulseAllExpected | pulseAnyExpected + case TrafficMonitored: + return pulseAnyMonitored + } + return 0 +} + +// periodicScratch holds reusable buffers owned by the Detector (one +// goroutine), so periodic evaluation neither allocates per event nor keeps +// buffers per key. +type periodicScratch struct { + resid, med, seen []int64 +} + +// periodicSignal is a hot evaluation result. +type periodicSignal struct { + ev PeriodicEvidence + label TrafficLabel +} + +// observe feeds one event. It returns a signal when the chain ending at a +// newly started pulse satisfies every criterion of r. +func (p *periodicState) observe(t int64, cls TrafficClass, r *PeriodicRule, sc *periodicScratch) (periodicSignal, bool) { + f := classFlags(cls) + if p.started && t-p.lastEvent <= int64(r.BurstGap) { + // same burst: merge into the newest pulse without re-evaluating + i := (p.head + p.n - 1) % len(p.times) + if p.flags[i]&pulseAllExpected != 0 && f&pulseAllExpected == 0 { + p.flags[i] &^= pulseAllExpected + } + p.flags[i] |= f &^ pulseAllExpected + if p.counts[i] < math.MaxUint16 { + p.counts[i]++ + } + p.lastEvent = t + return periodicSignal{}, false + } + p.started = true + p.lastEvent = t + if p.n == len(p.times) { + p.head = (p.head + 1) % len(p.times) + p.n-- + } + i := (p.head + p.n) % len(p.times) + p.times[i], p.flags[i], p.counts[i] = t, f, 1 + p.n++ + p.evaluated++ + return p.evaluate(t, r, sc) +} + +// chainInfo describes the longest suffix chain explained by one period. +type chainInfo struct { + period int64 + tol int64 + gaps int // explained consecutive gaps ending at the newest pulse + slots int // sum of k over those gaps + residual int64 // median |g - k*P| + coverage float64 // gaps / slots +} + +// meets reports whether a chain meets the rule's shape criteria (support, +// coverage, jitter); the null-model test comes on top. +func (c chainInfo) meets(r *PeriodicRule) bool { + return c.gaps > 0 && c.gaps+1 >= r.MinPulses && c.coverage >= r.MinCoverage && + float64(c.residual) <= r.MaxJitterFraction*float64(c.tol) +} + +// preferred ranks chains: meeting the criteria first, then better(). +func (c chainInfo) preferred(o chainInfo, r *PeriodicRule) bool { + if cm, om := c.meets(r), o.meets(r); cm != om { + return cm + } + return c.better(o) +} + +func (c chainInfo) better(o chainInfo) bool { + if c.gaps != o.gaps { + return c.gaps > o.gaps + } + if c.coverage != o.coverage { + return c.coverage > o.coverage + } + if c.residual != o.residual { + return c.residual < o.residual + } + return c.period < o.period +} + +func tolNanos(r *PeriodicRule, period int64) int64 { + rel := int64(r.JitterRel * float64(period)) + if int64(r.JitterAbs) > rel { + return int64(r.JitterAbs) + } + return rel +} + +// explain returns k (1..K) if gap g is explained by period P with tolerance +// tol, else 0. +func explain(g, period, tol int64, maxK int) (int, int64) { + if period <= 0 { + return 0, 0 + } + k := (g + period/2) / period + if k < 1 { + k = 1 + } + if k > int64(maxK) { + return 0, 0 + } + d := g - k*period + if d < 0 { + d = -d + } + if d > tol { + return 0, 0 + } + return int(k), d +} + +// chainFor measures the suffix chain for one period. +func (p *periodicState) chainFor(period int64, r *PeriodicRule, sc *periodicScratch) chainInfo { + c := chainInfo{period: period, tol: tolNanos(r, period)} + scratch := sc.resid[:0] + maxK := r.MaxMissing + 1 + for i := p.n - 1; i >= 1; i-- { + g := p.at(i) - p.at(i-1) + k, d := explain(g, period, c.tol, maxK) + if k == 0 { + break + } + c.gaps++ + c.slots += k + scratch = append(scratch, d) + } + sc.resid = scratch + if c.gaps > 0 { + c.coverage = float64(c.gaps) / float64(c.slots) + c.residual = medianInt64(scratch, sc) + } + return c +} + +// medianInt64 returns the median of v without modifying it. +func medianInt64(v []int64, sc *periodicScratch) int64 { + s := append(sc.med[:0], v...) + slices.Sort(s) + sc.med = s + n := len(s) + if n%2 == 1 { + return s[n/2] + } + return (s[n/2-1] + s[n/2]) / 2 +} + +func (p *periodicState) evaluate(now int64, r *PeriodicRule, sc *periodicScratch) (periodicSignal, bool) { + if p.n < r.MinPulses { + p.period, p.sufficientAt, p.alts = 0, 0, nil + return periodicSignal{}, false + } + var best chainInfo + // Keep the current hypothesis if it still explains the newest gap; search + // again when there is none or it no longer meets the criteria. A new + // chain restarts SufficientAt and the alternatives. + kept := false + if p.period > 0 { + newest := p.at(p.n-1) - p.at(p.n-2) + if k, _ := explain(newest, p.period, tolNanos(r, p.period), r.MaxMissing+1); k > 0 { + best, kept = p.chainFor(p.period, r, sc), true + } + } + // A sub-multiple that absorbed a stray pulse keeps explaining the true + // train with lower coverage. As soon as a multiple of the kept period + // meets the criteria on its own with higher coverage, it is the + // fundamental. + if kept && best.meets(r) { + base, fund := best.period, best + for m := int64(2); m <= int64(r.MaxMissing+1); m++ { + if mp := base * m; mp <= int64(r.MaxPeriod) { + if c := p.chainFor(mp, r, sc); c.meets(r) && c.coverage > fund.coverage { + fund = c + } + } + } + if fund.period != base { + best = fund + p.sufficientAt, p.alts = 0, nil + } + } + if !best.meets(r) { + if est := p.estimate(r, sc); !kept || (est.gaps > 0 && est.preferred(best, r)) { + best = est + p.sufficientAt, p.alts = 0, nil + } + } + if best.gaps == 0 { + p.period = 0 + return periodicSignal{}, false + } + p.period = best.period + support := best.gaps + 1 + if !best.meets(r) { + p.sufficientAt, p.alts = 0, nil + return periodicSignal{}, false + } + chance := p.chance(best, r) + if chance > r.MaxChance { + p.sufficientAt, p.alts = 0, nil + return periodicSignal{}, false + } + if p.sufficientAt == 0 { + p.sufficientAt = now + p.alts = p.alternatives(best, r, sc) + } + // traffic label and burst sizes over the chain's pulses + var total, exp, mon uint64 + events := 0 + for i := p.n - 1 - best.gaps; i < p.n; i++ { + fl := p.flagAt(i) + total++ + if fl&pulseAllExpected != 0 { + exp++ + } + if fl&pulseAnyMonitored != 0 { + mon++ + } + events += int(p.countAt(i)) + } + ev := PeriodicEvidence{ + Period: time.Duration(best.period), + Tolerance: time.Duration(best.tol), + JitterMedian: time.Duration(best.residual), + Support: support, + Events: events, + Coverage: best.coverage, + Chance: chance, + SufficientAt: time.Unix(0, p.sufficientAt).UTC(), + Alternatives: append([]PeriodAlternative(nil), p.alts...), + } + return periodicSignal{ev: ev, label: labelFromCounts(total, exp, mon)}, true +} + +// estimate searches candidate periods derived from recent gaps. +func (p *periodicState) estimate(r *PeriodicRule, sc *periodicScratch) chainInfo { + var best chainInfo + minP, maxP := int64(r.MinPeriod), int64(r.MaxPeriod) + maxK := r.MaxMissing + 1 + newest := p.at(p.n-1) - p.at(p.n-2) + // Quick reject: a newest gap no period in range can explain. + if newest+tolNanos(r, maxP) < minP-tolNanos(r, minP) || newest > int64(maxK)*maxP+tolNanos(r, maxP) { + return best + } + seen := sc.seen[:0] + for i := p.n - 1; i >= 1 && i >= p.n-recentGaps; i-- { + g := p.at(i) - p.at(i-1) + for k := 1; k <= maxK; k++ { + cand := g / int64(k) + if cand < minP || cand > maxP { + continue + } + if slices.Contains(seen, cand) { + continue + } + seen = append(seen, cand) + c := p.chainFor(cand, r, sc) + if c.gaps > 0 && (best.gaps == 0 || c.preferred(best, r)) { + best = c + } + } + } + sc.seen = seen + if best.gaps == 0 { + return best + } + // Refine with the phase estimator span/slots over the chain. It is + // robust to per-pulse jitter, whereas ranking by gap residual favours a + // biased gap. It replaces the estimate whenever it explains a chain at + // least as long with at least the same coverage. + span := p.at(p.n-1) - p.at(p.n-1-best.gaps) + if ref := span / int64(best.slots); ref >= minP && ref <= maxP && ref != best.period { + if c := p.chainFor(ref, r, sc); c.gaps >= best.gaps && c.coverage >= best.coverage && (c.meets(r) || !best.meets(r)) { + best = c + } + } + return best +} + +// alternatives reports multiples and sub-multiples of the chosen period +// whose own chain is at least half as long (ambiguous explanations). +func (p *periodicState) alternatives(best chainInfo, r *PeriodicRule, sc *periodicScratch) []PeriodAlternative { + var out []PeriodAlternative + for _, f := range []struct{ num, den int64 }{{2, 1}, {3, 1}, {1, 2}, {1, 3}} { + cand := best.period * f.num / f.den + if cand < int64(r.MinPeriod) || cand > int64(r.MaxPeriod) { + continue + } + if c := p.chainFor(cand, r, sc); c.gaps*2 >= best.gaps && c.gaps > 0 { + out = append(out, PeriodAlternative{Period: time.Duration(cand), Support: c.gaps + 1, Coverage: c.coverage}) + } + } + return out +} + +func (p *periodicState) chance(c chainInfo, r *PeriodicRule) float64 { + span := p.at(p.n-1) - p.at(0) + if span <= 0 { + return 1 + } + lambda := float64(p.n-1) / float64(span) + var pg float64 + for k := 1; k <= r.MaxMissing+1; k++ { + lo := float64(int64(k)*c.period - c.tol) + if lo < 0 { + lo = 0 + } + hi := float64(int64(k)*c.period + c.tol) + pg += math.Exp(-lambda*lo) - math.Exp(-lambda*hi) + } + if pg > 1 { + pg = 1 + } + // Union bound: every pulse evaluated so far and every candidate period the + // search may try is another chance; one gap is spent on fitting P. + return float64(p.evaluated) * float64(searchCells(r)) * math.Pow(pg, float64(c.gaps-1)) +} + +// searchCells is the number of candidate periods estimate may try per pulse: +// recentGaps gaps times MaxMissing+1 divisors, plus the refinement. +func searchCells(r *PeriodicRule) int { return recentGaps*(r.MaxMissing+1) + 1 } diff --git a/internal/anomaly/periodic_test.go b/internal/anomaly/periodic_test.go new file mode 100644 index 000000000..6a5687039 --- /dev/null +++ b/internal/anomaly/periodic_test.go @@ -0,0 +1,363 @@ +package anomaly + +import ( + "fmt" + "math/rand" + "reflect" + "testing" + "time" +) + +// experimentalPeriodic is a named test fixture, not a recommendation. +func experimentalPeriodic() PeriodicRule { + return PeriodicRule{Name: "per", Scope: ScopeStream, MinPeriod: time.Minute, MaxPeriod: time.Hour, + BurstGap: 5 * time.Second, JitterAbs: 5 * time.Second, JitterRel: 0.02, MaxMissing: 2, + MinPulses: 8, HistoryLen: 32, MinCoverage: 0.6, MaxJitterFraction: 0.5, MaxChance: 0.01, + State: StateParams{EndAfterQuiet: 30 * time.Minute}} +} + +func perDet(t testing.TB, mode ExpectedMode) *Detector { + return newDet(t, Config{Periodic: []PeriodicRule{experimentalPeriodic()}, Expected: ExpectedPolicy{Mode: mode}}) +} + +func firstActive(t *testing.T, cs []Candidate) Candidate { + t.Helper() + a := filter(cs, "per", StateActive) + if len(a) == 0 { + t.Fatalf("no periodic activation:\n%s", describe(cs)) + } + return a[0] +} + +func TestPeriodicPerfectFiveMinutesAlertsAtFirstSufficientPulse(t *testing.T) { + evs := series(t, "p", t0, 12, 5*time.Minute, nil, 0x11, 0x22) + c := firstActive(t, run(t, perDet(t, ExpectedInclude), evs)) + ev := c.Periodic + if ev.Period != 5*time.Minute || ev.Support != 8 || ev.Coverage != 1 || ev.JitterMedian != 0 { + t.Fatalf("evidence %+v", ev) + } + if !c.At.Equal(evs[7].Time) || !ev.SufficientAt.Equal(evs[7].Time) { + t.Fatalf("alert at %v / sufficient %v, want the 8th pulse %v", c.At, ev.SufficientAt, evs[7].Time) + } + if ev.Chance > 1e-6 { + t.Fatalf("chance %g", ev.Chance) + } +} + +func TestPeriodicWithJitter(t *testing.T) { + // Per-pulse jitter of up to 4 s gives gap residuals of up to 8 s (the + // detector measures gaps), so this rule uses an explicit 10 s tolerance. + pattern := []time.Duration{0, 3, -4, 2, -3, 4, -1, 1, -2, 3, -4, 2} + evs := series(t, "p", t0, 12, 5*time.Minute, func(i int) time.Duration { return pattern[i] * time.Second }, 0x11, 0x22) + // JitterMedian is measured on gaps, so a per-pulse jitter of +/-4 s gives + // a gap-residual median around 5 s: the explicit fraction is 0.6. + r := experimentalPeriodic() + r.JitterAbs = 10 * time.Second + r.MaxJitterFraction = 0.6 + ev := firstActive(t, run(t, newDet(t, Config{Periodic: []PeriodicRule{r}}), evs)).Periodic + // the same series with a 6 s tolerance is (correctly) not accepted + if a := filter(run(t, perDet(t, ExpectedInclude), evs), "per", StateActive); len(a) != 0 { + t.Fatalf("gap residuals above the tolerance were accepted: %+v", a[0].Periodic) + } + if d := ev.Period - 5*time.Minute; d < -2*time.Second || d > 2*time.Second { + t.Fatalf("period %v", ev.Period) + } + if ev.JitterMedian == 0 || float64(ev.JitterMedian) > 0.6*float64(ev.Tolerance) { + t.Fatalf("jitter median %v tol %v", ev.JitterMedian, ev.Tolerance) + } +} + +func TestPeriodicBurstsAroundEachPulse(t *testing.T) { + var evs []Event + for i := 0; i < 10; i++ { + for j := 0; j < 3; j++ { + evs = append(evs, relayEvent(t, fmt.Sprintf("b%02d-%d", i, j), at(time.Duration(i)*5*time.Minute+time.Duration(j)*time.Second), 0x11, 0x22)) + } + } + c := firstActive(t, run(t, perDet(t, ExpectedInclude), evs)) + if c.Periodic.Period != 5*time.Minute || c.Periodic.Support != 8 || c.Periodic.Events != 22 { + // 7 full pulses x 3 events + the first event of the 8th pulse + t.Fatalf("bursts: %+v", c.Periodic) + } + if !c.At.Equal(at(35 * time.Minute)) { + t.Fatalf("alert must be at the start of the 8th burst, got %v", c.At.Sub(t0)) + } +} + +func TestPeriodicSimultaneousTimestampsAreOnePulse(t *testing.T) { + r := experimentalPeriodic() + r.BurstGap = 0 + var evs []Event + for i := 0; i < 10; i++ { + for j := 0; j < 2; j++ { + evs = append(evs, relayEvent(t, fmt.Sprintf("s%02d-%d", i, j), at(time.Duration(i)*5*time.Minute), 0x11, 0x22)) + } + } + cs := run(t, newDet(t, Config{Periodic: []PeriodicRule{r}}), evs) + c := firstActive(t, cs) + if c.Periodic.Support != 8 || c.Periodic.Period != 5*time.Minute { + t.Fatalf("simultaneous events: %+v", c.Periodic) + } +} + +func TestPeriodicMissingPulses(t *testing.T) { + var ks []int + for k := 0; len(ks) < 14; k++ { + if k%3 != 2 { // every third pulse missing + ks = append(ks, k) + } + } + var evs []Event + for i, k := range ks { + evs = append(evs, relayEvent(t, fmt.Sprintf("m%02d", i), at(time.Duration(k)*5*time.Minute), 0x11, 0x22)) + } + ev := firstActive(t, run(t, perDet(t, ExpectedInclude), evs)).Periodic + if ev.Period != 5*time.Minute || ev.Coverage >= 1 || ev.Coverage < 0.6 { + t.Fatalf("missing pulses: %+v", ev) + } +} + +func TestPeriodicDoesNotPickTheWrongMultiple(t *testing.T) { + // true period 10 min: the 5-min sub-multiple explains the same gaps only + // with half the coverage and must lose; it is reported as an alternative + evs := series(t, "h", t0, 12, 10*time.Minute, nil, 0x11, 0x22) + c := firstActive(t, run(t, perDet(t, ExpectedInclude), evs)) + if c.Periodic.Period != 10*time.Minute { + t.Fatalf("picked %v for a 10-minute series", c.Periodic.Period) + } + found := false + for _, a := range c.Periodic.Alternatives { + if a.Period == 5*time.Minute && a.Coverage == 0.5 { + found = true + } + if a.Period == 20*time.Minute { + t.Fatalf("20 min cannot explain 10-min gaps: %+v", a) + } + } + if !found { + t.Fatalf("ambiguous sub-multiple not reported: %+v", c.Periodic.Alternatives) + } + // with a coverage floor both explanations meet, the fundamental must + // still win on coverage + r := experimentalPeriodic() + r.MinCoverage = 0.4 + c = firstActive(t, run(t, newDet(t, Config{Periodic: []PeriodicRule{r}}), series(t, "l", t0, 12, 10*time.Minute, nil, 0x11, 0x22))) + if c.Periodic.Period != 10*time.Minute { + t.Fatalf("picked %v for a 10-minute series when both explanations meet the criteria", c.Periodic.Period) + } + // true period 5 min: 10 min cannot explain the chain + evs = series(t, "f", t0, 12, 5*time.Minute, nil, 0x11, 0x22) + c = firstActive(t, run(t, perDet(t, ExpectedInclude), evs)) + if c.Periodic.Period != 5*time.Minute { + t.Fatalf("picked %v for a 5-minute series", c.Periodic.Period) + } +} + +func TestPeriodicCompetingFiveAndTenMinutes(t *testing.T) { + // pulses at 0, 5, 10, then 20, 25, 30, then 40...: gaps 5,5,10,5,5,10 + var evs []Event + for i, cycle := 0, 0; len(evs) < 15; cycle++ { + for _, off := range []int{0, 5, 10} { + evs = append(evs, relayEvent(t, fmt.Sprintf("c%02d", i), at(time.Duration(cycle*20+off)*time.Minute), 0x11, 0x22)) + i++ + } + } + ev := firstActive(t, run(t, perDet(t, ExpectedInclude), evs)).Periodic + // the chain at the 8th pulse spans gaps 5,5,10,5,5,10,5: 7 gaps over 9 periods + if ev.Period != 5*time.Minute || ev.Coverage != 7.0/9.0 { + t.Fatalf("5-min fundamental with 7/9 coverage expected, got %+v", ev) + } +} + +func TestPeriodicAlarmUsesNoFutureEvidence(t *testing.T) { + pattern := []time.Duration{0, 2, -3, 1, 4, -2, 0, 3, -1, 2, 1, -4, 0, 2, 3, -3, 1, 0} + evs := series(t, "c", t0, len(pattern), 5*time.Minute, func(i int) time.Duration { return pattern[i] * time.Second }, 0x11, 0x22) + // add unrelated noise on the same stream + evs = append(evs, relayEvent(t, "noise1", at(17*time.Minute), 0x11, 0x22), relayEvent(t, "noise2", at(52*time.Minute), 0x11, 0x22)) + sortEvents(evs) + full := run(t, perDet(t, ExpectedInclude), evs) + for cut := 1; cut <= len(evs); cut++ { + prefix := run(t, perDet(t, ExpectedInclude), evs[:cut]) + var want []Candidate + for _, c := range full { + if !c.At.After(evs[cut-1].Time) { + want = append(want, c) + } + } + if !reflect.DeepEqual(prefix, want) { + t.Fatalf("cut %d: candidates depend on later events\nprefix:\n%sfull:\n%s", cut, describe(prefix), describe(want)) + } + } +} + +func TestIrregularChatIsNotPeriodic(t *testing.T) { + for seed := int64(1); seed <= 25; seed++ { + rng := rand.New(rand.NewSource(seed)) + var evs []Event + cur := t0 + for i := 0; i < 200; i++ { + cur = cur.Add(time.Duration(rng.ExpFloat64() * float64(2*time.Minute))) + evs = append(evs, relayEvent(t, fmt.Sprintf("x%03d", i), cur, 0x11, 0x22)) + } + if a := filter(run(t, perDet(t, ExpectedInclude), evs), "per", StateActive); len(a) != 0 { + t.Fatalf("seed %d: random chat called periodic: %+v", seed, a[0].Periodic) + } + } +} + +func TestPeriodicExpectedTrafficIsLabelledNotDeleted(t *testing.T) { + evs := series(t, "e", t0, 12, 5*time.Minute, nil, 0x11, 0x22) + for i := range evs { + evs[i].Traffic = TrafficExpected + } + inc := firstActive(t, run(t, perDet(t, ExpectedInclude), evs)) + sup := firstActive(t, run(t, perDet(t, ExpectedSuppress), evs)) + if inc.Traffic != LabelExpected || inc.Suppressed { + t.Fatalf("include mode: %v suppressed=%v", inc.Traffic, inc.Suppressed) + } + if sup.Traffic != LabelExpected || !sup.Suppressed { + t.Fatalf("suppress mode must mark, not drop: %v suppressed=%v", sup.Traffic, sup.Suppressed) + } + // a single monitored pulse makes the label monitored, never suppressed + evs[3].Traffic = TrafficMonitored + m := firstActive(t, run(t, perDet(t, ExpectedSuppress), evs)) + if m.Traffic != LabelMonitored || m.Suppressed { + t.Fatalf("monitored: %v suppressed=%v", m.Traffic, m.Suppressed) + } +} + +func TestPeriodicEndsAfterQuiet(t *testing.T) { + evs := series(t, "q", t0, 12, 5*time.Minute, nil, 0x11, 0x22) + d := perDet(t, ExpectedInclude) + cs := run(t, d, evs) + cs = append(cs, d.Advance(evs[11].Time.Add(2*time.Hour)).Candidates...) + e := filter(cs, "per", StateEnded) + if len(e) != 1 || !e[0].At.Equal(evs[11].Time.Add(30*time.Minute)) || e[0].Reason != ReasonQuiet { + t.Fatalf("periodic episode must end 30 min after the last pulse:\n%s", describe(cs)) + } +} + +func sortEvents(evs []Event) { + for i := 1; i < len(evs); i++ { + for j := i; j > 0 && evs[j].Time.Before(evs[j-1].Time); j-- { + evs[j], evs[j-1] = evs[j-1], evs[j] + } + } +} + +func TestPeriodicNullModelRejectsChanceChains(t *testing.T) { + // Loose tolerance, coverage and jitter criteria: only the Poisson null + // stands between dense random traffic and a periodic claim. + r := PeriodicRule{Name: "per", Scope: ScopeStream, MinPeriod: time.Minute, MaxPeriod: 10 * time.Minute, + BurstGap: 0, JitterAbs: 20 * time.Second, MaxMissing: 8, MinPulses: 4, HistoryLen: 32, + MinCoverage: 0.1, MaxJitterFraction: 1, MaxChance: 0.01, State: StateParams{EndAfterQuiet: time.Hour}} + for seed := int64(1); seed <= 10; seed++ { + rng := rand.New(rand.NewSource(seed)) + var evs []Event + cur := t0 + for i := 0; i < 300; i++ { + cur = cur.Add(time.Second + time.Duration(rng.ExpFloat64()*float64(time.Minute))) + evs = append(evs, relayEvent(t, fmt.Sprintf("r%03d", i), cur, 0x11, 0x22)) + } + if a := filter(run(t, newDet(t, Config{Periodic: []PeriodicRule{r}}), evs), "per", StateActive); len(a) != 0 { + t.Fatalf("seed %d: chance chain accepted: %+v", seed, a[0].Periodic) + } + } + // a genuine series still passes once its chain is long enough + cs := run(t, newDet(t, Config{Periodic: []PeriodicRule{r}}), series(t, "g", t0, 20, time.Minute, nil, 0x11, 0x22)) + a := filter(cs, "per", StateActive) + if len(a) != 1 || a[0].Periodic.Support <= 4 || a[0].Periodic.Chance > 0.01 { + t.Fatalf("genuine series with a loose rule: %+v", a) + } +} + +// A stray extra pulse must not lock the detector onto a sub-harmonic: once +// the clean train continues, the true period is detected again within +// MinPulses pulses. +func TestPeriodicRecoversFromAStrayPulse(t *testing.T) { + var evs []Event + for i := 0; i <= 7; i++ { // 0..70 min + evs = append(evs, relayEvent(t, fmt.Sprintf("a%02d", i), at(time.Duration(i)*10*time.Minute), 0x11, 0x22)) + } + evs = append(evs, relayEvent(t, "stray", at(75*time.Minute), 0x11, 0x22)) + for i := 8; i <= 30; i++ { // 80..300 min + evs = append(evs, relayEvent(t, fmt.Sprintf("a%02d", i), at(time.Duration(i)*10*time.Minute), 0x11, 0x22)) + } + r := experimentalPeriodic() + r.State.EndAfterQuiet = 11 * time.Minute // end the first episode at the stray pulse + cs := run(t, newDet(t, Config{Periodic: []PeriodicRule{r}}), evs) + a := filter(cs, "per", StateActive) + if len(a) < 2 { + t.Fatalf("clean 10-minute train after one stray pulse was not detected again:\n%s", describe(cs)) + } + re := a[len(a)-1] + if re.Periodic.Period != 10*time.Minute { + t.Fatalf("re-detected period %v, want 10m", re.Periodic.Period) + } + // the chain restarts at 80 min; MinPulses=8 pulses end at 150 min + if re.At.After(at(150 * time.Minute)) { + t.Fatalf("re-detected at %v, want by 150m", re.At.Sub(t0)) + } +} + +// Calibration of the null model: on Poisson traffic, the share of keys that +// ever fire must not exceed MaxChance (the reported chance is an upper +// bound on the expected number of chance chains per key). +func TestPeriodicNullModelIsCalibratedOnPoissonTraffic(t *testing.T) { + r := PeriodicRule{Name: "cal", Scope: ScopeStream, MinPeriod: time.Minute, MaxPeriod: time.Hour, + JitterRel: 0.02, MaxMissing: 1, MinPulses: 4, HistoryLen: 32, MinCoverage: 0.5, + MaxJitterFraction: 1, MaxChance: 0.05, State: StateParams{EndAfterQuiet: time.Hour}} + const keys, pulses = 400, 300 + fired := 0 + var sc periodicScratch + for seed := 0; seed < keys; seed++ { + rng := rand.New(rand.NewSource(int64(seed) + 1000)) + p := newPeriodicState(r.HistoryLen) + tt := t0.UnixNano() + for i := 0; i < pulses; i++ { + tt += int64(rng.ExpFloat64() * float64(10*time.Minute)) + if _, ok := p.observe(tt, TrafficUnclassified, &r, &sc); ok { + fired++ + break + } + } + } + if limit := int(r.MaxChance * keys); fired > limit { + t.Fatalf("%d of %d Poisson keys fired, MaxChance %.2f allows about %d", fired, keys, r.MaxChance, limit) + } +} + +// The stray-pulse recovery must not depend on MinCoverage: with a floor of +// 0.5 or below the sub-multiple meets the criteria too, and a stray pulse +// before the first activation must not lock the period either. Within +// MinPulses clean pulses after the stray, signals report the true period. +func TestPeriodicStrayPulseRecoveryWithLowCoverageFloors(t *testing.T) { + for _, cov := range []float64{0.4, 0.5} { + for _, stray := range []time.Duration{75 * time.Minute, 25 * time.Minute} { + r := experimentalPeriodic() + r.MinCoverage = cov + p := newPeriodicState(r.HistoryLen) + var sc periodicScratch + var sig periodicSignal + var sigAt time.Duration + for i := 0; i <= 30; i++ { + when := time.Duration(i) * 10 * time.Minute + if when > stray && when-10*time.Minute < stray { + if s, ok := p.observe(at(stray).UnixNano(), TrafficUnclassified, &r, &sc); ok { + sig, sigAt = s, stray + } + } + if s, ok := p.observe(at(when).UnixNano(), TrafficUnclassified, &r, &sc); ok { + sig, sigAt = s, when + } + } + if sigAt != 300*time.Minute || sig.ev.Period != 10*time.Minute || sig.ev.Coverage != 1 { + t.Fatalf("MinCoverage %.1f, stray at %v: last signal at %v with %+v", cov, stray, sigAt, sig.ev) + } + // recovered within MinPulses clean pulses of the stray + if suff := sig.ev.SufficientAt.Sub(t0); suff > stray.Truncate(10*time.Minute)+time.Duration(r.MinPulses)*10*time.Minute { + t.Fatalf("MinCoverage %.1f, stray at %v: 10m sufficient only at %v", cov, stray, suff) + } + } + } +} diff --git a/internal/anomaly/rate_test.go b/internal/anomaly/rate_test.go new file mode 100644 index 000000000..984808447 --- /dev/null +++ b/internal/anomaly/rate_test.go @@ -0,0 +1,296 @@ +package anomaly + +import ( + "fmt" + "testing" + "time" +) + +func TestDedupSameIDCountsOnce(t *testing.T) { + d := newDet(t, Config{Rate: []RateRule{rateRule("r2", ScopeGlobal, time.Minute, 2)}}) + e := relayEvent(t, "same", t0, 0x11, 0x22) + if r := d.Observe(e); r.Status != StatusAccepted { + t.Fatal(r.Status) + } + e2 := e + e2.FullRoute = mustRoute(t, 1, 0x22, 0x77) // another observation's route: still the same transmission + if r := d.Observe(e2); r.Status != StatusDuplicate { + t.Fatalf("second copy of one transmission: %v", r.Status) + } + if cs := run(t, d, nil); len(cs) != 0 { + t.Fatalf("one transmission counted twice:\n%s", describe(cs)) + } + if s := d.Snapshot(); s.Duplicates != 1 || s.Accepted != 1 { + t.Fatalf("stats %+v", s) + } +} + +func TestReplayOfSameBatchDoesNotDoubleCount(t *testing.T) { + evs := series(t, "b", t0, 5, 10*time.Second, nil, 0x11, 0x22) + d := newDet(t, Config{Rate: []RateRule{rateRule("r6", ScopeStream, time.Minute, 6)}}) + cs := run(t, d, evs) + // replaying the same batch: all duplicates, still 5 < 6 + for _, e := range evs { + r := d.Observe(e) + if r.Status != StatusDuplicate { + t.Fatalf("replayed event: %v", r.Status) + } + cs = append(cs, r.Candidates...) + } + if len(cs) != 0 { + t.Fatalf("replay double-counted:\n%s", describe(cs)) + } +} + +func TestByteIdenticalButDistinctTransmissionsCountTwice(t *testing.T) { + a := relayEvent(t, "tx-a", t0, 0x11, 0x22) + b := a + b.ID = "tx-b" // identical content, different logical transmission + d := newDet(t, Config{Rate: []RateRule{rateRule("r2", ScopeStream, time.Minute, 2)}}) + cs := run(t, d, []Event{a, b}) + if len(filter(cs, "r2", StateActive)) != 1 { + t.Fatalf("two logical transmissions must count twice:\n%s", describe(cs)) + } +} + +func TestDedupHorizonIsExplicit(t *testing.T) { + lim := experimentalLimits() + lim.DedupHorizon = 2 * time.Minute + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeGlobal, time.Minute, 1)}}) + e := relayEvent(t, "x", t0, 0x11, 0x22) + d.Observe(e) + d.Observe(relayEvent(t, "later", at(3*time.Minute), 0x11, 0x22)) + again := e + again.Time = at(3 * time.Minute) + if r := d.Observe(again); r.Status != StatusAccepted { + t.Fatalf("an ID repeated after the dedup horizon is counted again (documented): %v", r.Status) + } +} + +func TestDedupCapacityEvictionIsReported(t *testing.T) { + lim := experimentalLimits() + lim.MaxDedupIDs = 3 + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeGlobal, time.Minute, 1)}}) + var reduced bool + for i := 0; i < 5; i++ { + r := d.Observe(relayEvent(t, fmt.Sprint(i), at(time.Duration(i)*time.Second), 0x11, 0x22)) + reduced = reduced || r.CoverageReduced + } + s := d.Snapshot() + if s.DedupIDs != 3 || s.DedupEvictedEarly != 2 || !reduced || !s.CoverageReduced() { + t.Fatalf("dedup cap: %+v reduced=%v", s, reduced) + } + if s.EffectiveDedupHorizon >= lim.DedupHorizon { + t.Fatalf("effective horizon must show the capacity cut: %v", s.EffectiveDedupHorizon) + } + // a repeat of an evicted ID is no longer caught: this is the reported limitation + if r := d.Observe(relayEvent(t, "0", at(5*time.Second), 0x11, 0x22)); r.Status != StatusAccepted { + t.Fatalf("evicted ID: %v", r.Status) + } +} + +func TestRateWindowBoundariesAreExact(t *testing.T) { + for _, w := range []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour} { + t.Run(w.String(), func(t *testing.T) { + // exactly W apart: the old event is outside (t-W, t] + d := newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, w, 2)}}) + cs := run(t, d, []Event{relayEvent(t, "a", t0, 1, 2), relayEvent(t, "b", t0.Add(w), 1, 2)}) + if len(cs) != 0 { + t.Fatalf("events exactly %v apart must not share a window:\n%s", w, describe(cs)) + } + // one nanosecond less: both inside + d = newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, w, 2)}}) + cs = run(t, d, []Event{relayEvent(t, "a", t0, 1, 2), relayEvent(t, "b", t0.Add(w-time.Nanosecond), 1, 2)}) + if len(filter(cs, "r", StateActive)) != 1 { + t.Fatalf("events %v-1ns apart must share a window:\n%s", w, describe(cs)) + } + if got := filter(cs, "r", StateActive)[0].Rate.Count; got != 2 { + t.Fatalf("count %d", got) + } + }) + } +} + +func TestSignalHitsOnlyTheMatchingWindows(t *testing.T) { + rules := []RateRule{ + rateRule("w1m", ScopeStream, time.Minute, 5), + rateRule("w5m", ScopeStream, 5*time.Minute, 12), + rateRule("w15m", ScopeStream, 15*time.Minute, 30), + rateRule("w60m", ScopeStream, time.Hour, 100), + } + // 12 events spread 20 s apart: 4 minutes. Max per minute = 3 (<5); per 5 min = 12 (hits); + // per 15 min = 12 (<30); per 60 min = 12 (<100). + evs := series(t, "s", t0, 12, 20*time.Second, nil, 0x11, 0x22) + cs := run(t, newDet(t, Config{Rate: rules}), evs) + for _, name := range []string{"w1m", "w15m", "w60m"} { + if len(filter(cs, name, StateSuspicious)) != 0 { + t.Fatalf("%s fired:\n%s", name, describe(cs)) + } + } + a := filter(cs, "w5m", StateActive) + if len(a) != 1 || !a[0].At.Equal(evs[11].Time) { + t.Fatalf("w5m must fire exactly at the 12th event:\n%s", describe(cs)) + } +} + +func TestSlidingWindowEvictsOldEvents(t *testing.T) { + // 3 events, then a gap longer than the window, then 3 more: never 4 in (t-W, t] + w := 5 * time.Minute + var evs []Event + evs = append(evs, series(t, "a", t0, 3, time.Minute, nil, 1, 2)...) + evs = append(evs, series(t, "b", t0.Add(2*time.Minute+w), 3, time.Minute, nil, 1, 2)...) + cs := run(t, newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, w, 4)}}), evs) + if len(cs) != 0 { + t.Fatalf("evicted events counted:\n%s", describe(cs)) + } + // the 4th event exactly inside the window does fire + evs = append(series(t, "a", t0, 3, time.Minute, nil, 1, 2), relayEvent(t, "x", t0.Add(w-time.Second), 1, 2)) + cs = run(t, newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, w, 4)}}), evs) + if len(filter(cs, "r", StateActive)) != 1 { + t.Fatalf("4 events within %v must fire:\n%s", w, describe(cs)) + } +} + +func TestThirtyPerFiveMinutesOnlyAsExplicitConfig(t *testing.T) { + // The phase-1 example "30 per 5 minutes" is an explicit test config here, + // never a default. + rule := rateRule("example_30_per_5m", ScopeStream, 5*time.Minute, 30) + for _, n := range []int{29, 30} { + evs := series(t, "e", t0, n, 5*time.Second, nil, 0x11, 0x22) + cs := run(t, newDet(t, Config{Rate: []RateRule{rule}}), evs) + got := len(filter(cs, rule.Name, StateActive)) + if (n == 30) != (got == 1) { + t.Fatalf("%d events: %d activations", n, got) + } + } +} + +func TestGlobalDetectsActivitySpreadOverStreamsWithoutAttribution(t *testing.T) { + rules := []RateRule{ + rateRule("per_stream", ScopeStream, 5*time.Minute, 10), + rateRule("per_route", ScopeRouteGroup, 5*time.Minute, 10), + rateRule("global", ScopeGlobal, 5*time.Minute, 30), + } + var evs []Event + // 5 channels x 2 first hops x 3 events = 30 events in 5 minutes + i := 0 + for ch := byte(1); ch <= 5; ch++ { + for hop := byte(0x60); hop <= 0x61; hop++ { + for k := 0; k < 3; k++ { + evs = append(evs, relayEvent(t, fmt.Sprintf("g%03d", i), at(time.Duration(i)*9*time.Second), ch, hop)) + i++ + } + } + } + cs := run(t, newDet(t, Config{Rate: rules}), evs) + if len(filter(cs, "per_stream", StateSuspicious)) != 0 || len(filter(cs, "per_route", StateSuspicious)) != 2 { + t.Fatalf("stream must stay quiet, both route groups (15 each) must fire:\n%s", describe(cs)) + } + g := filter(cs, "global", StateActive) + if len(g) != 1 { + t.Fatalf("global must fire once:\n%s", describe(cs)) + } + k := g[0].Key + if k.Scope != ScopeGlobal || !k.Channel.IsZero() || !k.FirstHop.IsZero() || g[0].Confidence != ConfidenceAggregate { + t.Fatalf("global candidate attributed to a stream/route: %s %v", k, g[0].Confidence) + } + if g[0].Kind != KindRate || g[0].Rate.Count != 30 { + t.Fatalf("global evidence %+v", g[0].Rate) + } +} + +func TestPayloadTypeFilterAndScopes(t *testing.T) { + r := rateRule("only5", ScopeGlobal, time.Minute, 2) + r.PayloadTypes = []PayloadType{5} + adv := relayEvent(t, "adv1", t0, 0x11, 0x22) + adv.PayloadType, adv.Channel = 4, ChannelHash{} + adv2 := adv + adv2.ID, adv2.Time = "adv2", at(time.Second) + cs := run(t, newDet(t, Config{Rate: []RateRule{r}}), []Event{adv, adv2}) + if len(cs) != 0 { + t.Fatal("payload-type filter ignored") + } +} + +func TestWindowSaturationIsFlaggedAndNeverUndercounts(t *testing.T) { + lim := experimentalLimits() + lim.MaxWindowEntries = 4 + rule := rateRule("r", ScopeStream, time.Hour, 20) + evs := series(t, "s", t0, 20, time.Second, nil, 1, 2) // 20 distinct timestamps + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rule}}) + cs := run(t, d, evs) + a := filter(cs, "r", StateActive) + if len(a) != 1 || a[0].Rate.Count != 20 || !a[0].Rate.Saturated || a[0].Coverage&CoverageWindowSaturated == 0 { + t.Fatalf("saturated window: %+v", a) + } + if s := d.Snapshot(); s.WindowMerges == 0 || !s.CoverageReduced() { + t.Fatalf("merges not counted: %+v", s) + } + for _, tb := range d.scopes { + if tb == nil { + continue + } + for _, k := range tb.m { + if k.win.live() > lim.MaxWindowEntries { + t.Fatalf("window holds %d entries > cap", k.win.live()) + } + } + } +} + +func TestWindowCountsMatchBruteForce(t *testing.T) { + widths := []int64{int64(time.Minute), int64(5 * time.Minute)} + w := newWindowSet(len(widths)) + var times []int64 + cur := t0.UnixNano() + for i := 0; i < 400; i++ { + cur += int64((i*7919)%90) * int64(time.Second) / 3 + w.add(cur, TrafficUnclassified, widths, 1_000_000) + times = append(times, cur) + for wi, width := range widths { + n := uint64(0) + for _, x := range times { + if x > cur-width && x <= cur { + n++ + } + } + if w.sums[wi].n != n { + t.Fatalf("i=%d width=%d: sliding %d, brute %d", i, width, w.sums[wi].n, n) + } + } + } +} + +func TestDecidedAtIsNeverBeforeTheEvidence(t *testing.T) { + // the triggering event was only final (all route evidence in) 30 s after + // its data time: the candidate must not claim an earlier decision + a := relayEvent(t, "a", t0, 1, 2) + b := relayEvent(t, "b", at(time.Second), 1, 2) + b.FinalAt = at(31 * time.Second) + cs := run(t, newDet(t, Config{Rate: []RateRule{rateRule("r", ScopeStream, time.Minute, 2)}}), []Event{a, b}) + act := filter(cs, "r", StateActive) + if len(act) != 1 || !act[0].At.Equal(b.Time) || !act[0].DecidedAt.Equal(b.FinalAt) { + t.Fatalf("At %v DecidedAt %v, want At=data time and DecidedAt=FinalAt", act[0].At.Sub(t0), act[0].DecidedAt.Sub(t0)) + } +} + +func TestSaturatedWindowRecoversAfterMergedEntriesExpire(t *testing.T) { + // overflow merges keep counts (overstated while inside the window) and + // must subtract them fully once they expire + lim := experimentalLimits() + lim.MaxWindowEntries = 4 + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeStream, 10*time.Second, 1000)}}) + var evs []Event + for i := 0; i < 30; i++ { + evs = append(evs, relayEvent(t, fmt.Sprint(i), at(time.Duration(i)*time.Second), 1, 2)) + } + evs = append(evs, relayEvent(t, "after", at(5*time.Minute), 1, 2)) + run(t, d, evs) + k := d.scopes[ScopeStream].m[StreamKeyOf(evs[0])] + if got := k.win.sums[0].n; got != 1 { + t.Fatalf("after every merged entry expired the window must hold exactly 1 event, got %d", got) + } + if d.Snapshot().WindowMerges == 0 { + t.Fatal("fixture should saturate the window") + } +} diff --git a/internal/anomaly/state.go b/internal/anomaly/state.go new file mode 100644 index 000000000..920a1d122 --- /dev/null +++ b/internal/anomaly/state.go @@ -0,0 +1,150 @@ +package anomaly + +// episode is the per-(rule, key) state machine +// +// normal -> suspicious -> active -> ended -> (cooldown) -> normal +// +// with every transition checked by legalTransition. It only reacts to hot +// signals and to deadlines in data time; it never reads the wall clock. +type episode struct { + state State + start int64 // first hot signal of the episode (stable ID basis) + suspiciousSince int64 + activeSince int64 + lastHot int64 + endedAt int64 + hot uint32 + version uint32 // bumped on every change; invalidates old deadlines + // label and cov summarize the hot signals of the episode, so that + // time-driven and eviction transitions carry the same traffic label and + // coverage flags as the transitions that opened it. + label TrafficLabel + labelSet bool + cov CoverageFlags +} + +// note merges one hot signal's label and coverage into the episode. +func (ep *episode) note(l TrafficLabel, c CoverageFlags) { + ep.cov |= c + if !ep.labelSet { + ep.label, ep.labelSet = l, true + return + } + ep.label = mergeLabel(ep.label, l) +} + +type transition struct { + from, to State + reason Reason + at int64 + start int64 + summary *EpisodeSummary + label TrafficLabel + cov CoverageFlags +} + +func (ep *episode) summary() *EpisodeSummary { + s := &EpisodeSummary{LastHot: nsTime(ep.lastHot), HotSignals: ep.hot} + if ep.activeSince != 0 { + s.ActiveSince = nsTime(ep.activeSince) + } + return s +} + +func (ep *episode) move(to State, r Reason, at int64, out []transition, withSummary bool) []transition { + if !legalTransition(ep.state, to, r) { + panic("anomaly: illegal state transition " + ep.state.String() + "->" + to.String() + " (" + r.String() + ")") + } + tr := transition{from: ep.state, to: to, reason: r, at: at, start: ep.start, label: ep.label, cov: ep.cov} + if withSummary { + tr.summary = ep.summary() + } + ep.state = to + ep.version++ + return append(out, tr) +} + +// onHot applies a hot signal at data time t. Due deadlines must already have +// been applied (the Detector processes deadlines at or before t first). +func (ep *episode) onHot(t int64, p *StateParams, out []transition) []transition { + if ep.state == StateEnded { + if t-ep.endedAt < int64(p.Cooldown) { + ep.lastHot = t + ep.hot++ + return ep.move(StateActive, ReasonReopened, t, out, false) + } + // cooldown over: a silent return to normal, then a new episode + *ep = episode{version: ep.version + 1} + } + switch ep.state { + case StateNormal: + *ep = episode{state: StateNormal, start: t, suspiciousSince: t, lastHot: t, hot: 1, version: ep.version} + out = ep.move(StateSuspicious, ReasonThresholdCrossed, t, out, false) + if p.ConfirmAfter == 0 { + ep.activeSince = t + out = ep.move(StateActive, ReasonConfirmed, t, out, false) + } + case StateSuspicious: + ep.lastHot = t + ep.hot++ + if t-ep.suspiciousSince >= int64(p.ConfirmAfter) { + ep.activeSince = t + out = ep.move(StateActive, ReasonConfirmed, t, out, false) + } else { + ep.version++ + } + case StateActive: + ep.lastHot = t + ep.hot++ + ep.version++ + } + return out +} + +// deadline returns the next data time at which the episode changes by +// itself, or 0 if none. +func (ep *episode) deadline(p *StateParams) int64 { + switch ep.state { + case StateSuspicious, StateActive: + return ep.lastHot + int64(p.EndAfterQuiet) + case StateEnded: + return ep.endedAt + int64(p.Cooldown) + } + return 0 +} + +// onTime applies every time-driven change due at or before now. Transitions +// are stamped with the time the condition became true, not with now, so the +// output does not depend on when the caller advanced time. +func (ep *episode) onTime(now int64, p *StateParams, out []transition) []transition { + for { + d := ep.deadline(p) + if d == 0 || d > now { + return out + } + switch ep.state { + case StateSuspicious: + out = ep.move(StateNormal, ReasonNotConfirmed, d, out, true) + *ep = episode{version: ep.version} + case StateActive: + out = ep.move(StateEnded, ReasonQuiet, d, out, true) + ep.endedAt = d + case StateEnded: + // cooldown expired: silent return to normal + *ep = episode{version: ep.version + 1} + } + } +} + +// evict closes an open episode because its key state is being dropped. +func (ep *episode) evict(now int64, out []transition) []transition { + switch ep.state { + case StateSuspicious: + out = ep.move(StateNormal, ReasonEvicted, now, out, true) + case StateActive: + out = ep.move(StateEnded, ReasonEvicted, now, out, true) + } + return out +} + +func (ep *episode) open() bool { return ep.state == StateSuspicious || ep.state == StateActive } diff --git a/internal/anomaly/types.go b/internal/anomaly/types.go new file mode 100644 index 000000000..7be5d36da --- /dev/null +++ b/internal/anomaly/types.go @@ -0,0 +1,310 @@ +package anomaly + +import ( + "encoding/hex" + "errors" + "fmt" +) + +// PayloadType is the 4-bit MeshCore payload type from the packet header +// (bits 2-5). The library treats it as an opaque grouping value; it never +// decodes payloads. +type PayloadType uint8 + +// MaxPayloadType is the largest valid PayloadType (4-bit field). +const MaxPayloadType PayloadType = 15 + +// PayloadTypeTrace is the MeshCore TRACE payload type. Its header path holds +// per-hop SNR bytes, not relay hashes, so a TRACE path is never used as route +// evidence. +const PayloadTypeTrace PayloadType = 9 + +// RouteClass is the header route type as observed. The zero value is +// RouteClassUnknown. +type RouteClass uint8 + +const ( + RouteClassUnknown RouteClass = iota + RouteClassFlood + RouteClassTransportFlood + RouteClassDirect + RouteClassTransportDirect + // RouteClassMixed marks a transmission whose observations disagree on the + // route class (for example a flood advert re-shared zero-hop). It is only + // set by the Normalizer and is always low confidence. + RouteClassMixed +) + +var routeClassNames = [...]string{"unknown", "flood", "tflood", "direct", "tdirect", "mixed"} + +func (c RouteClass) String() string { + if int(c) < len(routeClassNames) { + return routeClassNames[c] + } + return fmt.Sprintf("routeclass(%d)", uint8(c)) +} + +func (c RouteClass) valid() bool { return int(c) < len(routeClassNames) } + +// isFlood reports whether packets of this class carry the relay path in +// order of forwarding (path[0] is the first relay after the originator). +func (c RouteClass) isFlood() bool { + return c == RouteClassFlood || c == RouteClassTransportFlood +} + +// RouteKind says what the first hop of a transmission means. Only +// RouteKindRelay has a first hop; every other kind is a separate +// low-confidence class and never names a relay. +type RouteKind uint8 + +const ( + RouteKindUnknown RouteKind = iota + // RouteKindRelay: a flood observation with at least one hop. The first + // hop is the first relay after the originator as seen on the wire. It is + // observation evidence, not an identity of the originator. + RouteKindRelay + // RouteKindNoPath: a flood observation with zero hops. The observer heard + // the packet directly; the observer is NOT assumed to be the sender. + RouteKindNoPath + // RouteKindDirect: a direct-routed observation. path[0] is the next hop + // of a pre-set route, not a first relay, so no hop is kept. + RouteKindDirect +) + +var routeKindNames = [...]string{"unknown", "relay", "nopath", "direct"} + +func (k RouteKind) String() string { + if int(k) < len(routeKindNames) { + return routeKindNames[k] + } + return fmt.Sprintf("routekind(%d)", uint8(k)) +} + +func (k RouteKind) valid() bool { return int(k) < len(routeKindNames) } + +// MaxHopWidth is the widest hop hash MeshCore encodes (path_len bits 6-7). +const MaxHopWidth = 3 + +// Hop is one relay hash prefix together with its byte width. A 1-byte FD and +// the first byte of a 2-byte FD12 are different Hops: width is part of the +// value. The zero Hop means "no hop". +type Hop struct { + w uint8 + b [MaxHopWidth]byte +} + +// NewHop builds a Hop from 1 to 3 bytes. +func NewHop(b []byte) (Hop, error) { + if len(b) < 1 || len(b) > MaxHopWidth { + return Hop{}, fmt.Errorf("anomaly: hop width %d outside 1..%d", len(b), MaxHopWidth) + } + var h Hop + h.w = uint8(len(b)) + copy(h.b[:], b) + return h, nil +} + +// Width returns the hop width in bytes (0 for the zero Hop). +func (h Hop) Width() int { return int(h.w) } + +// IsZero reports whether h is the zero Hop. +func (h Hop) IsZero() bool { return h.w == 0 } + +// Bytes returns a copy of the hop bytes. +func (h Hop) Bytes() []byte { return append([]byte(nil), h.b[:h.w]...) } + +func (h Hop) encode() string { + if h.w == 0 { + return "-" + } + return fmt.Sprintf("%d:%s", h.w, hex.EncodeToString(h.b[:h.w])) +} + +// MaxChannelHashLen bounds the opaque channel hash. MeshCore uses 1 byte +// today; the type allows up to 4 so a wider hash does not change the API. +const MaxChannelHashLen = 4 + +// ChannelHash is an opaque channel identifier as it appears on the wire. The +// library never interprets it and never relies on names such as "enc_XX" or +// on whether this instance could decrypt the channel. The zero value means +// "no channel" (for payload types without one), which is different from a +// 1-byte hash 0x00. +type ChannelHash struct { + n uint8 + b [MaxChannelHashLen]byte +} + +// NewChannelHash builds a ChannelHash from 1 to 4 bytes. +func NewChannelHash(b []byte) (ChannelHash, error) { + if len(b) < 1 || len(b) > MaxChannelHashLen { + return ChannelHash{}, fmt.Errorf("anomaly: channel hash length %d outside 1..%d", len(b), MaxChannelHashLen) + } + var c ChannelHash + c.n = uint8(len(b)) + copy(c.b[:], b) + return c, nil +} + +// Len returns the hash length in bytes (0 = no channel). +func (c ChannelHash) Len() int { return int(c.n) } + +// IsZero reports whether c is "no channel". +func (c ChannelHash) IsZero() bool { return c.n == 0 } + +// Bytes returns a copy of the hash bytes. +func (c ChannelHash) Bytes() []byte { return append([]byte(nil), c.b[:c.n]...) } + +func (c ChannelHash) encode() string { + if c.n == 0 { + return "-" + } + return fmt.Sprintf("%d:%s", c.n, hex.EncodeToString(c.b[:c.n])) +} + +// MaxPathBytes is the MeshCore limit on path bytes (MAX_PATH_SIZE). +const MaxPathBytes = 64 + +// MaxWireBytes is the MeshCore maximum frame size (MAX_TRANS_UNIT). +const MaxWireBytes = 255 + +// Route is the full ordered route of one observation: hop width plus the hop +// bytes in forwarding order. It is evidence only and never part of a stream +// key, because full routes fragment far more than first hops. +type Route struct { + w uint8 + hops []byte +} + +// NewRoute builds a Route from a hop width (1..3) and the concatenated hop +// bytes. An empty route (no hops) is written as NewRoute(0, nil). +func NewRoute(width int, hops []byte) (Route, error) { + if len(hops) == 0 { + if width != 0 { + return Route{}, errors.New("anomaly: empty route must have width 0") + } + return Route{}, nil + } + if width < 1 || width > MaxHopWidth { + return Route{}, fmt.Errorf("anomaly: route hop width %d outside 1..%d", width, MaxHopWidth) + } + if len(hops)%width != 0 { + return Route{}, errors.New("anomaly: route length is not a multiple of its hop width") + } + if len(hops) > MaxPathBytes { + return Route{}, fmt.Errorf("anomaly: route longer than %d bytes", MaxPathBytes) + } + return Route{w: uint8(width), hops: append([]byte(nil), hops...)}, nil +} + +// Width returns the hop width (0 for an empty route). +func (r Route) Width() int { return int(r.w) } + +// Hops returns the number of hops. +func (r Route) Hops() int { + if r.w == 0 { + return 0 + } + return len(r.hops) / int(r.w) +} + +// Bytes returns a copy of the concatenated hop bytes. +func (r Route) Bytes() []byte { return append([]byte(nil), r.hops...) } + +// First returns the first hop (zero Hop for an empty route). +func (r Route) First() Hop { + if r.w == 0 { + return Hop{} + } + var h Hop + h.w = r.w + copy(h.b[:], r.hops[:r.w]) + return h +} + +func (r Route) equal(o Route) bool { + if r.w != o.w || len(r.hops) != len(o.hops) { + return false + } + for i := range r.hops { + if r.hops[i] != o.hops[i] { + return false + } + } + return true +} + +// DecryptStatus records whether the caller's instance decrypted the payload. +// It is metadata only: it is never part of a key and never changes counting. +// "Not decrypted here" is a property of this instance's keys, not of the +// radio traffic. +type DecryptStatus uint8 + +const ( + DecryptUnknown DecryptStatus = iota + DecryptDecrypted + DecryptNotDecrypted +) + +func (d DecryptStatus) valid() bool { return d <= DecryptNotDecrypted } + +// TrafficClass is a caller-supplied label. The library never infers it and +// never hardcodes channel names; a caller that knows a channel is expected +// automation (by its own configuration) labels those events. +type TrafficClass uint8 + +const ( + // TrafficUnclassified: no label supplied. + TrafficUnclassified TrafficClass = iota + // TrafficExpected: the caller asserts this is expected automated traffic. + TrafficExpected + // TrafficMonitored: the caller wants this traffic evaluated and never + // suppressed. + TrafficMonitored +) + +func (t TrafficClass) valid() bool { return t <= TrafficMonitored } + +// TrafficLabel summarizes the caller labels of the events behind a candidate. +type TrafficLabel uint8 + +const ( + LabelUnclassified TrafficLabel = iota // no event carried a label + LabelExpected // every event was TrafficExpected + LabelMixed // some expected, some not + LabelMonitored // at least one event was TrafficMonitored +) + +var labelNames = [...]string{"unclassified", "expected", "mixed", "monitored"} + +func (l TrafficLabel) String() string { + if int(l) < len(labelNames) { + return labelNames[l] + } + return fmt.Sprintf("label(%d)", uint8(l)) +} + +// mergeLabel combines the labels of two sets of events. +func mergeLabel(a, b TrafficLabel) TrafficLabel { + switch { + case a == b: + return a + case a == LabelMonitored || b == LabelMonitored: + return LabelMonitored + case a == LabelExpected || b == LabelExpected || a == LabelMixed || b == LabelMixed: + return LabelMixed + } + return LabelUnclassified +} + +func labelFromCounts(total, expected, monitored uint64) TrafficLabel { + switch { + case monitored > 0: + return LabelMonitored + case total > 0 && expected == total: + return LabelExpected + case expected > 0: + return LabelMixed + default: + return LabelUnclassified + } +} diff --git a/internal/anomaly/window.go b/internal/anomaly/window.go new file mode 100644 index 000000000..2056499cf --- /dev/null +++ b/internal/anomaly/window.go @@ -0,0 +1,130 @@ +package anomaly + +// Sliding-window counting for rate rules. +// +// Semantics: count(W, t) is the number of accepted events of the key with +// t-W < time <= t, evaluated at the time t of the event just added. An +// event exactly W older than t is outside the window. Events reach a key in +// non-decreasing data time (the Detector's reorder buffer guarantees it). +// +// Memory: one entry per distinct timestamp in the longest window, capped by +// Limits.MaxWindowEntries. On overflow the two oldest entries are merged; +// the merged counts stay in every window that held the newer entry until it +// expires, so counts can be overstated (never understated). Every merge is +// counted and flagged on candidates of that key. + +type winEntry struct { + t int64 + n, exp, mon uint32 + _ uint32 // pad to 24 bytes, keeps entries aligned +} + +type winSum struct { + n, exp, mon uint64 +} + +// windowSet tracks every distinct window width used by the rate rules of one +// scope, for one key. +type windowSet struct { + entries []winEntry + head int + starts []int // per width: absolute index of the first entry inside + sums []winSum + lastMerge int64 // data time of the last overflow merge (0 = never) +} + +func newWindowSet(nWidths int) *windowSet { + return &windowSet{starts: make([]int, nWidths), sums: make([]winSum, nWidths)} +} + +// add records one event at time t. widths are in nanoseconds; maxEntries +// bounds the live entries. It returns true if an overflow merge happened. +func (w *windowSet) add(t int64, cls TrafficClass, widths []int64, maxEntries int) (merged bool) { + var e, m uint32 + switch cls { + case TrafficExpected: + e = 1 + case TrafficMonitored: + m = 1 + } + if last := len(w.entries) - 1; last >= w.head && w.entries[last].t == t { + w.entries[last].n++ + w.entries[last].exp += e + w.entries[last].mon += m + } else { + w.entries = append(w.entries, winEntry{t: t, n: 1, exp: e, mon: m}) + } + for i, width := range widths { + s := &w.sums[i] + s.n++ + s.exp += uint64(e) + s.mon += uint64(m) + lo := t - width + for w.starts[i] < len(w.entries) && w.entries[w.starts[i]].t <= lo { + x := w.entries[w.starts[i]] + s.n -= uint64(x.n) + s.exp -= uint64(x.exp) + s.mon -= uint64(x.mon) + w.starts[i]++ + } + } + // Entries before every window start can go. + minStart := len(w.entries) + for _, st := range w.starts { + if st < minStart { + minStart = st + } + } + for w.head < minStart { + w.entries[w.head] = winEntry{} + w.head++ + } + for len(w.entries)-w.head > maxEntries { + w.mergeOldest() + w.lastMerge = t + merged = true + } + w.compact() + return merged +} + +// mergeOldest folds entries[head] into entries[head+1]. +func (w *windowSet) mergeOldest() { + old := w.entries[w.head] + nxt := &w.entries[w.head+1] + nxt.n += old.n + nxt.exp += old.exp + nxt.mon += old.mon + oldIdx := w.head + w.entries[oldIdx] = winEntry{} + w.head++ + for i := range w.starts { + switch { + case w.starts[i] <= oldIdx: + // old was inside; its counts now live in nxt, also inside. + w.starts[i] = w.head + case w.starts[i] == w.head: + // old was outside but nxt is inside: nxt now carries old's + // counts, so the window overstates by them until nxt expires. + w.sums[i].n += uint64(old.n) + w.sums[i].exp += uint64(old.exp) + w.sums[i].mon += uint64(old.mon) + } + } +} + +func (w *windowSet) compact() { + if w.head >= 32 && w.head*2 >= len(w.entries) { + n := copy(w.entries, w.entries[w.head:]) + for i := n; i < len(w.entries); i++ { + w.entries[i] = winEntry{} + } + w.entries = w.entries[:n] + for i := range w.starts { + w.starts[i] -= w.head + } + w.head = 0 + } +} + +func (w *windowSet) live() int { return len(w.entries) - w.head } From 91ccd4a2008cf311ddd558525002b3fe575e95f8 Mon Sep 17 00:00:00 2001 From: Openclaw Date: Fri, 25 Sep 2026 08:53:02 +0200 Subject: [PATCH 2/9] feat(anomaly): add offline JSON Lines replay harness Replay runs one Detector over normalized events from a neutral JSON Lines format with an explicit [Start, End) range and returns a deterministic report. It uses no database, no network and no wall clock, and there is no command-line tool: TestOfflineReplay runs it from go test with local files only. - Line length is bounded exactly. - Unknown fields and trailing data are rejected. - It drains after every call, so the harness itself never drops candidates. The fixtures are synthetic (2030 dates, neutral hashes), with a golden report. Co-Authored-By: Claude Opus 5.5 --- internal/anomaly/replay.go | 421 ++++++++++++++++ internal/anomaly/replay_test.go | 260 ++++++++++ internal/anomaly/testdata/replay_config.json | 75 +++ internal/anomaly/testdata/replay_events.jsonl | 54 ++ internal/anomaly/testdata/replay_golden.json | 468 ++++++++++++++++++ 5 files changed, 1278 insertions(+) create mode 100644 internal/anomaly/replay.go create mode 100644 internal/anomaly/replay_test.go create mode 100644 internal/anomaly/testdata/replay_config.json create mode 100644 internal/anomaly/testdata/replay_events.jsonl create mode 100644 internal/anomaly/testdata/replay_golden.json diff --git a/internal/anomaly/replay.go b/internal/anomaly/replay.go new file mode 100644 index 000000000..e19da4fc6 --- /dev/null +++ b/internal/anomaly/replay.go @@ -0,0 +1,421 @@ +package anomaly + +import ( + "bufio" + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +// Offline replay (EXPERIMENTAL). Replay reads normalized events from a +// neutral JSON Lines format, runs one Detector over an explicit [Start, End) +// data-time range and returns a deterministic report. It never opens a +// database, never touches the network and never reads the wall clock. There +// is deliberately no command-line tool; see TestOfflineReplay for running it +// from `go test` with local files. +// +// Input line format (one final event per line; unknown fields and trailing +// data after the JSON object are rejected): +// +// {"id":"tx-1","time":"2030-01-07T12:00:00Z","final_at":"2030-01-07T12:00:05Z", +// "payload_type":5,"channel":"ab","route_class":"flood","route_kind":"relay", +// "first_hop":"1:5c","full_route":"1:5c11","wire_bytes":60,"payload_bytes":35, +// "traffic":"unclassified","decryption":"unknown","truncated":false} +// +// "channel" is plain hex ("" = no channel); hops and routes are +// ":" ("" or "-" = none); "payload_bytes" may be omitted +// (unknown). Lines must be in data-time order within the config's +// ReorderDelay; older lines are counted as late (see Limits.ReorderDelay for +// lines with equal times). +type ReplayRecord struct { + ID string `json:"id"` + Time time.Time `json:"time"` + FinalAt time.Time `json:"final_at,omitempty"` + PayloadType int `json:"payload_type"` + Channel string `json:"channel"` + RouteClass string `json:"route_class"` + RouteKind string `json:"route_kind"` + FirstHop string `json:"first_hop"` + FullRoute string `json:"full_route"` + WireBytes int `json:"wire_bytes"` + PayloadBytes *int `json:"payload_bytes,omitempty"` + Traffic string `json:"traffic"` + Decryption string `json:"decryption"` + Truncated bool `json:"truncated,omitempty"` +} + +// maxReplayLineBytes caps ReplayOptions.MaxLineBytes. +const maxReplayLineBytes = 1 << 30 + +// onlyValue reports whether the decoder consumed all of b but whitespace. +func onlyValue(dec *json.Decoder, b []byte) bool { + return len(bytes.TrimSpace(b[dec.InputOffset():])) == 0 +} + +// ReplayOptions bound the replay explicitly. +type ReplayOptions struct { + // Start and End are required and must lie in the valid event time range + // (End plus the config's ReorderDelay included): events outside + // [Start, End) are skipped and counted. + Start, End time.Time + // MaxLineBytes bounds one input line, excluding its line terminator + // (required, > 0). + MaxLineBytes int +} + +// ReplayInputStats counts input lines by outcome. +type ReplayInputStats struct { + Lines, Accepted, Duplicates, Late, Invalid, OutOfRange int +} + +// ReplayReport is the deterministic output of Replay. +type ReplayReport struct { + Experimental bool + Start, End time.Time + Input ReplayInputStats + Candidates []Candidate + Stats Stats + // CoverageReduced is true if any capacity limit cut evidence. + CoverageReduced bool +} + +func parseEnum(s string, names []string, what string) (int, error) { + for i, n := range names { + if n == s { + return i, nil + } + } + return 0, fmt.Errorf("anomaly: unknown %s", what) +} + +func parseWidthHex(s string) (int, []byte, error) { + if s == "" || s == "-" { + return 0, nil, nil + } + i := strings.IndexByte(s, ':') + if i < 1 { + return 0, nil, errors.New("anomaly: expected :") + } + w, err := strconv.Atoi(s[:i]) + if err != nil { + return 0, nil, errors.New("anomaly: bad width") + } + b, err := hex.DecodeString(s[i+1:]) + if err != nil { + return 0, nil, errors.New("anomaly: bad hex") + } + return w, b, nil +} + +var trafficNames = []string{"unclassified", "expected", "monitored"} +var decryptNames = []string{"unknown", "decrypted", "not_decrypted"} + +// Event converts a record. Errors never echo the record's values. +func (r ReplayRecord) Event() (Event, error) { + e := Event{ID: TxID(r.ID), Time: r.Time.UTC(), WireBytes: r.WireBytes, Truncated: r.Truncated} + if !r.FinalAt.IsZero() { + e.FinalAt = r.FinalAt.UTC() + } + if r.PayloadType < 0 || r.PayloadType > int(MaxPayloadType) { + return Event{}, errors.New("anomaly: payload_type out of range") + } + e.PayloadType = PayloadType(r.PayloadType) + if r.Channel != "" { + b, err := hex.DecodeString(r.Channel) + if err != nil { + return Event{}, errors.New("anomaly: bad channel hex") + } + if e.Channel, err = NewChannelHash(b); err != nil { + return Event{}, err + } + } + rc, err := parseEnum(r.RouteClass, routeClassNames[:], "route_class") + if err != nil { + return Event{}, err + } + rk, err := parseEnum(r.RouteKind, routeKindNames[:], "route_kind") + if err != nil { + return Event{}, err + } + e.RouteClass, e.RouteKind = RouteClass(rc), RouteKind(rk) + if w, b, err := parseWidthHex(r.FirstHop); err != nil { + return Event{}, err + } else if w != 0 { + if w != len(b) { + return Event{}, errors.New("anomaly: first_hop width mismatch") + } + if e.FirstHop, err = NewHop(b); err != nil { + return Event{}, err + } + } + if w, b, err := parseWidthHex(r.FullRoute); err != nil { + return Event{}, err + } else if w != 0 { + if e.FullRoute, err = NewRoute(w, b); err != nil { + return Event{}, err + } + } + if r.PayloadBytes != nil { + e.PayloadBytes, e.PayloadKnown = *r.PayloadBytes, true + } + tc, err := parseEnum(orDefault(r.Traffic, "unclassified"), trafficNames, "traffic") + if err != nil { + return Event{}, err + } + dc, err := parseEnum(orDefault(r.Decryption, "unknown"), decryptNames, "decryption") + if err != nil { + return Event{}, err + } + e.Traffic, e.Decryption = TrafficClass(tc), DecryptStatus(dc) + return e, e.Validate() +} + +func orDefault(s, d string) string { + if s == "" { + return d + } + return s +} + +// Replay runs cfg over the JSON Lines in r. Invalid lines are counted, not +// fatal; a line longer than MaxLineBytes, a read error or an invalid config +// is fatal. +func Replay(r io.Reader, cfg Config, opt ReplayOptions) (ReplayReport, error) { + var rep ReplayReport + if opt.Start.IsZero() || opt.End.IsZero() || !opt.End.After(opt.Start) { + return rep, errors.New("anomaly: replay needs an explicit Start < End") + } + if opt.MaxLineBytes <= 0 || opt.MaxLineBytes > maxReplayLineBytes { + return rep, errors.New("anomaly: replay needs MaxLineBytes in 1..1 GiB") + } + if cfg.HistoryStart.IsZero() { + cfg.HistoryStart = opt.Start + } + d, err := New(cfg) + if err != nil { + return rep, err + } + // the final Advance must be inside the valid range, or deadlines after + // the last event would silently not run + if !validTime(opt.Start) || !validTime(opt.End.Add(cfg.Limits.ReorderDelay)) { + return rep, errors.New("anomaly: replay Start/End outside the valid time range") + } + rep.Experimental = true + rep.Start, rep.End = opt.Start.UTC(), opt.End.UTC() + sc := bufio.NewScanner(r) + // The scanner's limit is max(cap(buf), max) and includes the terminator + // ("\r\n"): keep the buffer within it and check the line length below. + limit := opt.MaxLineBytes + 2 + initial := 4096 + if limit < initial { + initial = limit + } + sc.Buffer(make([]byte, 0, initial), limit) + collect := func(res Result) { + rep.Candidates = append(rep.Candidates, res.Candidates...) + rep.CoverageReduced = rep.CoverageReduced || res.CoverageReduced + } + // drain after every call, so the per-call cap never overflows the + // detector's candidate buffer because of the harness + drain := func(res Result) { + for collect(res); res.More; { + res = d.Drain() + collect(res) + } + } + for sc.Scan() { + line := sc.Bytes() + if len(line) > opt.MaxLineBytes { + return rep, errors.New("anomaly: replay input: line longer than MaxLineBytes") + } + if len(strings.TrimSpace(string(line))) == 0 { + continue + } + rep.Input.Lines++ + var rec ReplayRecord + dec := json.NewDecoder(strings.NewReader(string(line))) + dec.DisallowUnknownFields() + if err := dec.Decode(&rec); err != nil || !onlyValue(dec, line) { + rep.Input.Invalid++ + continue + } + e, err := rec.Event() + if err != nil { + rep.Input.Invalid++ + continue + } + if e.Time.Before(opt.Start) || !e.Time.Before(opt.End) { + rep.Input.OutOfRange++ + continue + } + res := d.Observe(e) + switch res.Status { + case StatusAccepted: + rep.Input.Accepted++ + case StatusDuplicate: + rep.Input.Duplicates++ + case StatusLate: + rep.Input.Late++ + default: + rep.Input.Invalid++ + } + drain(res) + } + if err := sc.Err(); err != nil { + return rep, fmt.Errorf("anomaly: replay input: %w", err) + } + drain(d.Flush()) + // input is complete: run deadlines up to (not including) End + drain(d.Advance(opt.End.Add(cfg.Limits.ReorderDelay - time.Nanosecond))) + rep.Stats = d.Snapshot() + rep.CoverageReduced = rep.CoverageReduced || rep.Stats.CoverageReduced() + return rep, nil +} + +// ---- JSON config for the replay harness ---- + +// Duration is a JSON-friendly duration ("90s", "5m"). +type Duration time.Duration + +// UnmarshalJSON parses a Go duration string. +func (d *Duration) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return errors.New("anomaly: durations are strings such as \"5m\"") + } + v, err := time.ParseDuration(s) + if err != nil { + return err + } + *d = Duration(v) + return nil +} + +// ReplayConfig is the JSON form of Config used by the offline harness. +// Every field is required exactly as in Config; there are no defaults. +type ReplayConfig struct { + Limits struct { + MaxKeysPerScope int `json:"max_keys_per_scope"` + StateTTL Duration `json:"state_ttl"` + DedupHorizon Duration `json:"dedup_horizon"` + MaxDedupIDs int `json:"max_dedup_ids"` + ReorderDelay Duration `json:"reorder_delay"` + MaxReorderBuffer int `json:"max_reorder_buffer"` + MaxWindowEntries int `json:"max_window_entries"` + MaxCandidatesPerCall int `json:"max_candidates_per_call"` + MaxPendingCandidates int `json:"max_pending_candidates"` + } `json:"limits"` + Expected string `json:"expected"` // "include" or "suppress" + Rate []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Window Duration `json:"window"` + Threshold int `json:"threshold"` + PayloadTypes []PayloadType `json:"payload_types"` + State stateJSON `json:"state"` + } `json:"rate"` + NewStream []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Window Duration `json:"window"` + MinCount int `json:"min_count"` + CensorWindow Duration `json:"censor_window"` + QuietPeriod Duration `json:"quiet_period"` + PayloadTypes []PayloadType `json:"payload_types"` + State stateJSON `json:"state"` + } `json:"new_stream"` + Periodic []struct { + Name string `json:"name"` + Scope string `json:"scope"` + PayloadTypes []PayloadType `json:"payload_types"` + MinPeriod Duration `json:"min_period"` + MaxPeriod Duration `json:"max_period"` + BurstGap Duration `json:"burst_gap"` + JitterAbs Duration `json:"jitter_abs"` + JitterRel float64 `json:"jitter_rel"` + MaxMissing int `json:"max_missing"` + MinPulses int `json:"min_pulses"` + HistoryLen int `json:"history_len"` + MinCoverage float64 `json:"min_coverage"` + MaxJitterFraction float64 `json:"max_jitter_fraction"` + MaxChance float64 `json:"max_chance"` + State stateJSON `json:"state"` + } `json:"periodic"` +} + +type stateJSON struct { + ConfirmAfter Duration `json:"confirm_after"` + EndAfterQuiet Duration `json:"end_after_quiet"` + Cooldown Duration `json:"cooldown"` +} + +func (s stateJSON) params() StateParams { + return StateParams{ConfirmAfter: time.Duration(s.ConfirmAfter), EndAfterQuiet: time.Duration(s.EndAfterQuiet), Cooldown: time.Duration(s.Cooldown)} +} + +// ParseReplayConfig decodes a JSON config (unknown fields and trailing data +// rejected) and validates it. +func ParseReplayConfig(b []byte) (Config, error) { + var rc ReplayConfig + dec := json.NewDecoder(strings.NewReader(string(b))) + dec.DisallowUnknownFields() + if err := dec.Decode(&rc); err != nil { + return Config{}, fmt.Errorf("anomaly: replay config: %w", err) + } + if !onlyValue(dec, b) { + return Config{}, errors.New("anomaly: replay config: trailing data") + } + scope := func(s string) (Scope, error) { + i, err := parseEnum(s, scopeNames[:], "scope") + if err != nil || i == 0 { + return 0, errors.New("anomaly: unknown scope") + } + return Scope(i), nil + } + l := rc.Limits + cfg := Config{Limits: Limits{MaxKeysPerScope: l.MaxKeysPerScope, StateTTL: time.Duration(l.StateTTL), + DedupHorizon: time.Duration(l.DedupHorizon), MaxDedupIDs: l.MaxDedupIDs, ReorderDelay: time.Duration(l.ReorderDelay), + MaxReorderBuffer: l.MaxReorderBuffer, MaxWindowEntries: l.MaxWindowEntries, + MaxCandidatesPerCall: l.MaxCandidatesPerCall, MaxPendingCandidates: l.MaxPendingCandidates}} + switch rc.Expected { + case "include": + cfg.Expected.Mode = ExpectedInclude + case "suppress": + cfg.Expected.Mode = ExpectedSuppress + } + for _, r := range rc.Rate { + s, err := scope(r.Scope) + if err != nil { + return Config{}, err + } + cfg.Rate = append(cfg.Rate, RateRule{Name: r.Name, Scope: s, Window: time.Duration(r.Window), Threshold: r.Threshold, + PayloadTypes: r.PayloadTypes, State: r.State.params()}) + } + for _, r := range rc.NewStream { + s, err := scope(r.Scope) + if err != nil { + return Config{}, err + } + cfg.NewStream = append(cfg.NewStream, NewStreamRule{Name: r.Name, Scope: s, Window: time.Duration(r.Window), + MinCount: r.MinCount, CensorWindow: time.Duration(r.CensorWindow), QuietPeriod: time.Duration(r.QuietPeriod), + PayloadTypes: r.PayloadTypes, State: r.State.params()}) + } + for _, r := range rc.Periodic { + s, err := scope(r.Scope) + if err != nil { + return Config{}, err + } + cfg.Periodic = append(cfg.Periodic, PeriodicRule{Name: r.Name, Scope: s, PayloadTypes: r.PayloadTypes, + MinPeriod: time.Duration(r.MinPeriod), MaxPeriod: time.Duration(r.MaxPeriod), BurstGap: time.Duration(r.BurstGap), + JitterAbs: time.Duration(r.JitterAbs), JitterRel: r.JitterRel, MaxMissing: r.MaxMissing, MinPulses: r.MinPulses, + HistoryLen: r.HistoryLen, MinCoverage: r.MinCoverage, MaxJitterFraction: r.MaxJitterFraction, + MaxChance: r.MaxChance, State: r.State.params()}) + } + return cfg, cfg.Validate() +} diff --git a/internal/anomaly/replay_test.go b/internal/anomaly/replay_test.go new file mode 100644 index 000000000..5955f4e38 --- /dev/null +++ b/internal/anomaly/replay_test.go @@ -0,0 +1,260 @@ +package anomaly + +import ( + "bytes" + "encoding/json" + "flag" + "math" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +var ( + updateGolden = flag.Bool("anomaly.update", false, "rewrite testdata/replay_golden.json") + // Offline harness (EXPERIMENTAL): run a replay over local files, e.g. + // + // go test -run TestOfflineReplay -anomaly.replay.events=/path/events.jsonl \ + // -anomaly.replay.config=/path/config.json -anomaly.replay.start=2030-01-01T00:00:00Z \ + // -anomaly.replay.end=2030-02-01T00:00:00Z -anomaly.replay.out=/path/report.json + // + // It reads only the given files and writes only the given output file. + replayEvents = flag.String("anomaly.replay.events", "", "JSON Lines of normalized events") + replayConfig = flag.String("anomaly.replay.config", "", "replay config JSON") + replayStart = flag.String("anomaly.replay.start", "", "RFC3339 start (inclusive)") + replayEnd = flag.String("anomaly.replay.end", "", "RFC3339 end (exclusive)") + replayOut = flag.String("anomaly.replay.out", "", "report output path") +) + +func loadFixtureReplay(t *testing.T) ReplayReport { + t.Helper() + cb, err := os.ReadFile(filepath.Join("testdata", "replay_config.json")) + if err != nil { + t.Fatal(err) + } + cfg, err := ParseReplayConfig(cb) + if err != nil { + t.Fatal(err) + } + f, err := os.Open(filepath.Join("testdata", "replay_events.jsonl")) + if err != nil { + t.Fatal(err) + } + defer f.Close() + rep, err := Replay(f, cfg, ReplayOptions{Start: t0, End: t0.Add(24 * time.Hour), MaxLineBytes: 1 << 16}) + if err != nil { + t.Fatal(err) + } + return rep +} + +func TestReplayFixtureIsDeterministicAndMatchesGolden(t *testing.T) { + a, _ := json.MarshalIndent(loadFixtureReplay(t), "", " ") + b, _ := json.MarshalIndent(loadFixtureReplay(t), "", " ") + if !bytes.Equal(a, b) { + t.Fatal("replay output is not deterministic") + } + golden := filepath.Join("testdata", "replay_golden.json") + if *updateGolden { + if err := os.WriteFile(golden, append(a, '\n'), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatalf("missing golden file (run with -anomaly.update): %v", err) + } + if !bytes.Equal(append(a, '\n'), want) { + t.Fatal("replay output differs from testdata/replay_golden.json (run with -anomaly.update after reviewing)") + } +} + +func TestReplayFixtureContent(t *testing.T) { + rep := loadFixtureReplay(t) + in := rep.Input + if in.Duplicates != 1 || in.Invalid != 1 || in.Late != 1 || in.OutOfRange != 1 || in.Accepted != 50 { + t.Fatalf("input accounting %+v", in) + } + got := map[string]int{} + for _, c := range rep.Candidates { + if c.To == StateActive { + got[c.Rule]++ + } + } + // the global burst activates once; the spread burst never reaches 10 per + // stream in 5 minutes. The new-stream rule fires twice: for the periodic + // stream that starts at the start of the data (left-censored) and for the + // stream that appears later (not censored). The periodic rule fires for + // the 5-minute series and for the later stream, which is regular too. + want := map[string]int{"periodic_example": 2, "global_5m_example": 1, "new_stream_example": 2} + for k, v := range want { + if got[k] != v { + t.Fatalf("activations %v, want %v", got, want) + } + } + if got["stream_5m_example"] != 0 { + t.Fatalf("stream rule fired on a spread burst: %v", got) + } + var censored, fresh int + for _, c := range rep.Candidates { + if c.Rule == "new_stream_example" && c.To == StateActive { + if c.NewStream.Censored { + censored++ + if !c.At.Before(t0.Add(time.Hour)) || c.Confidence != ConfidenceLow { + t.Fatalf("censored candidate %+v", c) + } + } else { + fresh++ + } + } + } + if censored != 1 || fresh != 1 { + t.Fatalf("censored %d fresh %d", censored, fresh) + } + for _, c := range rep.Candidates { + if c.Rule == "global_5m_example" && (c.Key.Scope != ScopeGlobal || !c.Key.FirstHop.IsZero()) { + t.Fatal("global candidate attributed to a route") + } + } +} + +func TestReplayRejectsMissingBoundsAndLongLines(t *testing.T) { + cb, _ := os.ReadFile(filepath.Join("testdata", "replay_config.json")) + cfg, _ := ParseReplayConfig(cb) + if _, err := Replay(strings.NewReader(""), cfg, ReplayOptions{MaxLineBytes: 100}); err == nil { + t.Fatal("replay without explicit Start/End accepted") + } + long := strings.Repeat("x", 500) + "\n" + if _, err := Replay(strings.NewReader(long), cfg, ReplayOptions{Start: t0, End: t0.Add(time.Hour), MaxLineBytes: 100}); err == nil { + t.Fatal("over-long line accepted") + } + if _, err := ParseReplayConfig([]byte(`{"limits":{},"unknown":1}`)); err == nil { + t.Fatal("unknown config field accepted") + } + if _, err := ParseReplayConfig([]byte(`{}`)); err == nil { + t.Fatal("empty replay config accepted") + } +} + +func TestOfflineReplay(t *testing.T) { + if *replayEvents == "" { + t.Skip("offline harness: pass -anomaly.replay.events/-config/-start/-end/-out to run") + } + cb, err := os.ReadFile(*replayConfig) + if err != nil { + t.Fatal(err) + } + cfg, err := ParseReplayConfig(cb) + if err != nil { + t.Fatal(err) + } + start, err1 := time.Parse(time.RFC3339, *replayStart) + end, err2 := time.Parse(time.RFC3339, *replayEnd) + if err1 != nil || err2 != nil { + t.Fatal("start and end are required RFC3339 times") + } + f, err := os.Open(*replayEvents) + if err != nil { + t.Fatal(err) + } + defer f.Close() + rep, err := Replay(f, cfg, ReplayOptions{Start: start, End: end, MaxLineBytes: 1 << 20}) + if err != nil { + t.Fatal(err) + } + b, _ := json.MarshalIndent(rep, "", " ") + if *replayOut == "" { + t.Fatal("-anomaly.replay.out is required") + } + if err := os.WriteFile(*replayOut, append(b, '\n'), 0o600); err != nil { + t.Fatal(err) + } + t.Logf("replay: %+v, %d candidates", rep.Input, len(rep.Candidates)) +} + +func replayFixtureConfig(t *testing.T) Config { + t.Helper() + cfg, err := ParseReplayConfig([]byte(`{"limits":{"max_keys_per_scope":100,"state_ttl":"2h","dedup_horizon":"2h", + "max_dedup_ids":1000,"reorder_delay":"0s","max_reorder_buffer":10,"max_window_entries":100, + "max_candidates_per_call":1,"max_pending_candidates":3},"expected":"include", + "rate":[{"name":"r","scope":"stream","window":"1m","threshold":1,"state":{"confirm_after":"0s","end_after_quiet":"1m","cooldown":"0s"}}]}`)) + if err != nil { + t.Fatal(err) + } + return cfg +} + +func replayLine(id string) string { + return replayLineAt(id, "2030-01-07T12:00:00Z", "5c") +} + +func replayLineAt(id, when, hop string) string { + return `{"id":"` + id + `","time":"` + when + `","payload_type":5,"channel":"ab","route_class":"flood","route_kind":"relay","first_hop":"1:` + hop + `","wire_bytes":60}` +} + +func TestReplayLineLimitIsExact(t *testing.T) { + cfg := replayFixtureConfig(t) + line := replayLine("a") + opt := ReplayOptions{Start: t0, End: t0.Add(time.Hour), MaxLineBytes: len(line)} + for _, in := range []string{line + "\n", line + "\r\n", line} { + rep, err := Replay(strings.NewReader(in), cfg, opt) + if err != nil || rep.Input.Accepted != 1 { + t.Fatalf("line of exactly MaxLineBytes (%q terminator): %v %+v", in[len(line):], err, rep.Input) + } + } + opt.MaxLineBytes = len(line) - 1 + if _, err := Replay(strings.NewReader(line+"\n"), cfg, opt); err == nil { + t.Fatal("line longer than MaxLineBytes accepted") + } +} + +func TestReplayRejectsTrailingData(t *testing.T) { + cfg := replayFixtureConfig(t) + in := replayLine("a") + ` {"x":1}` + "\n" + replayLine("b") + "garbage\n" + replayLine("c") + "]\n" + replayLine("d") + "}\n" + rep, err := Replay(strings.NewReader(in), cfg, ReplayOptions{Start: t0, End: t0.Add(time.Hour), MaxLineBytes: 4096}) + if err != nil || rep.Input.Invalid != 4 || rep.Input.Accepted != 0 { + t.Fatalf("trailing data: %v %+v", err, rep.Input) + } + for _, tail := range []string{` {"limits":{}}`, `]`, `}`} { + if _, err := ParseReplayConfig([]byte(`{"limits":{}}` + tail)); err == nil || !strings.Contains(err.Error(), "trailing") { + t.Fatalf("config with trailing %q: %v", tail, err) + } + } + if _, err := Replay(strings.NewReader(""), cfg, ReplayOptions{Start: t0, End: t0.Add(time.Hour), MaxLineBytes: math.MaxInt}); err == nil { + t.Fatal("MaxLineBytes = MaxInt accepted") + } +} + +func TestReplayRejectsAnEndOutsideTheValidRange(t *testing.T) { + cfg := replayFixtureConfig(t) + _, err := Replay(strings.NewReader(""), cfg, ReplayOptions{Start: t0, End: time.Date(2300, 1, 1, 0, 0, 0, 0, time.UTC), MaxLineBytes: 100}) + if err == nil { + t.Fatal("End beyond the valid range accepted (its deadlines would silently not run)") + } +} + +// The harness drains after every call: each line yields up to 3 +// candidates (a quiet end and two opening transitions) with a per-call cap +// of 1 and a pending buffer of 3, so without draining the backlog would +// grow by 2 per line and overflow. +func TestReplayDrainsSoTheCandidateBufferDoesNotOverflow(t *testing.T) { + cfg := replayFixtureConfig(t) + var in strings.Builder + for i, id := range []string{"a", "b", "c", "d", "e"} { + when := t0.Add(time.Duration(i) * 2 * time.Minute).Format(time.RFC3339) + in.WriteString(replayLineAt(id, when, "0"+string(rune('1'+i))) + "\n") + } + rep, err := Replay(strings.NewReader(in.String()), cfg, ReplayOptions{Start: t0, End: t0.Add(time.Hour), MaxLineBytes: 4096}) + if err != nil { + t.Fatal(err) + } + if rep.Stats.CandidatesDropped != 0 || rep.CoverageReduced { + t.Fatalf("harness dropped candidates: %+v", rep.Stats) + } + // 5 streams x (threshold_crossed, confirmed, quiet end) + if len(rep.Candidates) != 15 { + t.Fatalf("candidates %d", len(rep.Candidates)) + } +} diff --git a/internal/anomaly/testdata/replay_config.json b/internal/anomaly/testdata/replay_config.json new file mode 100644 index 000000000..fe2e3093c --- /dev/null +++ b/internal/anomaly/testdata/replay_config.json @@ -0,0 +1,75 @@ +{ + "limits": { + "max_keys_per_scope": 1000, + "state_ttl": "48h", + "dedup_horizon": "6h", + "max_dedup_ids": 100000, + "reorder_delay": "0s", + "max_reorder_buffer": 64, + "max_window_entries": 4096, + "max_candidates_per_call": 1000, + "max_pending_candidates": 10000 + }, + "expected": "include", + "rate": [ + { + "name": "stream_5m_example", + "scope": "stream", + "window": "5m", + "threshold": 10, + "state": { + "confirm_after": "0s", + "end_after_quiet": "10m", + "cooldown": "30m" + } + }, + { + "name": "global_5m_example", + "scope": "global", + "window": "5m", + "threshold": 20, + "state": { + "confirm_after": "0s", + "end_after_quiet": "10m", + "cooldown": "30m" + } + } + ], + "new_stream": [ + { + "name": "new_stream_example", + "scope": "stream", + "window": "1h", + "min_count": 10, + "censor_window": "30m", + "quiet_period": "30m", + "state": { + "confirm_after": "0s", + "end_after_quiet": "30m", + "cooldown": "0s" + } + } + ], + "periodic": [ + { + "name": "periodic_example", + "scope": "stream", + "min_period": "1m", + "max_period": "1h", + "burst_gap": "5s", + "jitter_abs": "5s", + "jitter_rel": 0.02, + "max_missing": 2, + "min_pulses": 8, + "history_len": 32, + "min_coverage": 0.6, + "max_jitter_fraction": 0.5, + "max_chance": 0.01, + "state": { + "confirm_after": "0s", + "end_after_quiet": "30m", + "cooldown": "0s" + } + } + ] +} diff --git a/internal/anomaly/testdata/replay_events.jsonl b/internal/anomaly/testdata/replay_events.jsonl new file mode 100644 index 000000000..0fe99c632 --- /dev/null +++ b/internal/anomaly/testdata/replay_events.jsonl @@ -0,0 +1,54 @@ +{"id":"per-00","time":"2030-01-07T12:00:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"per-01","time":"2030-01-07T12:05:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"per-02","time":"2030-01-07T12:10:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"per-03","time":"2030-01-07T12:15:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"per-04","time":"2030-01-07T12:20:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"per-05","time":"2030-01-07T12:25:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"burst-00","time":"2030-01-07T12:30:00Z","payload_type":5,"channel":"1a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"per-06","time":"2030-01-07T12:30:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"burst-01","time":"2030-01-07T12:30:10Z","payload_type":5,"channel":"1a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-02","time":"2030-01-07T12:30:20Z","payload_type":5,"channel":"1a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"per-03","time":"2030-01-07T12:15:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"burst-03","time":"2030-01-07T12:30:30Z","payload_type":5,"channel":"1a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-04","time":"2030-01-07T12:30:40Z","payload_type":5,"channel":"1a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-05","time":"2030-01-07T12:30:50Z","payload_type":5,"channel":"1a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-06","time":"2030-01-07T12:31:00Z","payload_type":5,"channel":"1a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-07","time":"2030-01-07T12:31:10Z","payload_type":5,"channel":"1a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-08","time":"2030-01-07T12:31:20Z","payload_type":5,"channel":"2a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-09","time":"2030-01-07T12:31:30Z","payload_type":5,"channel":"2a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-10","time":"2030-01-07T12:31:40Z","payload_type":5,"channel":"2a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-11","time":"2030-01-07T12:31:50Z","payload_type":5,"channel":"2a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-12","time":"2030-01-07T12:32:00Z","payload_type":5,"channel":"2a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-13","time":"2030-01-07T12:32:10Z","payload_type":5,"channel":"2a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-14","time":"2030-01-07T12:32:20Z","payload_type":5,"channel":"2a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-15","time":"2030-01-07T12:32:30Z","payload_type":5,"channel":"2a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-16","time":"2030-01-07T12:32:40Z","payload_type":5,"channel":"3a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-17","time":"2030-01-07T12:32:50Z","payload_type":5,"channel":"3a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-18","time":"2030-01-07T12:33:00Z","payload_type":5,"channel":"3a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-19","time":"2030-01-07T12:33:10Z","payload_type":5,"channel":"3a","route_class":"flood","route_kind":"relay","first_hop":"1:1b","full_route":"1:1b00","wire_bytes":60} +{"id":"burst-20","time":"2030-01-07T12:33:20Z","payload_type":5,"channel":"3a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-21","time":"2030-01-07T12:33:30Z","payload_type":5,"channel":"3a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-22","time":"2030-01-07T12:33:40Z","payload_type":5,"channel":"3a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"burst-23","time":"2030-01-07T12:33:50Z","payload_type":5,"channel":"3a","route_class":"flood","route_kind":"relay","first_hop":"1:2b","full_route":"1:2b00","wire_bytes":60} +{"id":"per-07","time":"2030-01-07T12:35:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"new-00","time":"2030-01-07T12:40:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"per-08","time":"2030-01-07T12:40:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"new-01","time":"2030-01-07T12:41:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"new-02","time":"2030-01-07T12:42:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"new-03","time":"2030-01-07T12:43:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"new-04","time":"2030-01-07T12:44:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"new-05","time":"2030-01-07T12:45:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"per-09","time":"2030-01-07T12:45:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"new-06","time":"2030-01-07T12:46:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"new-07","time":"2030-01-07T12:47:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"new-08","time":"2030-01-07T12:48:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"new-09","time":"2030-01-07T12:49:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"new-10","time":"2030-01-07T12:50:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"per-10","time":"2030-01-07T12:50:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"new-11","time":"2030-01-07T12:51:00Z","payload_type":5,"channel":"4a","route_class":"flood","route_kind":"relay","first_hop":"2:4b4c","full_route":"2:4b4c0000","wire_bytes":60} +{"id":"per-11","time":"2030-01-07T12:55:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"per-12","time":"2030-01-07T13:00:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"per-13","time":"2030-01-07T13:05:00Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"bad","time":"2030-01-07T12:00:00Z","payload_type":99} +{"id":"late","time":"2030-01-07T12:01:40Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} +{"id":"outside","time":"2030-01-19T01:46:39Z","payload_type":5,"channel":"0a","route_class":"flood","route_kind":"relay","first_hop":"1:0b","full_route":"1:0b00","wire_bytes":60} diff --git a/internal/anomaly/testdata/replay_golden.json b/internal/anomaly/testdata/replay_golden.json new file mode 100644 index 000000000..23df531c3 --- /dev/null +++ b/internal/anomaly/testdata/replay_golden.json @@ -0,0 +1,468 @@ +{ + "Experimental": true, + "Start": "2030-01-07T12:00:00Z", + "End": "2030-01-08T12:00:00Z", + "Input": { + "Lines": 54, + "Accepted": 50, + "Duplicates": 1, + "Late": 1, + "Invalid": 1, + "OutOfRange": 1 + }, + "Candidates": [ + { + "EventID": "dd66c4630637845a0d95aeaa5d1ea6a4", + "Rule": "global_5m_example", + "Kind": "rate", + "Key": "v1|s=global|pt=5|ch=-|rc=unknown|rk=unknown|hop=-", + "From": "normal", + "To": "suspicious", + "Reason": "threshold_crossed", + "At": "2030-01-07T12:33:00Z", + "DecidedAt": "2030-01-07T12:33:00Z", + "EpisodeStart": "2030-01-07T12:33:00Z", + "Confidence": "aggregate", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "burst-18", + "TriggerRoute": "1:1b00", + "Rate": { + "Window": 300000000000, + "Threshold": 20, + "Count": 20, + "Expected": 0, + "Monitored": 0, + "Saturated": false + } + }, + { + "EventID": "dd66c4630637845a0d95aeaa5d1ea6a4", + "Rule": "global_5m_example", + "Kind": "rate", + "Key": "v1|s=global|pt=5|ch=-|rc=unknown|rk=unknown|hop=-", + "From": "suspicious", + "To": "active", + "Reason": "confirmed", + "At": "2030-01-07T12:33:00Z", + "DecidedAt": "2030-01-07T12:33:00Z", + "EpisodeStart": "2030-01-07T12:33:00Z", + "Confidence": "aggregate", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "burst-18", + "TriggerRoute": "1:1b00", + "Rate": { + "Window": 300000000000, + "Threshold": 20, + "Count": 20, + "Expected": 0, + "Monitored": 0, + "Saturated": false + } + }, + { + "EventID": "9430ea1e68ea5eecd037c1bfd9229644", + "Rule": "periodic_example", + "Kind": "periodic", + "Key": "v1|s=stream|pt=5|ch=1:0a|rc=flood|rk=relay|hop=1:0b", + "From": "normal", + "To": "suspicious", + "Reason": "threshold_crossed", + "At": "2030-01-07T12:35:00Z", + "DecidedAt": "2030-01-07T12:35:00Z", + "EpisodeStart": "2030-01-07T12:35:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "per-07", + "TriggerRoute": "1:0b00", + "Periodic": { + "Period": 300000000000, + "Tolerance": 6000000000, + "JitterMedian": 0, + "Support": 8, + "Events": 8, + "Coverage": 1, + "Chance": 4.593872801250516e-8, + "SufficientAt": "2030-01-07T12:35:00Z", + "Alternatives": [ + { + "Period": 150000000000, + "Support": 8, + "Coverage": 0.5 + }, + { + "Period": 100000000000, + "Support": 8, + "Coverage": 0.3333333333333333 + } + ] + } + }, + { + "EventID": "9430ea1e68ea5eecd037c1bfd9229644", + "Rule": "periodic_example", + "Kind": "periodic", + "Key": "v1|s=stream|pt=5|ch=1:0a|rc=flood|rk=relay|hop=1:0b", + "From": "suspicious", + "To": "active", + "Reason": "confirmed", + "At": "2030-01-07T12:35:00Z", + "DecidedAt": "2030-01-07T12:35:00Z", + "EpisodeStart": "2030-01-07T12:35:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "per-07", + "TriggerRoute": "1:0b00", + "Periodic": { + "Period": 300000000000, + "Tolerance": 6000000000, + "JitterMedian": 0, + "Support": 8, + "Events": 8, + "Coverage": 1, + "Chance": 4.593872801250516e-8, + "SufficientAt": "2030-01-07T12:35:00Z", + "Alternatives": [ + { + "Period": 150000000000, + "Support": 8, + "Coverage": 0.5 + }, + { + "Period": 100000000000, + "Support": 8, + "Coverage": 0.3333333333333333 + } + ] + } + }, + { + "EventID": "dd66c4630637845a0d95aeaa5d1ea6a4", + "Rule": "global_5m_example", + "Kind": "rate", + "Key": "v1|s=global|pt=5|ch=-|rc=unknown|rk=unknown|hop=-", + "From": "active", + "To": "ended", + "Reason": "quiet", + "At": "2030-01-07T12:45:00Z", + "DecidedAt": "2030-01-07T12:45:00Z", + "EpisodeStart": "2030-01-07T12:33:00Z", + "Confidence": "aggregate", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "", + "TriggerRoute": "-", + "Summary": { + "ActiveSince": "2030-01-07T12:33:00Z", + "LastHot": "2030-01-07T12:35:00Z", + "HotSignals": 7 + } + }, + { + "EventID": "7771b1680b2f2d77043ed1601a6e2dbb", + "Rule": "new_stream_example", + "Kind": "new_stream", + "Key": "v1|s=stream|pt=5|ch=1:0a|rc=flood|rk=relay|hop=1:0b", + "From": "normal", + "To": "suspicious", + "Reason": "threshold_crossed", + "At": "2030-01-07T12:45:00Z", + "DecidedAt": "2030-01-07T12:45:00Z", + "EpisodeStart": "2030-01-07T12:45:00Z", + "Confidence": "low", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "per-09", + "TriggerRoute": "1:0b00", + "NewStream": { + "Window": 3600000000000, + "MinCount": 10, + "Count": 10, + "Age": 2700000000000, + "Censored": true, + "Reactivated": false + } + }, + { + "EventID": "7771b1680b2f2d77043ed1601a6e2dbb", + "Rule": "new_stream_example", + "Kind": "new_stream", + "Key": "v1|s=stream|pt=5|ch=1:0a|rc=flood|rk=relay|hop=1:0b", + "From": "suspicious", + "To": "active", + "Reason": "confirmed", + "At": "2030-01-07T12:45:00Z", + "DecidedAt": "2030-01-07T12:45:00Z", + "EpisodeStart": "2030-01-07T12:45:00Z", + "Confidence": "low", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "per-09", + "TriggerRoute": "1:0b00", + "NewStream": { + "Window": 3600000000000, + "MinCount": 10, + "Count": 10, + "Age": 2700000000000, + "Censored": true, + "Reactivated": false + } + }, + { + "EventID": "5e6b6ce788c0112e18763a0a354d5259", + "Rule": "periodic_example", + "Kind": "periodic", + "Key": "v1|s=stream|pt=5|ch=1:4a|rc=flood|rk=relay|hop=2:4b4c", + "From": "normal", + "To": "suspicious", + "Reason": "threshold_crossed", + "At": "2030-01-07T12:47:00Z", + "DecidedAt": "2030-01-07T12:47:00Z", + "EpisodeStart": "2030-01-07T12:47:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "new-07", + "TriggerRoute": "2:4b4c0000", + "Periodic": { + "Period": 60000000000, + "Tolerance": 5000000000, + "JitterMedian": 0, + "Support": 8, + "Events": 8, + "Coverage": 1, + "Chance": 0.00024196527078255232, + "SufficientAt": "2030-01-07T12:47:00Z", + "Alternatives": null + } + }, + { + "EventID": "5e6b6ce788c0112e18763a0a354d5259", + "Rule": "periodic_example", + "Kind": "periodic", + "Key": "v1|s=stream|pt=5|ch=1:4a|rc=flood|rk=relay|hop=2:4b4c", + "From": "suspicious", + "To": "active", + "Reason": "confirmed", + "At": "2030-01-07T12:47:00Z", + "DecidedAt": "2030-01-07T12:47:00Z", + "EpisodeStart": "2030-01-07T12:47:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "new-07", + "TriggerRoute": "2:4b4c0000", + "Periodic": { + "Period": 60000000000, + "Tolerance": 5000000000, + "JitterMedian": 0, + "Support": 8, + "Events": 8, + "Coverage": 1, + "Chance": 0.00024196527078255232, + "SufficientAt": "2030-01-07T12:47:00Z", + "Alternatives": null + } + }, + { + "EventID": "a709e43bd806a51d793c9a4078af9d05", + "Rule": "new_stream_example", + "Kind": "new_stream", + "Key": "v1|s=stream|pt=5|ch=1:4a|rc=flood|rk=relay|hop=2:4b4c", + "From": "normal", + "To": "suspicious", + "Reason": "threshold_crossed", + "At": "2030-01-07T12:49:00Z", + "DecidedAt": "2030-01-07T12:49:00Z", + "EpisodeStart": "2030-01-07T12:49:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "new-09", + "TriggerRoute": "2:4b4c0000", + "NewStream": { + "Window": 3600000000000, + "MinCount": 10, + "Count": 10, + "Age": 540000000000, + "Censored": false, + "Reactivated": false + } + }, + { + "EventID": "a709e43bd806a51d793c9a4078af9d05", + "Rule": "new_stream_example", + "Kind": "new_stream", + "Key": "v1|s=stream|pt=5|ch=1:4a|rc=flood|rk=relay|hop=2:4b4c", + "From": "suspicious", + "To": "active", + "Reason": "confirmed", + "At": "2030-01-07T12:49:00Z", + "DecidedAt": "2030-01-07T12:49:00Z", + "EpisodeStart": "2030-01-07T12:49:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "new-09", + "TriggerRoute": "2:4b4c0000", + "NewStream": { + "Window": 3600000000000, + "MinCount": 10, + "Count": 10, + "Age": 540000000000, + "Censored": false, + "Reactivated": false + } + }, + { + "EventID": "7771b1680b2f2d77043ed1601a6e2dbb", + "Rule": "new_stream_example", + "Kind": "new_stream", + "Key": "v1|s=stream|pt=5|ch=1:0a|rc=flood|rk=relay|hop=1:0b", + "From": "active", + "To": "ended", + "Reason": "quiet", + "At": "2030-01-07T13:15:00Z", + "DecidedAt": "2030-01-07T13:15:00Z", + "EpisodeStart": "2030-01-07T12:45:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "", + "TriggerRoute": "-", + "Summary": { + "ActiveSince": "2030-01-07T12:45:00Z", + "LastHot": "2030-01-07T12:45:00Z", + "HotSignals": 1 + } + }, + { + "EventID": "a709e43bd806a51d793c9a4078af9d05", + "Rule": "new_stream_example", + "Kind": "new_stream", + "Key": "v1|s=stream|pt=5|ch=1:4a|rc=flood|rk=relay|hop=2:4b4c", + "From": "active", + "To": "ended", + "Reason": "quiet", + "At": "2030-01-07T13:19:00Z", + "DecidedAt": "2030-01-07T13:19:00Z", + "EpisodeStart": "2030-01-07T12:49:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "", + "TriggerRoute": "-", + "Summary": { + "ActiveSince": "2030-01-07T12:49:00Z", + "LastHot": "2030-01-07T12:49:00Z", + "HotSignals": 1 + } + }, + { + "EventID": "5e6b6ce788c0112e18763a0a354d5259", + "Rule": "periodic_example", + "Kind": "periodic", + "Key": "v1|s=stream|pt=5|ch=1:4a|rc=flood|rk=relay|hop=2:4b4c", + "From": "active", + "To": "ended", + "Reason": "quiet", + "At": "2030-01-07T13:21:00Z", + "DecidedAt": "2030-01-07T13:21:00Z", + "EpisodeStart": "2030-01-07T12:47:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "", + "TriggerRoute": "-", + "Summary": { + "ActiveSince": "2030-01-07T12:47:00Z", + "LastHot": "2030-01-07T12:51:00Z", + "HotSignals": 5 + } + }, + { + "EventID": "9430ea1e68ea5eecd037c1bfd9229644", + "Rule": "periodic_example", + "Kind": "periodic", + "Key": "v1|s=stream|pt=5|ch=1:0a|rc=flood|rk=relay|hop=1:0b", + "From": "active", + "To": "ended", + "Reason": "quiet", + "At": "2030-01-07T13:35:00Z", + "DecidedAt": "2030-01-07T13:35:00Z", + "EpisodeStart": "2030-01-07T12:35:00Z", + "Confidence": "route_observed", + "Traffic": "unclassified", + "Suppressed": false, + "Coverage": [], + "TriggerID": "", + "TriggerRoute": "-", + "Summary": { + "ActiveSince": "2030-01-07T12:35:00Z", + "LastHot": "2030-01-07T13:05:00Z", + "HotSignals": 7 + } + } + ], + "Stats": { + "Accepted": 50, + "Duplicates": 1, + "DuplicateConflicts": 0, + "Late": 1, + "Invalid": 0, + "Processed": 50, + "TruncatedInputs": 0, + "InvalidAdvance": 0, + "Buffered": 0, + "DedupIDs": 50, + "DedupEvictedEarly": 0, + "EffectiveDedupHorizon": 86399999999999, + "Keys": [ + 0, + 8, + 0, + 0, + 1 + ], + "CapacityEvictions": [ + 0, + 0, + 0, + 0, + 0 + ], + "TTLEvictions": [ + 0, + 0, + 0, + 0, + 0 + ], + "EpisodesEvicted": 0, + "WindowMerges": 0, + "ReorderForced": 0, + "CandidatesEmitted": 15, + "CandidatesDropped": 0, + "PendingCandidates": 0, + "MaxCandidateBurst": 4, + "Frontier": "2030-01-07T13:35:00Z" + }, + "CoverageReduced": false +} From 4500e439715509c18bea09c0831849df67fc293a Mon Sep 17 00:00:00 2001 From: Openclaw Date: Fri, 25 Sep 2026 08:53:02 +0200 Subject: [PATCH 3/9] test(anomaly): add fuzz targets for keys, dedup, normalization and bounds Six targets: - key encoding round trip and canonical parsing; - detector memory bounds and no panics on arbitrary event streams; - dedup idempotence; - normalization independent of observation order within the settle window; - legal state-machine transitions only. Seeds live in code; no corpus files are kept. Co-Authored-By: Claude Opus 5.5 --- internal/anomaly/fuzz_test.go | 340 ++++++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 internal/anomaly/fuzz_test.go diff --git a/internal/anomaly/fuzz_test.go b/internal/anomaly/fuzz_test.go new file mode 100644 index 000000000..780f482a8 --- /dev/null +++ b/internal/anomaly/fuzz_test.go @@ -0,0 +1,340 @@ +package anomaly + +import ( + "fmt" + "reflect" + "testing" + "time" +) + +// Fuzz targets. Seeds are added in code only; no corpus files are kept. + +func keyFromBytes(b []byte) Key { + get := func(i int) byte { + if i < len(b) { + return b[i] + } + return 0 + } + k := Key{Scope: Scope(get(0)%4 + 1), PayloadType: PayloadType(get(1) & 15)} + if n := int(get(2) % 5); n > 0 { + cb := make([]byte, n) + for i := range cb { + cb[i] = get(3 + i) + } + k.Channel, _ = NewChannelHash(cb) + } + k.RouteClass = RouteClass(get(8) % 6) + k.RouteKind = RouteKind(get(9) % 4) + if w := int(get(10) % 4); w > 0 { + k.FirstHop, _ = NewHop([]byte{get(11), get(12), get(13)}[:w]) + } + // make the key canonical for its scope + switch k.Scope { + case ScopeStream: + if k.RouteKind == RouteKindRelay && k.FirstHop.IsZero() { + k.FirstHop, _ = NewHop([]byte{get(11)}) + } + if k.RouteKind != RouteKindRelay { + k.FirstHop = Hop{} + } + if !kindFitsClass(k.RouteKind, k.RouteClass) { + k.RouteClass = RouteClassMixed // allowed for every kind + } + case ScopeChannel: + k.RouteClass, k.RouteKind, k.FirstHop = 0, 0, Hop{} + case ScopeRouteGroup: + k.Channel, k.RouteClass, k.RouteKind = ChannelHash{}, 0, RouteKindRelay + if k.FirstHop.IsZero() { + k.FirstHop, _ = NewHop([]byte{get(11)}) + } + case ScopeGlobal: + k.Channel, k.RouteClass, k.RouteKind, k.FirstHop = ChannelHash{}, 0, 0, Hop{} + } + return k +} + +func FuzzKeyEncoding(f *testing.F) { + f.Add([]byte{0, 5, 1, 0xab, 0, 0, 0, 0, 1, 1, 1, 0x5c}, []byte{0, 5, 1, 0xab, 0, 0, 0, 0, 1, 1, 2, 0x5c, 0x12}) + f.Add([]byte{3, 5}, []byte{2, 5, 1, 0}) + f.Fuzz(func(t *testing.T, a, b []byte) { + k1, k2 := keyFromBytes(a), keyFromBytes(b) + for _, k := range []Key{k1, k2} { + if err := k.Validate(); err != nil { + t.Fatalf("generated key invalid: %v", err) + } + back, err := ParseKey(k.String()) + if err != nil || back != k { + t.Fatalf("round trip failed for %s: %v", k, err) + } + } + if (k1 == k2) != (k1.String() == k2.String()) { + t.Fatalf("encoding not injective: %s vs %s", k1, k2) + } + }) +} + +func FuzzParseKeyIsCanonical(f *testing.F) { + f.Add("v1|s=stream|pt=5|ch=1:ab|rc=flood|rk=relay|hop=1:5c") + f.Add("v1|s=global|pt=0|ch=-|rc=unknown|rk=unknown|hop=-") + f.Add("v1|s=route|pt=5|ch=-|rc=unknown|rk=relay|hop=3:000000") + f.Fuzz(func(t *testing.T, s string) { + k, err := ParseKey(s) + if err == nil && k.String() != s { + t.Fatalf("accepted non-canonical %q (canonical %q)", s, k.String()) + } + }) +} + +// fuzzEvent decodes 8 bytes into an event near a moving clock. +func fuzzEvent(b []byte, i int, clock *time.Time) Event { + g := func(j int) byte { + if j < len(b) { + return b[j] + } + return 0 + } + // time step in [-10 s, +245 s]: negative steps create late / reordered events + step := time.Duration(int(g(0))-10) * time.Second + *clock = clock.Add(step) + e := Event{ID: TxID(fmt.Sprintf("f%d-%d", g(7)%16, i)), Time: *clock, PayloadType: PayloadType(g(1) % 16), Traffic: TrafficClass(g(6) % 3)} + if g(1)%3 == 0 { + e.ID = TxID(fmt.Sprintf("dup%d", g(7)%8)) // frequent duplicate IDs + } + if e.PayloadType == 5 || e.PayloadType == 6 { + e.Channel, _ = NewChannelHash([]byte{g(2) % 8}) + } + switch g(3) % 4 { + case 0: + e.RouteClass, e.RouteKind = RouteClassFlood, RouteKindNoPath + case 1: + e.RouteClass, e.RouteKind = RouteClassDirect, RouteKindDirect + default: + e.RouteClass, e.RouteKind = RouteClassFlood, RouteKindRelay + if e.PayloadType == PayloadTypeTrace { + e.RouteKind = RouteKindUnknown + } else { + w := int(g(4)%3) + 1 + e.FirstHop, _ = NewHop([]byte{g(5) % 4, g(5), g(4)}[:w]) + } + } + e.WireBytes = int(g(6)) + return e +} + +func fuzzConfig() Config { + lim := Limits{MaxKeysPerScope: 8, StateTTL: 7 * time.Hour, DedupHorizon: 2 * time.Hour, MaxDedupIDs: 16, + ReorderDelay: 20 * time.Second, MaxReorderBuffer: 4, MaxWindowEntries: 6, MaxCandidatesPerCall: 3, MaxPendingCandidates: 12} + st := StateParams{ConfirmAfter: time.Minute, EndAfterQuiet: 10 * time.Minute, Cooldown: 20 * time.Minute} + return Config{Limits: lim, Expected: ExpectedPolicy{Mode: ExpectedSuppress}, + Rate: []RateRule{ + {Name: "s1", Scope: ScopeStream, Window: time.Minute, Threshold: 2, State: st}, + {Name: "c5", Scope: ScopeChannel, Window: 5 * time.Minute, Threshold: 3, State: st}, + {Name: "r15", Scope: ScopeRouteGroup, Window: 15 * time.Minute, Threshold: 3, State: st}, + {Name: "g60", Scope: ScopeGlobal, Window: time.Hour, Threshold: 5, State: st}, + }, + NewStream: []NewStreamRule{{Name: "ns", Scope: ScopeStream, Window: time.Hour, MinCount: 2, CensorWindow: time.Hour, QuietPeriod: time.Hour, + State: StateParams{EndAfterQuiet: 10 * time.Minute, Cooldown: 20 * time.Minute}}}, + Periodic: []PeriodicRule{{Name: "p", Scope: ScopeStream, MinPeriod: time.Minute, MaxPeriod: 2 * time.Hour, BurstGap: 2 * time.Second, + JitterAbs: 5 * time.Second, JitterRel: 0.05, MaxMissing: 2, MinPulses: 4, HistoryLen: 8, MinCoverage: 0.5, + MaxJitterFraction: 1, MaxChance: 1, State: st}}, + } +} + +func checkInvariants(t *testing.T, d *Detector, r Result) { + t.Helper() + lim := d.cfg.Limits + for _, c := range r.Candidates { + if !legalTransition(c.From, c.To, c.Reason) { + t.Fatalf("illegal transition %v->%v %v", c.From, c.To, c.Reason) + } + if c.DecidedAt.Before(c.At) { + t.Fatal("DecidedAt before At") + } + if c.Key.Validate() != nil { + t.Fatal("candidate key invalid") + } + if c.Key.Scope == ScopeGlobal && (!c.Key.FirstHop.IsZero() || !c.Key.Channel.IsZero()) { + t.Fatal("global candidate attributed") + } + } + if len(r.Candidates) > lim.MaxCandidatesPerCall { + t.Fatal("per-call cap exceeded") + } + s := d.Snapshot() + for sc, n := range s.Keys { + if n > lim.MaxKeysPerScope { + t.Fatalf("scope %d holds %d keys", sc, n) + } + } + if s.DedupIDs > lim.MaxDedupIDs || s.Buffered > lim.MaxReorderBuffer || s.PendingCandidates > lim.MaxPendingCandidates { + t.Fatalf("bounded structure exceeded: %+v", s) + } + for _, tb := range d.scopes { + if tb == nil { + continue + } + for _, k := range tb.m { + if k.win == nil { + continue + } + if k.win.live() > lim.MaxWindowEntries { + t.Fatal("window entries exceed cap") + } + for _, sm := range k.win.sums { + if sm.n > s.Accepted || sm.exp > sm.n || sm.mon > sm.n { + t.Fatalf("window sum out of range: %+v accepted=%d", sm, s.Accepted) + } + } + } + } + if d.deadlines.Len() > 4*d.liveEpisodes()+1024 { + t.Fatal("deadline heap unbounded") + } +} + +func FuzzDetectorNoPanicAndBounded(f *testing.F) { + f.Add([]byte{10, 5, 1, 2, 0, 7, 0, 1, 70, 5, 1, 2, 0, 7, 0, 2, 0, 5, 1, 2, 0, 7, 0, 3}) + f.Add([]byte{255, 4, 0, 0, 1, 2, 1, 9, 0, 9, 3, 3, 2, 2, 2, 2}) + f.Fuzz(func(t *testing.T, data []byte) { + d, err := New(fuzzConfig()) + if err != nil { + t.Fatal(err) + } + clock := t0 + for i := 0; i+8 <= len(data) && i < 8*400; i += 8 { + chunk := data[i : i+8] + var r Result + if chunk[7]%11 == 0 { + r = d.Advance(clock.Add(time.Duration(chunk[0]) * time.Minute)) + } else { + r = d.Observe(fuzzEvent(chunk, i, &clock)) + } + checkInvariants(t, d, r) + } + checkInvariants(t, d, d.Flush()) + for r := d.Drain(); ; r = d.Drain() { + checkInvariants(t, d, r) + if !r.More { + break + } + } + }) +} + +func collectAll(t *testing.T, cfg Config, evs []Event) []Candidate { + d, err := New(cfg) + if err != nil { + t.Fatal(err) + } + var out []Candidate + for _, e := range evs { + out = append(out, d.Observe(e).Candidates...) + } + out = append(out, d.Flush().Candidates...) + for r := d.Drain(); ; r = d.Drain() { + out = append(out, r.Candidates...) + if !r.More { + break + } + } + return out +} + +func FuzzDedupIdempotent(f *testing.F) { + f.Add([]byte{10, 5, 1, 2, 0, 7, 0, 1, 20, 5, 1, 2, 0, 7, 0, 2, 30, 5, 1, 2, 0, 7, 0, 3}) + f.Fuzz(func(t *testing.T, data []byte) { + cfg := fuzzConfig() + cfg.Limits.MaxDedupIDs = 100_000 + cfg.Limits.MaxReorderBuffer = 100_000 + cfg.Limits.MaxCandidatesPerCall = 100_000 + cfg.Limits.MaxPendingCandidates = 100_000 + clock := t0 + var once, twice []Event + for i := 0; i+8 <= len(data) && i < 8*300; i += 8 { + e := fuzzEvent(data[i:i+8], i, &clock) + if e.Validate() != nil { + continue + } + once = append(once, e) + twice = append(twice, e, e) + } + a, b := collectAll(t, cfg, once), collectAll(t, cfg, twice) + if !reflect.DeepEqual(a, b) { + t.Fatalf("duplicate events changed the output: %d vs %d candidates", len(a), len(b)) + } + }) +} + +func FuzzNormalizePermutation(f *testing.F) { + f.Add([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, uint8(3)) + f.Fuzz(func(t *testing.T, data []byte, rot uint8) { + var os []Observation + for i := 0; i+4 <= len(data) && len(os) < 32; i += 4 { + o := Observation{TxID: "tx", ObsID: fmt.Sprintf("o%02d", i/4), Time: t0.Add(time.Duration(data[i]) * time.Second), + ArrivedAt: t0.Add(time.Duration(data[i+1]) * time.Second), PayloadType: 5, Channel: mustChan(t, 0x42), WireBytes: 50} + switch data[i+2] % 3 { + case 0: + o.RouteClass = RouteClassFlood + w := int(data[i+3]%3) + 1 + hops := make([]byte, w*int(data[i+3]%4+1)) + for j := range hops { + hops[j] = data[i+3] + byte(j) + } + o.Path, _ = NewRoute(w, hops) + case 1: + o.RouteClass = RouteClassFlood + default: + o.RouteClass = RouteClassDirect + } + os = append(os, o) + } + if len(os) == 0 { + return + } + a, ra, errA := NormalizeTransmission(os, 60*time.Second) + p := make([]Observation, len(os)) + for i := range os { + p[(i+int(rot))%len(os)] = os[i] + } + for i, j := 0, len(p)-1; i < j && rot%2 == 1; i, j = i+1, j-1 { + p[i], p[j] = p[j], p[i] + } + b, rb, errB := NormalizeTransmission(p, 60*time.Second) + if (errA == nil) != (errB == nil) || !reflect.DeepEqual(a, b) || ra != rb { + t.Fatalf("observation order changed the event: %+v vs %+v", a, b) + } + if errA == nil { + if err := a.Validate(); err != nil { + t.Fatalf("normalized event invalid: %v", err) + } + } + }) +} + +func FuzzStateMachineLegal(f *testing.F) { + f.Add([]byte{0, 1, 1, 5, 2, 200, 0, 9}) + f.Fuzz(func(t *testing.T, ops []byte) { + p := StateParams{ConfirmAfter: time.Duration(len(ops)%3) * time.Minute, EndAfterQuiet: 5 * time.Minute, Cooldown: time.Duration(len(ops)%4) * 10 * time.Minute} + var ep episode + now := t0.UnixNano() + for i := 0; i+1 < len(ops) && i < 2000; i += 2 { + now += int64(ops[i+1]) * int64(15*time.Second) + var trs []transition + switch ops[i] % 3 { + case 0: + trs = ep.onTime(now, &p, nil) + trs = ep.onHot(now, &p, trs) + case 1: + trs = ep.onTime(now, &p, nil) + case 2: + trs = ep.evict(now, nil) + } + for _, tr := range trs { + if !legalTransition(tr.from, tr.to, tr.reason) || tr.at > now { + t.Fatalf("illegal or future transition %+v", tr) + } + } + } + }) +} From 012a7717fea9ba638e2c8049b6a579df754eade1 Mon Sep 17 00:00:00 2001 From: Openclaw Date: Fri, 25 Sep 2026 08:53:02 +0200 Subject: [PATCH 4/9] test(anomaly): add benchmarks for streams, allocations, eviction and bursts Covers: - Observe at 1k/10k/100k streams, with and without periodicity; - bytes per stream; - 1M events over 10k Zipf streams at all four scopes (detector peak heap); - allocations per event in steady state; - TTL expiration and capacity eviction; - duplicate fan-out through the Normalizer; - the worst single-call candidate burst. Co-Authored-By: Claude Opus 5.5 --- internal/anomaly/bench_test.go | 298 +++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 internal/anomaly/bench_test.go diff --git a/internal/anomaly/bench_test.go b/internal/anomaly/bench_test.go new file mode 100644 index 000000000..abb834ee3 --- /dev/null +++ b/internal/anomaly/bench_test.go @@ -0,0 +1,298 @@ +package anomaly + +import ( + "fmt" + "math/rand" + "runtime" + "strconv" + "testing" + "time" +) + +// Benchmarks use synthetic events only. Thresholds are high so that the +// measurement is dominated by state updates, not by candidate emission +// (except in the burst benchmark). + +func benchLimits(maxKeys int) Limits { + return Limits{MaxKeysPerScope: maxKeys, StateTTL: 48 * time.Hour, DedupHorizon: 2 * time.Hour, MaxDedupIDs: 4_000_000, + ReorderDelay: 0, MaxReorderBuffer: 1024, MaxWindowEntries: 4096, MaxCandidatesPerCall: 1 << 20, MaxPendingCandidates: 1 << 22} +} + +func benchConfig(maxKeys int, periodic, allScopes bool) Config { + st := StateParams{EndAfterQuiet: 10 * time.Minute, Cooldown: 30 * time.Minute} + cfg := Config{Limits: benchLimits(maxKeys), Expected: ExpectedPolicy{Mode: ExpectedInclude}} + for _, w := range []time.Duration{time.Minute, 5 * time.Minute, 15 * time.Minute, time.Hour} { + cfg.Rate = append(cfg.Rate, RateRule{Name: "s" + strconv.Itoa(int(w/time.Minute)), Scope: ScopeStream, Window: w, Threshold: 1 << 20, State: st}) + if allScopes { + for _, s := range []Scope{ScopeChannel, ScopeRouteGroup, ScopeGlobal} { + cfg.Rate = append(cfg.Rate, RateRule{Name: fmt.Sprintf("%s%d", s, int(w/time.Minute)), Scope: s, Window: w, Threshold: 1 << 20, State: st}) + } + } + } + if periodic { + cfg.Periodic = []PeriodicRule{{Name: "p", Scope: ScopeStream, MinPeriod: time.Minute, MaxPeriod: time.Hour, BurstGap: 5 * time.Second, + JitterAbs: 5 * time.Second, JitterRel: 0.02, MaxMissing: 2, MinPulses: 8, HistoryLen: 32, MinCoverage: 0.6, + MaxJitterFraction: 0.5, MaxChance: 0.01, State: st}} + } + return cfg +} + +func streamEvent(i, stream int, when time.Time) Event { + hop := [3]byte{byte(stream), byte(stream >> 8), byte(stream >> 16)} + w := 1 + stream%3 + h, _ := NewHop(hop[:w]) + c, _ := NewChannelHash([]byte{byte(stream % 251)}) + return Event{ID: TxID("e" + strconv.Itoa(i)), Time: when, PayloadType: 5, Channel: c, + RouteClass: RouteClassFlood, RouteKind: RouteKindRelay, FirstHop: h, WireBytes: 60} +} + +func BenchmarkObserve(b *testing.B) { + for _, n := range []int{1_000, 10_000, 100_000} { + for _, per := range []bool{false, true} { + b.Run(fmt.Sprintf("streams=%d/periodic=%v", n, per), func(b *testing.B) { + d, err := New(benchConfig(2*n, per, false)) + if err != nil { + b.Fatal(err) + } + // warm up: every stream exists with a few events + clock := t0 + id := 0 + for r := 0; r < 3; r++ { + for s := 0; s < n; s++ { + clock = clock.Add(time.Millisecond) + d.Observe(streamEvent(id, s, clock)) + id++ + } + } + evs := make([]Event, b.N) + for i := range evs { + clock = clock.Add(10 * time.Millisecond) + evs[i] = streamEvent(id+i, i%n, clock) + } + b.ReportAllocs() + b.ResetTimer() + for i := range evs { + d.Observe(evs[i]) + } + }) + } + } +} + +func heapInUse() uint64 { + runtime.GC() + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m.HeapAlloc +} + +// BenchmarkBytesPerStream reports steady-state heap per stream with full +// 1/5/15/60-minute windows (4 entries each) and optionally periodicity. +func BenchmarkBytesPerStream(b *testing.B) { + for _, n := range []int{1_000, 10_000, 100_000} { + for _, per := range []bool{false, true} { + b.Run(fmt.Sprintf("streams=%d/periodic=%v", n, per), func(b *testing.B) { + for it := 0; it < b.N; it++ { + b.StopTimer() + before := heapInUse() + d, _ := New(benchConfig(2*n, per, false)) + clock := t0 + id := 0 + for r := 0; r < 4; r++ { + for s := 0; s < n; s++ { + clock = clock.Add(time.Millisecond) + d.Observe(streamEvent(id, s, clock)) + id++ + } + } + // the dedup set is bounded separately (MaxDedupIDs); drop it so + // only per-stream state is measured + d.dedup, d.dedupLog, d.dedupHead = map[TxID]int64{}, nil, 0 + after := heapInUse() + b.ReportMetric(float64(after-before)/float64(n), "B/stream") + runtime.KeepAlive(d) + b.StartTimer() + } + }) + } + } +} + +// zipf picks streams with a skewed distribution. +func zipfStreams(n, count int, seed int64) []int { + r := rand.New(rand.NewSource(seed)) + z := rand.NewZipf(r, 1.1, 1, uint64(n-1)) + out := make([]int, count) + for i := range out { + out[i] = int(z.Uint64()) + } + return out +} + +// BenchmarkMillionEvents runs 1M events over 10k streams, multi-scale windows +// at all four scopes, with and without periodicity. It reports ns/event, +// allocs/event, the peak heap during the run and the largest burst. +func BenchmarkMillionEvents(b *testing.B) { + const events, streams = 1_000_000, 10_000 + pick := zipfStreams(streams, events, 1) + for _, per := range []bool{false, true} { + b.Run(fmt.Sprintf("periodic=%v", per), func(b *testing.B) { + var peak uint64 + for it := 0; it < b.N; it++ { + b.StopTimer() + d, _ := New(benchConfig(4*streams, per, true)) + evs := make([]Event, events) + clock := t0 + for i := range evs { + clock = clock.Add(50 * time.Millisecond) + evs[i] = streamEvent(i, pick[i], clock) + } + runtime.GC() + var base runtime.MemStats + runtime.ReadMemStats(&base) + b.StartTimer() + start := time.Now() + for i := range evs { + d.Observe(evs[i]) + if i%100_000 == 0 { + b.StopTimer() + var m runtime.MemStats + runtime.ReadMemStats(&m) + // detector growth only: the input slice is in the baseline + if m.HeapInuse-base.HeapInuse > peak { + peak = m.HeapInuse - base.HeapInuse + } + b.StartTimer() + } + } + el := time.Since(start) + b.ReportMetric(float64(el.Nanoseconds())/float64(events), "ns/event") + s := d.Snapshot() + b.ReportMetric(float64(s.MaxCandidateBurst), "max-burst") + } + b.ReportMetric(float64(peak)/(1<<20), "detector-peak-heap-MiB") + }) + } +} + +// BenchmarkAllocsPerEvent isolates allocations in steady state. +func BenchmarkAllocsPerEvent(b *testing.B) { + d, _ := New(benchConfig(40_000, true, true)) + pick := zipfStreams(10_000, 200_000, 2) + clock := t0 + for i, s := range pick { + clock = clock.Add(50 * time.Millisecond) + d.Observe(streamEvent(i, s, clock)) + } + evs := make([]Event, b.N) + for i := range evs { + clock = clock.Add(50 * time.Millisecond) + evs[i] = streamEvent(200_000+i, pick[i%len(pick)], clock) + } + b.ReportAllocs() + b.ResetTimer() + for i := range evs { + d.Observe(evs[i]) + } +} + +// BenchmarkExpiration: every event creates a new stream; old ones expire by +// TTL, so eviction runs continuously. +func BenchmarkExpiration(b *testing.B) { + cfg := benchConfig(1_000_000, false, false) + cfg.Limits.StateTTL = time.Hour + cfg.Rate = cfg.Rate[:1] + cfg.Rate[0].State.Cooldown = 0 + d, err := New(cfg) + if err != nil { + b.Fatal(err) + } + evs := make([]Event, b.N) + clock := t0 + for i := range evs { + clock = clock.Add(time.Second) + evs[i] = streamEvent(i, i%16_000_000, clock) + } + b.ReportAllocs() + b.ResetTimer() + for i := range evs { + d.Observe(evs[i]) + } + b.ReportMetric(float64(d.Snapshot().TTLEvictions[ScopeStream])/float64(b.N), "ttl-evictions/event") +} + +// BenchmarkCapacityEviction: a flood of unique keys against a 10k cap. +func BenchmarkCapacityEviction(b *testing.B) { + d, _ := New(benchConfig(10_000, false, false)) + evs := make([]Event, b.N) + clock := t0 + for i := range evs { + clock = clock.Add(time.Millisecond) + evs[i] = streamEvent(i, 10_000+i%16_000_000, clock) + } + b.ReportAllocs() + b.ResetTimer() + for i := range evs { + d.Observe(evs[i]) + } + b.StopTimer() + if d.Snapshot().Keys[ScopeStream] > 10_000 { + b.Fatal("cap exceeded") + } +} + +// BenchmarkDuplicateFanout: 20 observations per transmission through the +// Normalizer, each final event offered twice to the Detector. +func BenchmarkDuplicateFanout(b *testing.B) { + n, _ := NewNormalizer(NormalizerConfig{SettleWindow: 30 * time.Second, MaxPendingTransmissions: 100_000, + MaxObservationsPerTransmission: 64, FinalizedHorizon: time.Hour, MaxFinalizedIDs: 1_000_000}) + d, _ := New(benchConfig(100_000, true, true)) + route, _ := NewRoute(1, []byte{1, 2, 3, 4}) + ch, _ := NewChannelHash([]byte{0x42}) + obs := make([]Observation, 0, b.N) + clock := t0 + for i := 0; len(obs) < b.N; i++ { + clock = clock.Add(2 * time.Second) + for j := 0; j < 20 && len(obs) < b.N; j++ { + obs = append(obs, Observation{TxID: TxID("t" + strconv.Itoa(i)), ObsID: strconv.Itoa(j), Time: clock.Add(time.Duration(j) * 100 * time.Millisecond), + PayloadType: 5, Channel: ch, RouteClass: RouteClassFlood, Path: route, WireBytes: 60}) + } + } + b.ReportAllocs() + b.ResetTimer() + for i := range obs { + for _, e := range n.Add(obs[i]).Events { + d.Observe(e) + d.Observe(e) + } + } + b.StopTimer() + b.ReportMetric(float64(d.Snapshot().Duplicates), "duplicates") +} + +// BenchmarkWorstCandidateBurst: 10k streams become quiet at the same instant; +// one Advance emits 10k 'ended' transitions. +func BenchmarkWorstCandidateBurst(b *testing.B) { + const n = 10_000 + for it := 0; it < b.N; it++ { + b.StopTimer() + cfg := benchConfig(2*n, false, false) + cfg.Rate = []RateRule{{Name: "hot", Scope: ScopeStream, Window: time.Minute, Threshold: 1, + State: StateParams{EndAfterQuiet: 10 * time.Minute}}} + d, _ := New(cfg) + for s := 0; s < n; s++ { + d.Observe(streamEvent(s, s, t0)) + } + for r := d.Drain(); r.More; r = d.Drain() { + } + b.StartTimer() + start := time.Now() + r := d.Advance(t0.Add(time.Hour)) + el := time.Since(start) + b.StopTimer() + b.ReportMetric(float64(len(r.Candidates)), "burst-candidates") + b.ReportMetric(float64(el.Microseconds()), "burst-us") + b.StartTimer() + } +} From c3c6cc9c4465861342097e408ef0b1b29ecfd90e Mon Sep 17 00:00:00 2001 From: Openclaw Date: Fri, 25 Sep 2026 09:37:54 +0200 Subject: [PATCH 5/9] ci: vet and test internal/anomaly (with -race) in Go Build & Test internal/anomaly is a standalone Go module that no binary imports, so the existing server, ingestor and channel steps never reached its tests. This adds one step to the go-test job that runs go vet, go test and go test -race -v in that module. The step fails if no test passed. It does not fuzz (fuzz targets run only their in-code seeds) and does not run benchmarks. Triggers, permissions, jobs, needs, concurrency, fork guards and deploy/publish conditions are unchanged. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 90ee4dc34..88ec68110 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -85,6 +85,20 @@ jobs: go test ./... echo "--- Decrypt CLI tests passed ---" + # internal/anomaly is its own module (not wired into any binary), so the + # server/ingestor steps above do not reach it. No -fuzz and no -bench: + # fuzz targets run their in-code seeds only, benchmarks do not run. + - name: Vet and test anomaly library (internal/anomaly, with -race) + run: | + set -e -o pipefail + cd internal/anomaly + go vet ./... + go test ./... + go test -race -v ./... 2>&1 | tee anomaly-race-test.log + passed=$(grep -c -- '^--- PASS: ' anomaly-race-test.log || true) + echo "--- Anomaly library: ${passed} top-level tests passed under -race ---" + test "${passed}" -gt 0 + - name: Verify Dockerfile COPY invariants (issue #1316) run: bash scripts/check-dockerfile-internal-pkgs.sh From ad3ab30fa714b417edef37e1b6cdfba9bf15c487 Mon Sep 17 00:00:00 2001 From: Openclaw Date: Fri, 25 Sep 2026 10:38:52 +0200 Subject: [PATCH 6/9] fix(anomaly): make periodic Chance stable across CPU architectures The golden replay test failed on linux/amd64 while it passed on arm64. Both machines produced the same report except for PeriodicEvidence.Chance, which differed in its last digits. Chance used exp(-l*lo) - exp(-l*hi). For small tolerances that subtraction cancels, and Pow(pg, gaps-1) then amplifies the lost digits. Over valid parameters, amd64 and arm64 differed by up to 5e-5 relative. - Compute the same probability without cancellation: exp(-l*lo) * -expm1(-l*(hi-lo)). Measured over 400k cases on both architectures, the spread is now at most 5.2e-13 relative. - The detector still compares the full value with MaxChance. - JSON writes Chance rounded to 6 significant digits (ChanceDigits), a margin of about 1e6 over that spread, so replay output is byte-identical on amd64 and arm64. - The golden file changes only in its four Chance values. A test pins the rest of the file by hash. - The golden test now names the first differing line on failure. Co-Authored-By: Claude Opus 5.5 --- internal/anomaly/candidate.go | 36 +++ internal/anomaly/chance_test.go | 308 +++++++++++++++++++ internal/anomaly/periodic.go | 6 +- internal/anomaly/replay_test.go | 17 +- internal/anomaly/testdata/replay_golden.json | 8 +- 5 files changed, 368 insertions(+), 7 deletions(-) create mode 100644 internal/anomaly/chance_test.go diff --git a/internal/anomaly/candidate.go b/internal/anomaly/candidate.go index 9b3f60583..f96b211bb 100644 --- a/internal/anomaly/candidate.go +++ b/internal/anomaly/candidate.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" + "math" "strconv" "time" ) @@ -158,6 +159,8 @@ type PeriodicEvidence struct { // equally long chains under a Poisson null with the key's local pulse // rate, over all pulses evaluated on the key so far and every candidate // period the search may try; one gap is discounted for fitting Period. + // The detector compares the full value with MaxChance; JSON carries it + // rounded to ChanceDigits significant digits (see MarshalJSON). Chance float64 SufficientAt time.Time // first time the evidence met every criterion Alternatives []PeriodAlternative @@ -237,6 +240,39 @@ func (r Route) MarshalJSON() ([]byte, error) { return json.Marshal(strconv.Itoa(int(r.w)) + ":" + hex.EncodeToString(r.hops)) } +// ChanceDigits is the number of significant digits PeriodicEvidence.Chance +// keeps in JSON. Chance comes from Exp/Expm1/Pow, whose last bits differ +// between CPU architectures (measured: at most about 5e-13 relative over the +// valid parameter range); six digits leave a margin of about 1e6 over that. +// Chance is a union-bound estimate compared with a MaxChance set to one or +// two significant digits, so six digits keep far more precision than the +// value carries. +const ChanceDigits = 6 + +// canonicalFloat rounds v to sig significant digits (correctly rounded, +// half to even on the exact binary value). Zero, NaN and infinities are +// returned unchanged. +func canonicalFloat(v float64, sig int) float64 { + if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) { + return v + } + r, err := strconv.ParseFloat(strconv.FormatFloat(v, 'e', sig-1, 64), 64) + if err != nil { + return v + } + return r +} + +// MarshalJSON writes Chance rounded to ChanceDigits significant digits, so +// the JSON is byte-identical on every architecture. Every other field is +// written unchanged. +func (e PeriodicEvidence) MarshalJSON() ([]byte, error) { + type plain PeriodicEvidence // no methods: avoids recursion + p := plain(e) + p.Chance = canonicalFloat(e.Chance, ChanceDigits) + return json.Marshal(p) +} + // MarshalJSON lists the set flags by name. func (f CoverageFlags) MarshalJSON() ([]byte, error) { names := []string{} diff --git a/internal/anomaly/chance_test.go b/internal/anomaly/chance_test.go new file mode 100644 index 000000000..d32c85cd9 --- /dev/null +++ b/internal/anomaly/chance_test.go @@ -0,0 +1,308 @@ +package anomaly + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "math" + "os" + "path/filepath" + "testing" + "time" +) + +// Numerical stability of the periodic null model and the canonical JSON of +// PeriodicEvidence.Chance. Chance comes from Exp/Expm1/Pow, whose last bits +// differ between CPU architectures; these tests pin what must not differ. + +// stateWithRate returns a pulse ring of n pulses spread evenly over span, so +// chance() sees the local rate (n-1)/span. +func stateWithRate(n int, span time.Duration, evaluated uint64) *periodicState { + p := newPeriodicState(maxHistoryLen) + for i := 0; i < n; i++ { + p.times[i] = t0.UnixNano() + int64(span)*int64(i)/int64(n-1) + } + p.n, p.evaluated = n, evaluated + return p +} + +// directChance is the textbook form with the subtraction +// exp(-l*lo) - exp(-l*hi): accurate only when the tolerance is not small. +func directChance(p *periodicState, c chainInfo, r *PeriodicRule) float64 { + lambda := float64(p.n-1) / float64(p.at(p.n-1)-p.at(0)) + var pg float64 + for k := 1; k <= r.MaxMissing+1; k++ { + lo := math.Max(0, float64(int64(k)*c.period-c.tol)) + hi := float64(int64(k)*c.period + c.tol) + pg += math.Exp(-lambda*lo) - math.Exp(-lambda*hi) + } + pg = math.Min(pg, 1) + return float64(p.evaluated) * float64(searchCells(r)) * math.Pow(pg, float64(c.gaps-1)) +} + +func relDiff(a, b float64) float64 { + if a == b { + return 0 + } + return math.Abs(a-b) / math.Max(math.Abs(a), math.Abs(b)) +} + +// Where the direct form is well conditioned, the expm1 form gives the same +// value (to within rounding). +func TestChanceExpm1FormMatchesTheDirectFormWhereWellConditioned(t *testing.T) { + period := int64(5 * time.Minute) + for _, rateTimesP := range []float64{0.2, 0.5, 1, 2, 5} { + span := time.Duration(float64(30*period) / rateTimesP) + p := stateWithRate(31, span, 1000) + for _, tolFrac := range []float64{0.01, 0.05, 0.1, 0.25} { + for missing := 0; missing <= 3; missing++ { + for _, gaps := range []int{3, 10, 30} { + r := PeriodicRule{MaxMissing: missing} + c := chainInfo{period: period, tol: int64(tolFrac * float64(period)), gaps: gaps} + got, want := p.chance(c, &r), directChance(p, c, &r) + if d := relDiff(got, want); d > 1e-12 { + t.Fatalf("lambda*P=%g tol=%g missing=%d gaps=%d: expm1 form %g, direct %g (rel %g)", + rateTimesP, tolFrac, missing, gaps, got, want, d) + } + } + } + } + } +} + +// For small (valid) tolerances the probability of one gap falling in +// [lo, hi] is exp(-l*lo) * x * (1 - x/2 + x^2/6 - ...) with x = l*(hi-lo). +// The expm1 form must stay at full precision there, even after Pow raises it +// to the longest chain; the direct form does not. +func TestChanceIsAccurateForSmallTolerances(t *testing.T) { + period := int64(time.Hour) + p := stateWithRate(2, time.Duration(period), 1) // local rate 1/P + lambda := 1 / float64(period) + r := PeriodicRule{MaxMissing: 0} + cells := float64(searchCells(&r)) + for _, x := range []float64{1e-12, 1e-9, 1e-6, 1e-4} { + tol := int64(x / lambda / 2) + if tol < 1 { + tol = 1 + } + xa := lambda * float64(2*tol) // the exact width after integer rounding + lo := float64(period - tol) + pgRef := math.Exp(-lambda*lo) * xa * (1 - xa/2 + xa*xa/6 - xa*xa*xa/24) + // the longest chain whose chance stays a normal float: Pow amplifies + // the relative error of pg by the exponent + longest := 1 + int(250/-math.Log10(pgRef)) + if longest > maxHistoryLen-1 { + longest = maxHistoryLen - 1 + } + for _, gaps := range []int{2, 3, longest} { + c := chainInfo{period: period, tol: tol, gaps: gaps} + want := cells * math.Pow(pgRef, float64(gaps-1)) + if !(want >= 1e-250) { + t.Fatalf("x=%g gaps=%d: reference %g left the normal range", x, gaps, want) + } + if d := relDiff(p.chance(c, &r), want); d > 1e-12 { + t.Fatalf("x=%g gaps=%d: chance off by %g relative", x, gaps, d) + } + } + } + // the oracle is sharp enough to see the cancellation it guards against + c := chainInfo{period: period, tol: int64(1e-9 / lambda / 2), gaps: 2} + xa := lambda * float64(2*c.tol) + pgRef := math.Exp(-lambda*float64(period-c.tol)) * xa * (1 - xa/2 + xa*xa/6) + if d := relDiff(directChance(p, c, &r), cells*pgRef); d < 1e-9 { + t.Fatalf("direct form unexpectedly accurate (rel %g): the oracle cannot tell the forms apart", d) + } +} + +// The detector decides on the full Chance, not on its rounded JSON form. +// Thresholds just below, at and just above the value give the same decision +// on every architecture: the literals below sit 1e-10 relative away from the +// value, far outside the measured cross-architecture spread (about 5e-13). +func TestMaxChanceDecisionAroundTheBoundary(t *testing.T) { + const ( + canonical = 4.59387e-08 // the value rounded to ChanceDigits + justBelow = 4.5938728007910717e-08 // value * (1 - 1e-10) + justAbove = 4.5938728017098456e-08 // value * (1 + 1e-10) + ) + signalAt := func(maxChance float64) (PeriodicEvidence, bool) { + r := experimentalPeriodic() + r.MaxChance = maxChance + p := newPeriodicState(r.HistoryLen) + var sc periodicScratch + var sig periodicSignal + var ok bool + for i := 0; i < r.MinPulses; i++ { + sig, ok = p.observe(at(time.Duration(i)*5*time.Minute).UnixNano(), TrafficUnclassified, &r, &sc) + } + return sig.ev, ok + } + ev, ok := signalAt(1) + if !ok { + t.Fatal("fixture: no signal at the first sufficient pulse") + } + raw := ev.Chance + if got := canonicalFloat(raw, ChanceDigits); got != canonical { + t.Fatalf("canonical Chance %v, want %v (raw %.17g)", got, canonical, raw) + } + for _, c := range []struct { + name string + maxChance float64 + fires bool + }{ + {"just below (1e-10)", justBelow, false}, + {"one ulp below", math.Nextafter(raw, 0), false}, + {"at the value", raw, true}, + {"one ulp above", math.Nextafter(raw, 1), true}, + {"just above (1e-10)", justAbove, true}, + } { + if _, fired := signalAt(c.maxChance); fired != c.fires { + t.Errorf("MaxChance %s (%.17g): fired=%v, want %v", c.name, c.maxChance, fired, c.fires) + } + } + // rounding does not leak into the decision: the rounded value lies on one + // side of raw, and a threshold between them decides by the raw value + mid := (raw + canonical) / 2 + if _, fired := signalAt(mid); fired != (raw <= mid) { + t.Errorf("threshold between raw %.17g and canonical %v decided by the rounded value", raw, canonical) + } +} + +func TestCanonicalFloat(t *testing.T) { + for _, c := range []struct{ in, want float64 }{ + {0, 0}, + {1.2345649999999, 1.23456}, // just below a rounding boundary + {1.2345650000001, 1.23457}, // just above it + {1234565, 1.23456e6}, // exact binary tie: half to even + {1234575, 1.23458e6}, // exact binary tie: half to even + {-1.2345650000001, -1.23457}, // negative + {-0.000241965270782, -0.000241965}, + {4.593872801250516e-8, 4.59387e-8}, + {4.593872801250437e-8, 4.59387e-8}, + {1e-310, 1e-310}, // subnormal + {2.459010123456e-164, 2.45901e-164}, + {144867615, 1.44868e8}, // large + {math.MaxFloat64, 1.79769e308}, + } { + if got := canonicalFloat(c.in, ChanceDigits); got != c.want { + t.Errorf("canonicalFloat(%.17g) = %.17g, want %.17g", c.in, got, c.want) + } + } + if got := canonicalFloat(math.Copysign(0, -1), ChanceDigits); !math.Signbit(got) || got != 0 { + t.Error("negative zero not kept") + } + for _, v := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + got := canonicalFloat(v, ChanceDigits) + if !(math.IsNaN(v) && math.IsNaN(got)) && got != v { + t.Errorf("canonicalFloat(%v) = %v", v, got) + } + } +} + +// Values that differ only in their last bits (as on amd64 vs arm64) give +// byte-identical JSON; every other field is written unchanged. +func TestPeriodicEvidenceJSONIsCanonical(t *testing.T) { + base := PeriodicEvidence{Period: 5 * time.Minute, Tolerance: 6 * time.Second, JitterMedian: time.Second, + Support: 8, Events: 9, Coverage: 1.0 / 3, SufficientAt: t0, + Alternatives: []PeriodAlternative{{Period: 10 * time.Minute, Support: 4, Coverage: 0.5}}} + for _, pair := range [][2]float64{ + {4.593872801250516e-8, 4.593872801250437e-8}, // the two sides of the CI diff + {0.00024196527078255232, 0.00024196527078255262}, // likewise + {1.2345612345678, math.Nextafter(1.2345612345678, 2)}, + } { + a, b := base, base + a.Chance, b.Chance = pair[0], pair[1] + ja, err := json.Marshal(a) + if err != nil { + t.Fatal(err) + } + jb, _ := json.Marshal(b) + if !bytes.Equal(ja, jb) { + t.Fatalf("JSON differs for %.17g and %.17g:\n%s\n%s", pair[0], pair[1], ja, jb) + } + } + e := base + e.Chance = 4.593872801250516e-8 + got, _ := json.Marshal(e) + type plain PeriodicEvidence + p := plain(base) + p.Chance = 4.59387e-8 + want, _ := json.Marshal(p) + if !bytes.Equal(got, want) { + t.Fatalf("only Chance may change:\n got %s\nwant %s", got, want) + } + if !bytes.Contains(got, []byte(`"Chance":4.59387e-8`)) { + t.Fatalf("Chance not written with %d significant digits: %s", ChanceDigits, got) + } +} + +// goldenWithoutChance is the SHA-256 of testdata/replay_golden.json with every +// "Chance" member removed (numbers kept verbatim, keys sorted), taken from +// the golden file before Chance was canonicalized. Timestamps, transitions, +// reason codes, coverage, counters and the number of candidates are therefore +// unchanged; only the documented Chance values changed. +const goldenWithoutChance = "2538014b83e1976a614da59c3db2eeb8d403ba76de63f3f58d55aad221eaf959" + +func stripChance(v interface{}) interface{} { + switch x := v.(type) { + case map[string]interface{}: + delete(x, "Chance") + for k, c := range x { + x[k] = stripChance(c) + } + case []interface{}: + for i, c := range x { + x[i] = stripChance(c) + } + } + return v +} + +func collectChance(v interface{}, out *[]string) { + switch x := v.(type) { + case map[string]interface{}: + if c, ok := x["Chance"]; ok { + *out = append(*out, string(c.(json.Number))) + } + for _, c := range x { + collectChance(c, out) + } + case []interface{}: + for _, c := range x { + collectChance(c, out) + } + } +} + +func TestGoldenChangedOnlyInTheDocumentedChanceValues(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("testdata", "replay_golden.json")) + if err != nil { + t.Fatal(err) + } + decode := func() interface{} { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v interface{} + if err := dec.Decode(&v); err != nil { + t.Fatal(err) + } + return v + } + var chances []string + collectChance(decode(), &chances) + n := map[string]int{} + for _, c := range chances { + n[c]++ + } + if len(chances) != 4 || n["4.59387e-8"] != 2 || n["0.000241965"] != 2 { + t.Fatalf("golden Chance values %v, want 4.59387e-8 x2 and 0.000241965 x2", chances) + } + b, err := json.Marshal(stripChance(decode())) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(b) + if got := hex.EncodeToString(sum[:]); got != goldenWithoutChance { + t.Fatalf("golden changed outside Chance: sha256 %s, want %s", got, goldenWithoutChance) + } +} diff --git a/internal/anomaly/periodic.go b/internal/anomaly/periodic.go index 2b3b4782c..edc2e5245 100644 --- a/internal/anomaly/periodic.go +++ b/internal/anomaly/periodic.go @@ -396,7 +396,11 @@ func (p *periodicState) chance(c chainInfo, r *PeriodicRule) float64 { lo = 0 } hi := float64(int64(k)*c.period + c.tol) - pg += math.Exp(-lambda*lo) - math.Exp(-lambda*hi) + // P(lo <= gap <= hi) = exp(-l*lo) - exp(-l*hi), written without the + // subtraction of two nearly equal numbers: for small tolerances + // that subtraction loses digits, which Pow then amplifies (and the + // lost digits differ between CPU architectures). + pg += math.Exp(-lambda*lo) * -math.Expm1(-lambda*(hi-lo)) } if pg > 1 { pg = 1 diff --git a/internal/anomaly/replay_test.go b/internal/anomaly/replay_test.go index 5955f4e38..3dc7c3e09 100644 --- a/internal/anomaly/replay_test.go +++ b/internal/anomaly/replay_test.go @@ -66,8 +66,21 @@ func TestReplayFixtureIsDeterministicAndMatchesGolden(t *testing.T) { if err != nil { t.Fatalf("missing golden file (run with -anomaly.update): %v", err) } - if !bytes.Equal(append(a, '\n'), want) { - t.Fatal("replay output differs from testdata/replay_golden.json (run with -anomaly.update after reviewing)") + if got := append(a, '\n'); !bytes.Equal(got, want) { + // name the first differing line so a CI log shows the actual diff + gl, wl := strings.Split(string(got), "\n"), strings.Split(string(want), "\n") + for i := 0; i < len(gl) || i < len(wl); i++ { + var g, w string + if i < len(gl) { + g = gl[i] + } + if i < len(wl) { + w = wl[i] + } + if g != w { + t.Fatalf("replay output differs from testdata/replay_golden.json at line %d:\n got: %s\nwant: %s\n(run with -anomaly.update after reviewing)", i+1, g, w) + } + } } } diff --git a/internal/anomaly/testdata/replay_golden.json b/internal/anomaly/testdata/replay_golden.json index 23df531c3..5cf04da4d 100644 --- a/internal/anomaly/testdata/replay_golden.json +++ b/internal/anomaly/testdata/replay_golden.json @@ -87,7 +87,7 @@ "Support": 8, "Events": 8, "Coverage": 1, - "Chance": 4.593872801250516e-8, + "Chance": 4.59387e-8, "SufficientAt": "2030-01-07T12:35:00Z", "Alternatives": [ { @@ -127,7 +127,7 @@ "Support": 8, "Events": 8, "Coverage": 1, - "Chance": 4.593872801250516e-8, + "Chance": 4.59387e-8, "SufficientAt": "2030-01-07T12:35:00Z", "Alternatives": [ { @@ -242,7 +242,7 @@ "Support": 8, "Events": 8, "Coverage": 1, - "Chance": 0.00024196527078255232, + "Chance": 0.000241965, "SufficientAt": "2030-01-07T12:47:00Z", "Alternatives": null } @@ -271,7 +271,7 @@ "Support": 8, "Events": 8, "Coverage": 1, - "Chance": 0.00024196527078255232, + "Chance": 0.000241965, "SufficientAt": "2030-01-07T12:47:00Z", "Alternatives": null } From 5b816f6644f58b0e63e1dfc26a1dad2a92ae53f1 Mon Sep 17 00:00:00 2001 From: Dennis Jakobsen Date: Fri, 25 Sep 2026 15:21:52 +0200 Subject: [PATCH 7/9] fix(anomaly): address the six independent-review findings 1. Periodic: an alternating pattern within tolerance (294s/306s around 300s) was missed because every candidate came from a single gap/k. Each seed now also yields a refined candidate: the periods common to the 16 newest gaps (integer interval intersection), checked with their own tolerance. The seeds' estimate is unchanged; a refined candidate replaces it only when it would signal (meets the criteria, Chance within MaxChance) and strictly outranks it. searchCells, the union bound and every Chance value stay as before. 2. Dedup: a repeated ID could stay a duplicate forever. A repeat more than DedupHorizon after the stored copy now counts again (inclusive boundary); an older copy stays a duplicate. 3. HistoryStart: the automatic start is set when the first event is processed, not when it is accepted, so reorder cannot change results. 4. Config: New now also copies each rule's PayloadTypes. 5. Replay: width 0 with bytes is rejected for full_route and first_hop, as NewRoute does. 6. BurstGap validation no longer overflows (BurstGap >= MinPeriod-BurstGap). The ad3ab30f numeric fix, its tests and the golden file are unchanged. Cost, accepted for this fix: the periodic search is up to about 30% slower in the worst case and end to end about 4-10% slower, with no new allocations; a test enforces a per-pulse bound and pins the total work. Co-Authored-By: Claude Opus 5.5 --- internal/anomaly/config.go | 2 +- internal/anomaly/detector.go | 25 +- internal/anomaly/independent_review_test.go | 121 +++ internal/anomaly/periodic.go | 177 +++- internal/anomaly/periodic_bench_test.go | 178 ++++ internal/anomaly/replay.go | 10 +- internal/anomaly/review_fixes_test.go | 949 ++++++++++++++++++++ 7 files changed, 1435 insertions(+), 27 deletions(-) create mode 100644 internal/anomaly/independent_review_test.go create mode 100644 internal/anomaly/periodic_bench_test.go create mode 100644 internal/anomaly/review_fixes_test.go diff --git a/internal/anomaly/config.go b/internal/anomaly/config.go index ecd4a1dc7..0600c74a0 100644 --- a/internal/anomaly/config.go +++ b/internal/anomaly/config.go @@ -323,7 +323,7 @@ func (c *Config) Validate() error { switch { case r.MinPeriod <= 0 || r.MaxPeriod <= r.MinPeriod || r.MaxPeriod > maxWindow: return fmt.Errorf("anomaly: rule %s: need 0 < MinPeriod < MaxPeriod <= 24h", r.Name) - case r.BurstGap < 0 || 2*r.BurstGap >= r.MinPeriod: + case r.BurstGap < 0 || r.BurstGap >= r.MinPeriod-r.BurstGap: // 2*BurstGap >= MinPeriod, without overflow return fmt.Errorf("anomaly: rule %s: BurstGap must be in [0, MinPeriod/2)", r.Name) case r.JitterAbs < 0 || !finiteIn(r.JitterRel, 0, 0.25) || (r.JitterAbs == 0 && r.JitterRel == 0): return fmt.Errorf("anomaly: rule %s: need JitterAbs >= 0, JitterRel in [0,0.25], not both 0", r.Name) diff --git a/internal/anomaly/detector.go b/internal/anomaly/detector.go index b9a732d08..a451eb083 100644 --- a/internal/anomaly/detector.go +++ b/internal/anomaly/detector.go @@ -2,6 +2,7 @@ package anomaly import ( "container/heap" + "slices" "time" ) @@ -133,6 +134,17 @@ func New(cfg Config) (*Detector, error) { c.Rate = append([]RateRule(nil), cfg.Rate...) c.NewStream = append([]NewStreamRule(nil), cfg.NewStream...) c.Periodic = append([]PeriodicRule(nil), cfg.Periodic...) + // PayloadTypes are the only slices inside the rules: copy them too, so the + // caller's config and the Detector never share a backing array. + for i := range c.Rate { + c.Rate[i].PayloadTypes = slices.Clone(c.Rate[i].PayloadTypes) + } + for i := range c.NewStream { + c.NewStream[i].PayloadTypes = slices.Clone(c.NewStream[i].PayloadTypes) + } + for i := range c.Periodic { + c.Periodic[i].PayloadTypes = slices.Clone(c.Periodic[i].PayloadTypes) + } d := &Detector{cfg: c, dedup: make(map[TxID]int64)} for i := range c.Rate { d.table(c.Rate[i].Scope).addRate(i, c.Rate[i].Window) @@ -171,7 +183,10 @@ func (d *Detector) Observe(e Event) Result { if f := e.finalAt().UnixNano(); f > d.clock { d.clock = f } - if prev, ok := d.dedup[e.ID]; ok { + // A repeat more than DedupHorizon after the remembered copy counts again, + // even if no accepted event has expired that copy yet. An older copy + // (t < prev) stays a duplicate, so it can never revive the ID. + if prev, ok := d.dedup[e.ID]; ok && t-prev <= int64(d.cfg.Limits.DedupHorizon) { d.stats.Duplicates++ if prev != t { d.stats.DuplicateConflicts++ @@ -189,9 +204,6 @@ func (d *Detector) Observe(e Event) Result { return d.finish(res) } d.started = true - if !d.historySet { - d.history, d.historySet = t, true - } d.rememberID(e.ID, t) d.stats.Accepted++ if e.Truncated { @@ -574,6 +586,11 @@ type emitCtx struct { func (d *Detector) handle(e *Event) { d.stats.Processed++ t := e.Time.UnixNano() + if !d.historySet { + // Config.HistoryStart zero: the first event processed, which with a + // reorder delay need not be the first one accepted. + d.history, d.historySet = t, true + } d.lastEvT, d.lastEvID, d.evDone = t, e.ID, true for s := Scope(1); s < numScopes; s++ { tb := d.scopes[s] diff --git a/internal/anomaly/independent_review_test.go b/internal/anomaly/independent_review_test.go new file mode 100644 index 000000000..2c8e8a66b --- /dev/null +++ b/internal/anomaly/independent_review_test.go @@ -0,0 +1,121 @@ +package anomaly + +import ( + "encoding/json" + "fmt" + "math" + "strings" + "testing" + "time" +) + +func TestIndependentRejectZeroWidthRouteWithBytes(t *testing.T) { + var rec ReplayRecord + if err := json.Unmarshal([]byte(replayLine("badroute")), &rec); err != nil { + t.Fatal(err) + } + rec.FullRoute = "0:deadbeef" + line, err := json.Marshal(rec) + if err != nil { + t.Fatal(err) + } + rep, err := Replay(strings.NewReader(string(line)), replayFixtureConfig(t), ReplayOptions{Start: t0, End: at(time.Hour), MaxLineBytes: 4096}) + if err != nil { + t.Fatal(err) + } + if rep.Input.Accepted != 0 || rep.Input.Invalid != 1 { + t.Fatalf("nonempty route with width zero was accepted: %+v", rep.Input) + } +} + +func TestIndependentRejectBurstGapOverflow(t *testing.T) { + rule := experimentalPeriodic() + rule.BurstGap = time.Duration(math.MaxInt64) + cfg := Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}, Periodic: []PeriodicRule{rule}} + if err := cfg.Validate(); err == nil { + t.Fatal("BurstGap=MaxInt64 passed validation despite BurstGap >= MinPeriod/2") + } +} + +func TestIndependentDedupExpiresBeforeRepeatedID(t *testing.T) { + lim := experimentalLimits() + lim.DedupHorizon = 2 * time.Minute + d := newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeGlobal, time.Minute, 1)}}) + if got := d.Observe(relayEvent(t, "same", t0, 1, 2)).Status; got != StatusAccepted { + t.Fatal(got) + } + if got := d.Observe(relayEvent(t, "same", at(3*time.Minute), 1, 2)).Status; got != StatusAccepted { + t.Fatalf("ID repeated after 3m with a 2m horizon: got %v, want accepted", got) + } +} + +func TestIndependentHistoryStartsAtFirstProcessedEvent(t *testing.T) { + lim := experimentalLimits() + lim.ReorderDelay = 30 * time.Second + rule := NewStreamRule{Name: "ns", Scope: ScopeStream, Window: time.Minute, MinCount: 1, + CensorWindow: 10 * time.Second, QuietPeriod: 10 * time.Second, State: quickState()} + cfg := Config{Limits: lim, NewStream: []NewStreamRule{rule}} + early := relayEvent(t, "early", t0, 1, 2) + later := relayEvent(t, "later", at(20*time.Second), 1, 3) + for _, tc := range []struct { + name string + events []Event + }{{"ordered", []Event{early, later}}, {"reordered", []Event{later, early}}} { + t.Run(tc.name, func(t *testing.T) { + cs := filter(run(t, newDet(t, cfg), tc.events), "ns", StateActive) + for _, c := range cs { + if c.TriggerID == later.ID { + if c.NewStream.Censored { + t.Fatalf("event 20s after history start is censored despite 10s censor window; confidence=%v", c.Confidence) + } + return + } + } + t.Fatal("later stream candidate missing") + }) + } +} + +func TestIndependentConfigCopiesPayloadFilters(t *testing.T) { + rule := rateRule("r", ScopeGlobal, time.Minute, 1) + rule.PayloadTypes = []PayloadType{5} + cfg := Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}, Rate: []RateRule{rule}} + d, err := New(cfg) + if err != nil { + t.Fatal(err) + } + cfg.Rate[0].PayloadTypes[0] = 4 + r := d.Observe(relayEvent(t, "type5", t0, 1, 2)) + if len(filter(r.Candidates, "r", StateActive)) != 1 { + t.Fatalf("mutating caller's config removed the expected type-5 activation; candidates=%d", len(r.Candidates)) + } +} + +func TestIndependentPeriodicAlternatingAllowedJitter(t *testing.T) { + rule := experimentalPeriodic() + rule.JitterAbs, rule.JitterRel, rule.MaxJitterFraction = 6*time.Second, 0, 1 + cfg := Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}, Periodic: []PeriodicRule{rule}} + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + p := newPeriodicState(rule.HistoryLen) + var scratch periodicScratch + var events []Event + for i := 0; i < 32; i++ { + when := time.Duration(i) * 5 * time.Minute + if i%2 == 1 { + when -= 6 * time.Second + } + events = append(events, relayEvent(t, fmt.Sprintf("pulse-%02d", i), at(when), 1, 2)) + p.observe(at(when).UnixNano(), TrafficUnclassified, &rule, &scratch) + } + ideal := p.chainFor(int64(5*time.Minute), &rule, &scratch) + chance := p.chance(ideal, &rule) + if !ideal.meets(&rule) || chance > rule.MaxChance { + t.Fatalf("invalid repro: ideal=%+v chance=%g", ideal, chance) + } + cs := run(t, newDet(t, cfg), events) + if len(filter(cs, rule.Name, StateActive)) == 0 { + t.Fatalf("32 valid pulses were not detected: period=5m, gaps=294s/306s, tolerance=6s, coverage=%g, chance=%g", ideal.coverage, chance) + } +} diff --git a/internal/anomaly/periodic.go b/internal/anomaly/periodic.go index edc2e5245..74e00df4f 100644 --- a/internal/anomaly/periodic.go +++ b/internal/anomaly/periodic.go @@ -15,8 +15,13 @@ import ( // the newest pulse with one period P: a gap g is explained if // |g - k*P| <= tol(P) for some k in 1..MaxMissing+1 (k-1 missing pulses). // -// Candidate periods come from recent gaps divided by k, refined with the -// phase estimator (chain span / periods spanned). They are ranked by +// Candidate periods come from recent gaps divided by k, and the best is +// refined with the phase estimator (chain span / periods spanned). Where a +// candidate misses some of the newest gaps that one common period explains, +// that period is measured too (chainRefined); the best such period replaces +// the estimate only if it would signal (meets the criteria, Chance within +// MaxChance) and strictly outranks it. +// Candidates are ranked by // (meets MinPulses/MinCoverage/MaxJitterFraction first, then chain length // desc, coverage desc, median residual asc, period asc). A multiple of the // true period cannot explain the gaps of the true period, so its chain @@ -92,7 +97,10 @@ func classFlags(c TrafficClass) pulseFlags { // goroutine), so periodic evaluation neither allocates per event nor keeps // buffers per key. type periodicScratch struct { - resid, med, seen []int64 + resid, med, seen, seenRef []int64 + // work counters (chains measured, gaps visited), for bound tests and + // benchmarks + chains, steps uint64 } // periodicSignal is a hot evaluation result. @@ -156,6 +164,21 @@ func (c chainInfo) preferred(o chainInfo, r *PeriodicRule) bool { return c.better(o) } +// outranks is preferred without the final tie-break on the period: an exact +// tie on everything that describes the chain keeps o. +func (c chainInfo) outranks(o chainInfo, r *PeriodicRule) bool { + if cm, om := c.meets(r), o.meets(r); cm != om { + return cm + } + if c.gaps != o.gaps { + return c.gaps > o.gaps + } + if c.coverage != o.coverage { + return c.coverage > o.coverage + } + return c.residual < o.residual +} + func (c chainInfo) better(o chainInfo) bool { if c.gaps != o.gaps { return c.gaps > o.gaps @@ -205,7 +228,9 @@ func (p *periodicState) chainFor(period int64, r *PeriodicRule, sc *periodicScra c := chainInfo{period: period, tol: tolNanos(r, period)} scratch := sc.resid[:0] maxK := r.MaxMissing + 1 + sc.chains++ for i := p.n - 1; i >= 1; i-- { + sc.steps++ g := p.at(i) - p.at(i-1) k, d := explain(g, period, c.tol, maxK) if k == 0 { @@ -332,7 +357,14 @@ func (p *periodicState) estimate(r *PeriodicRule, sc *periodicScratch) chainInfo if newest+tolNanos(r, maxP) < minP-tolNanos(r, minP) || newest > int64(maxK)*maxP+tolNanos(r, maxP) { return best } - seen := sc.seen[:0] + // The seeds and the phase refinement below give exactly the estimate + // they give without refined candidates; the best refined candidate + // (ranked on its own) replaces that estimate only if it would signal and + // strictly outranks it. So the search never returns a chain the seeds + // alone would rank higher, and until a refined period would signal it + // changes nothing. + var bestRef chainInfo + seen, seenRef := sc.seen[:0], sc.seenRef[:0] for i := p.n - 1; i >= 1 && i >= p.n-recentGaps; i-- { g := p.at(i) - p.at(i-1) for k := 1; k <= maxK; k++ { @@ -344,29 +376,140 @@ func (p *periodicState) estimate(r *PeriodicRule, sc *periodicScratch) chainInfo continue } seen = append(seen, cand) - c := p.chainFor(cand, r, sc) + c, ref := p.chainRefined(cand, r, sc) if c.gaps > 0 && (best.gaps == 0 || c.preferred(best, r)) { best = c } + if ref != cand && !slices.Contains(seenRef, ref) && !slices.Contains(seen, ref) { + seenRef = append(seenRef, ref) + if c := p.chainFor(ref, r, sc); c.gaps > 0 && (bestRef.gaps == 0 || c.preferred(bestRef, r)) { + bestRef = c + } + } } } - sc.seen = seen - if best.gaps == 0 { - return best - } - // Refine with the phase estimator span/slots over the chain. It is - // robust to per-pulse jitter, whereas ranking by gap residual favours a - // biased gap. It replaces the estimate whenever it explains a chain at - // least as long with at least the same coverage. - span := p.at(p.n-1) - p.at(p.n-1-best.gaps) - if ref := span / int64(best.slots); ref >= minP && ref <= maxP && ref != best.period { - if c := p.chainFor(ref, r, sc); c.gaps >= best.gaps && c.coverage >= best.coverage && (c.meets(r) || !best.meets(r)) { - best = c + sc.seen, sc.seenRef = seen, seenRef + if best.gaps > 0 { + // Refine with the phase estimator span/slots over the chain. It is + // robust to per-pulse jitter, whereas ranking by gap residual favours + // a biased gap. It replaces the estimate whenever it explains a chain + // at least as long with at least the same coverage. + span := p.at(p.n-1) - p.at(p.n-1-best.gaps) + if ref := span / int64(best.slots); ref >= minP && ref <= maxP && ref != best.period { + if c := p.chainFor(ref, r, sc); c.gaps >= best.gaps && c.coverage >= best.coverage && (c.meets(r) || !best.meets(r)) { + best = c + } } } + // only a refined candidate that would signal (meets the criteria and its + // Chance is within MaxChance): until one does, the hypothesis kept from + // pulse to pulse stays the one the seeds give, and a kept hypothesis + // that meets the criteria stops the search, so it must not be one that + // cannot signal yet + if bestRef.meets(r) && p.chance(bestRef, r) <= r.MaxChance && (best.gaps == 0 || bestRef.outranks(best, r)) { + best = bestRef + } return best } +// chainRefined measures seed's chain exactly as chainFor does and returns it +// with the period this search cell tests: seed, or a refinement of it. +// +// A single gap fixes P only to within tol/k. When the jitter of neighbouring +// gaps cancels (gaps of 294s, 306s, 294s... around 300s with tol 6s) no seed +// taken from one gap explains the next one, so no chain forms. Every period +// in the intersection of the ranges [(g-tol)/k, (g+tol)/k] of the newest gaps +// explains all of them. In the same pass as the chain, over the recentGaps +// newest gaps (the window the seeds come from) and with the multiples k the +// seed assigns them, chainRefined intersects those ranges. If the common +// range covers at least two gaps and more than the seed's chain, it also +// returns the phase estimate span/slots clamped into it, which estimate() +// measures as a refined candidate. +// +// A refined period is fitted to up to recentGaps gaps, not one, so a chance +// chain fits it more often than pg^(gaps-1) alone says. The union bound in +// chance() still covers it: a period explaining n of the newest gaps exists +// roughly as often as one of those n gaps' own seeds explains the others, +// and searchCells already charges each of those seeds. So searchCells, and +// the Chance values, are unchanged. +// +// Cost: the chain pass as before, the range adds O(1) per gap in the window; +// a refined cell adds one walk of recentGaps gaps and one chainFor. No +// allocation; see BenchmarkPeriodicSearch and TestPeriodicSearchWorkIsBounded. +func (p *periodicState) chainRefined(seed int64, r *PeriodicRule, sc *periodicScratch) (chainInfo, int64) { + c := chainInfo{period: seed, tol: tolNanos(r, seed)} + resid := sc.resid[:0] + maxK := r.MaxMissing + 1 + lo, hi := int64(1), int64(math.MaxInt64) + var n int + var span, slots int64 + inChain, inRange := true, true + sc.chains++ + for i := p.n - 1; i >= 1; i-- { + if inRange = inRange && i >= p.n-recentGaps; !inChain && !inRange { + break + } + sc.steps++ + g := p.at(i) - p.at(i-1) + if inChain { + if k, d := explain(g, seed, c.tol, maxK); k > 0 { + c.gaps++ + c.slots += k + resid = append(resid, d) + } else { + inChain = false + } + } + if inRange { + k := max((g+seed/2)/seed, 1) + l, h := lo, min(hi, (g+c.tol)/k) + if g-c.tol > 0 { + l = max(l, (g-c.tol+k-1)/k) + } + if k > int64(maxK) || l > h { + inRange = false + } else { + lo, hi = l, h + n++ + span += g + slots += k + } + } + } + sc.resid = resid + if c.gaps > 0 { + c.coverage = float64(c.gaps) / float64(c.slots) + c.residual = medianInt64(resid, sc) + } + // With one gap the phase estimate is newest/k, a seed the search tries + // anyway. + if n < 2 || n <= c.gaps { + return c, seed + } + ref := min(max(span/slots, lo), hi) + // The range uses the seed's tolerance, but the tolerance grows with the + // period (JitterRel): keep ref only if, with its own tolerance, it + // explains more of the newest gaps than the seed does. + if ref < int64(r.MinPeriod) || ref > int64(r.MaxPeriod) || p.recentExplained(ref, r, sc) <= c.gaps { + return c, seed + } + return c, ref +} + +// recentExplained counts the consecutive newest gaps, at most recentGaps, +// that period explains with its own tolerance. +func (p *periodicState) recentExplained(period int64, r *PeriodicRule, sc *periodicScratch) int { + tol, maxK, n := tolNanos(r, period), r.MaxMissing+1, 0 + for i := p.n - 1; i >= 1 && i >= p.n-recentGaps; i-- { + sc.steps++ + if k, _ := explain(p.at(i)-p.at(i-1), period, tol, maxK); k == 0 { + break + } + n++ + } + return n +} + // alternatives reports multiples and sub-multiples of the chosen period // whose own chain is at least half as long (ambiguous explanations). func (p *periodicState) alternatives(best chainInfo, r *PeriodicRule, sc *periodicScratch) []PeriodAlternative { diff --git a/internal/anomaly/periodic_bench_test.go b/internal/anomaly/periodic_bench_test.go new file mode 100644 index 000000000..e3a4a46f9 --- /dev/null +++ b/internal/anomaly/periodic_bench_test.go @@ -0,0 +1,178 @@ +package anomaly + +import ( + "fmt" + "math/rand" + "testing" + "time" +) + +// Cost of the periodic candidate search per pulse, at the fixture rule and at +// the largest allowed ring and MaxMissing, for the traffic shapes that drive +// the search differently. + +func maxPeriodicRule() PeriodicRule { + r := experimentalPeriodic() + r.HistoryLen, r.MaxMissing, r.JitterAbs = maxHistoryLen, maxMissingLimit, 6*time.Second + return r +} + +// worstPeriodicRule never lets a chain meet the criteria, so no hypothesis is +// kept and the full search runs on every pulse, over long chains. +func worstPeriodicRule() PeriodicRule { + r := maxPeriodicRule() + r.MinCoverage, r.MaxJitterFraction = 1, 0.01 + return r +} + +var benchShapes = []string{"periodic", "alternating", "jitter", "missing", "bursty", "noise"} + +// pulseTimes returns n increasing pulse times (ns) of one traffic shape. +func pulseTimes(shape string, n int) []int64 { + rng := rand.New(rand.NewSource(1)) + const p = int64(5 * time.Minute) + out := make([]int64, 0, n) + t := t0.UnixNano() + for len(out) < n { + i := int64(len(out)) + switch shape { + case "periodic": + out = append(out, t+i*p) + case "alternating": + d := int64(0) + if i%2 == 1 { + d = -int64(6 * time.Second) + } + out = append(out, t+i*p+d) + case "jitter": + out = append(out, t+i*p+rng.Int63n(int64(6*time.Second))-int64(3*time.Second)) + case "missing": + if rng.Intn(2) == 0 { + t += p + } + t += p + out = append(out, t) + case "bursty": + t += int64(10*time.Second) + rng.Int63n(int64(10*time.Minute)) + out = append(out, t) + case "noise": + t += int64(10*time.Second) + int64(rng.ExpFloat64()*float64(p)) + out = append(out, t) + default: + panic(shape) + } + } + return out +} + +// pulseFeeder feeds a shape's pulse times into one state per rule, cycling +// through the times with a growing offset so time keeps increasing. +type pulseFeeder struct { + rules []PeriodicRule + states []*periodicState + times []int64 + span int64 + i int + sc periodicScratch +} + +func newPulseFeeder(rules []PeriodicRule, shape string) *pulseFeeder { + f := &pulseFeeder{rules: rules, times: pulseTimes(shape, 4096)} + for i := range rules { + f.states = append(f.states, newPeriodicState(rules[i].HistoryLen)) + } + f.span = f.times[len(f.times)-1] - f.times[0] + int64(time.Hour) + return f +} + +func (f *pulseFeeder) feed(pulses int) { + for n := 0; n < pulses; n++ { + t := f.times[f.i%len(f.times)] + int64(f.i/len(f.times))*f.span + f.i++ + for j := range f.rules { + f.states[j].observe(t, TrafficUnclassified, &f.rules[j], &f.sc) + } + } +} + +var benchConfigs = []struct { + name string + rules []PeriodicRule +}{ + {"fixture", []PeriodicRule{experimentalPeriodic()}}, + {"max", []PeriodicRule{maxPeriodicRule()}}, + {"max-4rules", []PeriodicRule{maxPeriodicRule(), maxPeriodicRule(), maxPeriodicRule(), maxPeriodicRule()}}, + {"worst", []PeriodicRule{worstPeriodicRule()}}, +} + +// periodicWorkBound is the most work one pulse evaluation may do: chain +// measurements for the kept hypothesis and its MaxMissing multiples, at most +// two per seed (the seed's pass and, for a refined cell, its period), the +// final refinement and four alternatives, each visiting at most HistoryLen-1 +// gaps; plus one verification walk of recentGaps gaps per seed. +func periodicWorkBound(r *PeriodicRule) (chains, steps uint64) { + seeds := uint64(recentGaps * (r.MaxMissing + 1)) + chains = uint64(1+r.MaxMissing) + 2*seeds + 1 + 4 + return chains, chains*uint64(r.HistoryLen-1) + seeds*recentGaps +} + +// measuredSteps is the total number of gaps visited by the search over the +// deterministic runs of TestPeriodicSearchWorkIsBounded (4*maxHistoryLen +// pulses per shape). The test allows 25% above it, so a change that makes +// the search markedly more expensive fails here even though it stays +// within the loose analytic periodicWorkBound. +var measuredSteps = map[string]uint64{ + "periodic/0.6": 236631, "alternating/0.6": 1434346, "jitter/0.6": 246825, + "missing/0.6": 243414, "bursty/0.6": 196561, "noise/0.6": 144778, + "periodic/1": 236631, "alternating/1": 1434346, "jitter/1": 5846059, + "missing/1": 1174373, "bursty/1": 196561, "noise/1": 144778, +} + +// Every single pulse stays within periodicWorkBound, at the largest allowed +// ring and MaxMissing, for every traffic shape, including the configuration +// where no chain ever meets the criteria and the search runs on every pulse; +// and the total work stays within 25% of measuredSteps. +func TestPeriodicSearchWorkIsBounded(t *testing.T) { + for _, r := range []PeriodicRule{maxPeriodicRule(), worstPeriodicRule()} { + maxChains, maxSteps := periodicWorkBound(&r) + for _, shape := range benchShapes { + f := newPulseFeeder([]PeriodicRule{r}, shape) + var peakC, peakS uint64 + for i := 0; i < 4*maxHistoryLen; i++ { + c0, s0 := f.sc.chains, f.sc.steps + f.feed(1) + dc, ds := f.sc.chains-c0, f.sc.steps-s0 + if dc > maxChains || ds > maxSteps { + t.Fatalf("%s MinCoverage=%g: pulse %d did %d chains and %d steps, bound %d and %d", + shape, r.MinCoverage, i, dc, ds, maxChains, maxSteps) + } + peakC, peakS = max(peakC, dc), max(peakS, ds) + } + key := fmt.Sprintf("%s/%g", shape, r.MinCoverage) + if m, ok := measuredSteps[key]; !ok || f.sc.steps > m+m/4 { + t.Errorf("%s: %d steps in total, measured %d (+25%% allowed)", key, f.sc.steps, m) + } + t.Logf("%-11s MinCoverage=%g: peak %d chains (bound %d), %d steps (bound %d), total %d steps", + shape, r.MinCoverage, peakC, maxChains, peakS, maxSteps, f.sc.steps) + } + } +} + +func BenchmarkPeriodicSearch(b *testing.B) { + for _, cfg := range benchConfigs { + for _, shape := range benchShapes { + b.Run(fmt.Sprintf("%s/%s", cfg.name, shape), func(b *testing.B) { + f := newPulseFeeder(cfg.rules, shape) + f.feed(2 * maxHistoryLen) // full rings and grown scratch buffers + c0, s0 := f.sc.chains, f.sc.steps + b.ReportAllocs() + b.ResetTimer() + f.feed(b.N) + b.StopTimer() + n := float64(b.N * len(cfg.rules)) + b.ReportMetric(float64(f.sc.chains-c0)/n, "chains/pulse") + b.ReportMetric(float64(f.sc.steps-s0)/n, "steps/pulse") + }) + } + } +} diff --git a/internal/anomaly/replay.go b/internal/anomaly/replay.go index e19da4fc6..8dd879c60 100644 --- a/internal/anomaly/replay.go +++ b/internal/anomaly/replay.go @@ -144,9 +144,11 @@ func (r ReplayRecord) Event() (Event, error) { return Event{}, err } e.RouteClass, e.RouteKind = RouteClass(rc), RouteKind(rk) + // Width 0 is valid only with no bytes (an empty hop or route); NewRoute + // enforces the same, so nonempty bytes never vanish silently. if w, b, err := parseWidthHex(r.FirstHop); err != nil { return Event{}, err - } else if w != 0 { + } else if w != 0 || len(b) != 0 { if w != len(b) { return Event{}, errors.New("anomaly: first_hop width mismatch") } @@ -156,10 +158,8 @@ func (r ReplayRecord) Event() (Event, error) { } if w, b, err := parseWidthHex(r.FullRoute); err != nil { return Event{}, err - } else if w != 0 { - if e.FullRoute, err = NewRoute(w, b); err != nil { - return Event{}, err - } + } else if e.FullRoute, err = NewRoute(w, b); err != nil { + return Event{}, err } if r.PayloadBytes != nil { e.PayloadBytes, e.PayloadKnown = *r.PayloadBytes, true diff --git a/internal/anomaly/review_fixes_test.go b/internal/anomaly/review_fixes_test.go new file mode 100644 index 000000000..c42cb1fb6 --- /dev/null +++ b/internal/anomaly/review_fixes_test.go @@ -0,0 +1,949 @@ +package anomaly + +import ( + "encoding/json" + "math" + "math/rand" + "reflect" + "slices" + "strings" + "testing" + "time" +) + +// Boundaries and paths around the six independent-review findings that the +// reproductions in independent_review_test.go do not cover on their own. + +// ---- periodic candidate search (finding 1) ---- + +// The same alternating pattern with a purely relative tolerance, the jitter +// exactly at tol(P): P=320s, JitterRel=1/32 (exact in binary), so tol(P) is +// exactly 10s and the gaps are 310s and 330s. Seeds below P have a smaller +// tolerance than P itself, so the refined period must be checked with its +// own tolerance, not the seed's or the range's lowest. +func TestPeriodicAlternatingJitterAtARelativeTolerance(t *testing.T) { + rule := experimentalPeriodic() + rule.JitterAbs, rule.JitterRel, rule.MaxJitterFraction = 0, 1.0/32, 1 + cfg := Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}, Periodic: []PeriodicRule{rule}} + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + const period = 320 * time.Second + p := newPeriodicState(rule.HistoryLen) + var sc periodicScratch + var events []Event + for i := 0; i < 32; i++ { + when := time.Duration(i) * period + if i%2 == 1 { + when -= 10 * time.Second + } + events = append(events, relayEvent(t, "pulse-"+string(rune('A'+i)), at(when), 1, 2)) + p.observe(at(when).UnixNano(), TrafficUnclassified, &rule, &sc) + } + ideal := p.chainFor(int64(period), &rule, &sc) + if ideal.tol != int64(10*time.Second) || !ideal.meets(&rule) || ideal.gaps != 31 || p.chance(ideal, &rule) > rule.MaxChance { + t.Fatalf("invalid repro: ideal=%+v chance=%g", ideal, p.chance(ideal, &rule)) + } + cs := run(t, newDet(t, cfg), events) + a := filter(cs, rule.Name, StateActive) + if len(a) == 0 { + t.Fatalf("not detected: period=320s, gaps=310s/330s, relative tolerance 10s") + } + if a[0].Periodic.Period != period { + t.Fatalf("detected period %v, want %v", a[0].Periodic.Period, period) + } +} + +// A refined cell must not hide its seed. Pulse 1 is a stray start 305s +// before a steady 294s train, so the seed 294s explains a clean chain that +// meets every criterion once the train has MinPulses pulses (pulse +// 1+MinPulses, as on ad3ab30f), while the refinement, pulled by the 305s +// gap, explains a longer chain that fails MaxJitterFraction. Both are +// candidates, and the one that meets wins. +func TestRefinementDoesNotHideAMeetingSeedChain(t *testing.T) { + rule := experimentalPeriodic() + p := newPeriodicState(rule.HistoryLen) + var sc periodicScratch + when := 305 * time.Second + first := 0 + for i := 1; i <= 24 && first == 0; i++ { + ts := time.Duration(0) + if i > 1 { + ts = when + time.Duration(i-2)*294*time.Second + } + if _, ok := p.observe(at(ts).UnixNano(), TrafficUnclassified, &rule, &sc); ok { + first = i + } + } + if want := 1 + rule.MinPulses; first != want { + t.Fatalf("first signal at pulse %d, want %d (the first sufficient pulse)", first, want) + } +} + +// A refined candidate replaces the seeds' estimate only once it would +// signal. With a tight MaxChance a refined chain can meet the criteria and +// outrank the seeds while its Chance is still too high; kept as the +// hypothesis, it would stop the search while a seed chain that passes +// MaxChance exists. The gaps are two trains from an independent Monte Carlo +// (rule: JitterAbs 1s, JitterRel 0.03, MaxChance 1e-9); without the Chance +// condition the first signalled 27 pulses late and the second never. +// ad3ab30f's signals are the reference. +func TestRefinedCandidateReplacesOnlyWhenItWouldSignal(t *testing.T) { + for _, c := range []struct { + name string + gaps []int64 // ns + first, count, last int + }{ + {"train 740 (period 4m16s)", []int64{ + 253473301057, 258989191336, 248808871358, 260765027015, 259220274542, 260339684107, + 262600378762, 258738291871, 248944813011, 258606895992, 256467098643, 241784077500, + 257070483133, 258859646680, 250520765238, 257227581530, 249266780471, 252126136834, + 250445492269, 263258476954, 249429701539, 255178245932, 260308587824, 262717898963, + 253196559566, 248241298572, 261418029655, 248770098153, 255363287212, 258491054663, + 258376829668, 249750285464, 260387262917, 257399278175, 263913884834, 254618106104, + 263253087435, 261094534382, 261639049616, 251513354176, 258897024009, 262539896680, + 258308071838, 261975360249, 262945608084, 257866559143, 253898288909, 252200311541, + 252505890454, 248219752799, 260187439605, 256025054915, 255771405888, 263583781023, + 258520832693, 250677715026, 254379206898, 250134628599, 248673262343, 253559894187, + 260685391656, 244190473175, 259409506745, 263956802350, 249234009113, 262085624807, + 252595914023, 268807348304, 263888676328, 257006252683, + }, 10, 14, 59}, + {"train 1240 (period 2m53s)", []int64{ + 174171937807, 173259145459, 174531540411, 340444992491, 168600002187, 172130714765, + 178185967610, 167672548385, 175582782198, 178930825973, 170987808172, 178812895590, + 169727494636, 175341588223, 170041727849, 173205068154, 170345765780, 175075327296, + 175235169903, 169332171444, 188333639312, 167408995031, 168616014872, 176732465155, + 178045458259, 174315631381, 178741743767, 174044112872, 167394578413, 175381847267, + 172583705123, 168739362204, 174321044684, 168217101116, 167035424717, 170618088704, + 165346946332, 176915627713, 176029405703, 173136516273, 168960152899, 175535230297, + 169200234897, 177781560705, 177969402460, 172228240331, 169581467830, 172286294966, + 168458815458, 178829076893, 168522218864, 173637838896, 175170616840, 171330601291, + 174342474654, 169534334907, 175681725697, 173915952191, 187517102381, 174359296774, + 169014698377, 174123854812, 175505387461, 175828494557, 173758510591, 168915340027, + 171752353392, 167303471723, 175039029128, 174776307324, + }, 57, 1, 57}, + } { + t.Run(c.name, func(t *testing.T) { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel, r.MaxChance = time.Second, 0.03, 1e-9 + p := newPeriodicState(r.HistoryLen) + var sc periodicScratch + first, count, last := -1, 0, -1 + var cur int64 + for i, g := range c.gaps { + cur += g + if _, ok := p.observe(t0.UnixNano()+cur, TrafficUnclassified, &r, &sc); ok { + if first < 0 { + first = i + } + count, last = count+1, i + } + } + if first != c.first || count != c.count || last != c.last { + t.Fatalf("signals first=%d count=%d last=%d, want %d %d %d (as on ad3ab30f)", first, count, last, c.first, c.count, c.last) + } + }) + } +} + +// ringOf returns a periodic state holding pulses at the given offsets from t0. +func ringOf(r *PeriodicRule, offsets ...time.Duration) *periodicState { + p := newPeriodicState(r.HistoryLen) + for i, o := range offsets { + p.times[i] = at(o).UnixNano() + } + p.n = len(offsets) + return p +} + +// The refined period is checked with its own tolerance. Here the common +// range, built with the seed's larger relative tolerance, reaches 319.69s, +// but a period there misses even the newest gap (330s) with its own 9.99s +// tolerance, so the cell keeps the seed. +func TestRefinedPeriodNeverExplainsFewerGapsThanTheSeed(t *testing.T) { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel = 0, 1.0/32 + p := ringOf(&r, 0, 400*time.Second, 709700*time.Millisecond, 1039700*time.Millisecond) + var sc periodicScratch + seed := int64(330 * time.Second) + c, ref := p.chainRefined(seed, &r, &sc) + if c.gaps != 1 { + t.Fatalf("fixture: seed explains %d gaps, want 1", c.gaps) + } + if got := p.recentExplained(ref, &r, &sc); got < c.gaps { + t.Fatalf("cell tests %v, which explains %d newest gaps; the seed explains %d", time.Duration(ref), got, c.gaps) + } +} + +// The common range is exact to the nanosecond: its bounds are the ceiling +// of (g-tol)/k and the floor of (g+tol)/k, and the phase estimate is clamped +// into it. With k=2, tol=6s and X=300s+1ns, the gaps allow only P=X, and +// span/slots falls below X (first case) or above it (second case). The cell +// must test exactly X. The oldest gap, 1000s (k=3), ends the range. +func TestRefinementRangeIsExactAndClamped(t *testing.T) { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel = 6*time.Second, 0 + const x = 300*time.Second + 1 + for _, c := range []struct { + name string + gaps []time.Duration // oldest first + seedGap time.Duration // the seed is this gap / 2 + own int + }{ + // 606s+1ns sets the lower bound X (ceil), 594s+2ns the upper bound X; + // span/slots = 300s + {"estimate below the range", []time.Duration{1000 * time.Second, 594*time.Second + 2, 606*time.Second + 1}, 594*time.Second + 2, 0}, + // 594s+3ns sets the upper bound X (floor), 606s+1ns and 606s+2ns keep + // the lower bound X; span/slots = 301s+1ns + {"estimate above the range", []time.Duration{1000 * time.Second, 606*time.Second + 2, 606*time.Second + 1, 594*time.Second + 3}, 594*time.Second + 3, 1}, + } { + t.Run(c.name, func(t *testing.T) { + offs := []time.Duration{0} + for _, g := range c.gaps { + offs = append(offs, offs[len(offs)-1]+g) + } + p := ringOf(&r, offs...) + var sc periodicScratch + chain, ref := p.chainRefined(int64(c.seedGap)/2, &r, &sc) + if chain.gaps != c.own { + t.Fatalf("fixture: the seed explains %d gaps, want %d", chain.gaps, c.own) + } + if ref != int64(x) { + t.Fatalf("refined period %v, want %v", time.Duration(ref), x) + } + if got := p.chainFor(ref, &r, &sc); got.gaps != len(c.gaps)-1 { + t.Fatalf("refined period explains %d gaps, want %d", got.gaps, len(c.gaps)-1) + } + }) + } +} + +// The refinement costs only what its rules say: one pass over the gaps the +// seed's chain or the recentGaps window needs, plus one verification walk +// when (and only when) a refinement is proposed. +func TestRefinementWorkIsExact(t *testing.T) { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel = 6*time.Second, 0 + alternating := func(n int) *periodicState { + var offs []time.Duration + for i := 0; i < n; i++ { + o := time.Duration(i) * 5 * time.Minute + if i%2 == 1 { + o -= 6 * time.Second + } + offs = append(offs, o) + } + return ringOf(&r, offs...) + } + periodic := func(n int) *periodicState { + var offs []time.Duration + for i := 0; i < n; i++ { + offs = append(offs, time.Duration(i)*5*time.Minute) + } + return ringOf(&r, offs...) + } + for _, c := range []struct { + name string + p *periodicState + seed int64 + steps uint64 + refined bool + wantGaps int + wantRange string + }{ + // the seed explains all 16 gaps: no verification walk + {"seed explains every walked gap", periodic(17), int64(5 * time.Minute), 16, false, 16, ""}, + // the range stops after one gap: no refinement, no verification walk + {"range of one gap", ringOf(&r, 0, 1000*time.Second, 1300*time.Second), int64(1000 * time.Second), 2, false, 0, ""}, + // 31 gaps: the range walk stops at the 16-gap window, then one + // verification walk of 16 gaps + {"window of recentGaps", alternating(32), int64(294 * time.Second), recentGaps + recentGaps, true, 1, ""}, + } { + t.Run(c.name, func(t *testing.T) { + var sc periodicScratch + got, ref := c.p.chainRefined(c.seed, &r, &sc) + if sc.steps != c.steps || (ref != c.seed) != c.refined || got.gaps != c.wantGaps { + t.Fatalf("steps %d refined %v gaps %d, want %d %v %d", sc.steps, ref != c.seed, got.gaps, c.steps, c.refined, c.wantGaps) + } + }) + } +} + +// Each distinct period is measured once per search: many seeds of the +// alternating train refine to the same 300s. +func TestRefinedPeriodsAreMeasuredOnce(t *testing.T) { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel, r.MaxJitterFraction = 6*time.Second, 0, 1 + var offs []time.Duration + for i := 0; i < 32; i++ { + o := time.Duration(i) * 5 * time.Minute + if i%2 == 1 { + o -= 6 * time.Second + } + offs = append(offs, o) + } + p := ringOf(&r, offs...) + var sc periodicScratch + best := p.estimate(&r, &sc) + if best.period != int64(5*time.Minute) { + t.Fatalf("fixture: estimate %v, want 5m", time.Duration(best.period)) + } + distinct := map[int64]bool{} + for _, v := range append(append([]int64(nil), sc.seen...), sc.seenRef...) { + distinct[v] = true + } + // one measurement per distinct period, plus at most the final refinement + if n := len(sc.seen) + len(sc.seenRef); n != len(distinct) || sc.chains > uint64(n)+1 { + t.Fatalf("%d chains measured, %d periods recorded, %d distinct", sc.chains, n, len(distinct)) + } +} + +// estimateSeedsOnly is estimate as of ad3ab30f, before refined candidates: +// the reference the search must never fall below. +func estimateSeedsOnly(p *periodicState, r *PeriodicRule, sc *periodicScratch) chainInfo { + var best chainInfo + minP, maxP := int64(r.MinPeriod), int64(r.MaxPeriod) + maxK := r.MaxMissing + 1 + newest := p.at(p.n-1) - p.at(p.n-2) + if newest+tolNanos(r, maxP) < minP-tolNanos(r, minP) || newest > int64(maxK)*maxP+tolNanos(r, maxP) { + return best + } + var seen []int64 + for i := p.n - 1; i >= 1 && i >= p.n-recentGaps; i-- { + g := p.at(i) - p.at(i-1) + for k := 1; k <= maxK; k++ { + cand := g / int64(k) + if cand < minP || cand > maxP || slices.Contains(seen, cand) { + continue + } + seen = append(seen, cand) + if c := p.chainFor(cand, r, sc); c.gaps > 0 && (best.gaps == 0 || c.preferred(best, r)) { + best = c + } + } + } + if best.gaps == 0 { + return best + } + span := p.at(p.n-1) - p.at(p.n-1-best.gaps) + if ref := span / int64(best.slots); ref >= minP && ref <= maxP && ref != best.period { + if c := p.chainFor(ref, r, sc); c.gaps >= best.gaps && c.coverage >= best.coverage && (c.meets(r) || !best.meets(r)) { + best = c + } + } + return best +} + +// Over random rings, rule shapes and both tolerance kinds, the search +// returns exactly what the seeds alone return, or a refined chain that +// would signal (meets the criteria, Chance within MaxChance) and strictly +// outranks it. +func TestEstimateNeverRanksBelowTheSeedsAlone(t *testing.T) { + rng := rand.New(rand.NewSource(11)) + var sc periodicScratch + improved := 0 + for trial := 0; trial < 3000; trial++ { + r := experimentalPeriodic() + r.MaxMissing = rng.Intn(3) + r.MaxJitterFraction = []float64{0.25, 0.5, 1}[rng.Intn(3)] + r.MinCoverage = []float64{0.6, 0.8, 1}[rng.Intn(3)] + if trial%2 == 0 { + r.JitterAbs, r.JitterRel = 6*time.Second, 0 + } else { + r.JitterAbs, r.JitterRel = 0, 1.0/32 + } + base := time.Duration(60+rng.Intn(900)) * time.Second + jitter := time.Duration(1+rng.Intn(8)) * time.Second + offs := []time.Duration{0} + for i := 1; i < r.HistoryLen; i++ { + k := time.Duration(1) + if rng.Intn(10) == 0 { + k = 2 + } + d := k*base + time.Duration(rng.Int63n(int64(2*jitter))) - jitter + if rng.Intn(15) == 0 { + d = time.Duration(rng.Int63n(int64(base))) + 10*time.Second + } + offs = append(offs, offs[i-1]+d) + } + p := ringOf(&r, offs...) + want := estimateSeedsOnly(p, &r, &sc) + got := p.estimate(&r, &sc) + if got != want { + if !got.meets(&r) || p.chance(got, &r) > r.MaxChance || (want.gaps > 0 && !got.outranks(want, &r)) { + t.Fatalf("trial %d: estimate %+v replaced the seeds' %+v without being able to signal and outranking it", trial, got, want) + } + improved++ + } + } + if improved == 0 { + t.Fatal("fixture: refined candidates never changed the estimate") + } + t.Logf("%d of 3000 estimates improved by a refined candidate", improved) +} + +// The seeds are ranked exactly as without refined candidates, even when a +// seed equals a period an earlier refinement already measured: here the +// newest seed (306s) refines to 300s, and the oldest gap in the window is +// exactly 300s. Every distinct in-range gap/k of the window is a seed. +func TestSeedsAreRankedAsWithoutRefinement(t *testing.T) { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel, r.MaxJitterFraction = 6*time.Second, 0, 1 + offs := []time.Duration{0, 300 * time.Second} + for j := 2; j <= recentGaps; j++ { + g := 294 * time.Second + if j%2 == 0 { + g = 306 * time.Second + } + offs = append(offs, offs[len(offs)-1]+g) + } + p := ringOf(&r, offs...) + var sc periodicScratch + p.estimate(&r, &sc) + want := map[int64]bool{} + for i := p.n - 1; i >= 1 && i >= p.n-recentGaps; i-- { + g := p.at(i) - p.at(i-1) + for k := 1; k <= r.MaxMissing+1; k++ { + if s := g / int64(k); s >= int64(r.MinPeriod) && s <= int64(r.MaxPeriod) { + want[s] = true + } + } + } + if !slices.Contains(sc.seenRef, int64(300*time.Second)) { + t.Fatalf("fixture: 300s was not measured as a refined period (refined %v)", sc.seenRef) + } + got := map[int64]bool{} + for _, s := range sc.seen { + got[s] = true + } + if len(sc.seen) != len(want) || !reflect.DeepEqual(got, want) { + t.Fatalf("seed pool %d values %v, want the %d distinct seeds %v", len(sc.seen), sc.seen, len(want), want) + } +} + +// Over random rings and both tolerance kinds: the fused pass measures the +// seed's chain exactly as chainFor does, and the period a cell tests never +// explains fewer of the newest gaps than its seed. +func TestChainRefinedMatchesChainForAndNeverLosesGaps(t *testing.T) { + rng := rand.New(rand.NewSource(7)) + abs := experimentalPeriodic() + abs.JitterAbs, abs.JitterRel, abs.MaxMissing = 6*time.Second, 0, 2 + rel := experimentalPeriodic() + rel.JitterAbs, rel.JitterRel, rel.MaxMissing = 0, 1.0/32, 2 + var sc periodicScratch + refined := 0 + for trial := 0; trial < 2000; trial++ { + r := abs + if trial%2 == 1 { + r = rel + } + base := time.Duration(60+rng.Intn(600)) * time.Second + offs := []time.Duration{0} + for i := 1; i < r.HistoryLen; i++ { + k := time.Duration(1 + rng.Intn(2)) + jit := time.Duration(rng.Int63n(int64(12*time.Second))) - 6*time.Second + if rng.Intn(8) == 0 { + jit = time.Duration(rng.Int63n(int64(base))) + } + offs = append(offs, offs[i-1]+k*base+jit) + } + p := ringOf(&r, offs...) + for i := p.n - 1; i >= p.n-recentGaps; i-- { + g := p.at(i) - p.at(i-1) + for k := 1; k <= r.MaxMissing+1; k++ { + seed := g / int64(k) + if seed < int64(r.MinPeriod) || seed > int64(r.MaxPeriod) { + continue + } + c, ref := p.chainRefined(seed, &r, &sc) + if want := p.chainFor(seed, &r, &sc); c != want { + t.Fatalf("trial %d seed %v: fused chain %+v, chainFor %+v", trial, time.Duration(seed), c, want) + } + if ref != seed { + refined++ + if got, own := p.recentExplained(ref, &r, &sc), p.recentExplained(seed, &r, &sc); got <= own { + t.Fatalf("trial %d seed %v: refined %v explains %d newest gaps, seed %d", trial, time.Duration(seed), time.Duration(ref), got, own) + } + } + } + } + } + if refined == 0 { + t.Fatal("fixture: no cell was refined") + } + t.Logf("%d refined cells", refined) +} + +// ---- dedup horizon (finding 2) ---- + +func dedupDet(t *testing.T, horizon, reorder time.Duration) *Detector { + t.Helper() + lim := experimentalLimits() + lim.DedupHorizon, lim.ReorderDelay = horizon, reorder + return newDet(t, Config{Limits: lim, Rate: []RateRule{rateRule("r", ScopeGlobal, time.Minute, 1)}}) +} + +func observeStatus(t *testing.T, d *Detector, id string, when time.Time) Status { + t.Helper() + return d.Observe(relayEvent(t, id, when, 1, 2)).Status +} + +// A repeat within the horizon (inclusive) is a duplicate; one ns beyond it +// counts again, with or without another ID expiring the first copy. +func TestDedupHorizonBoundaryIsInclusive(t *testing.T) { + const h = 2 * time.Minute + for _, c := range []struct { + name string + after time.Duration + want Status + }{ + {"one ns before", h - time.Nanosecond, StatusDuplicate}, + {"at the horizon", h, StatusDuplicate}, + {"one ns after", h + time.Nanosecond, StatusAccepted}, + } { + for _, other := range []bool{false, true} { + name := c.name + if other { + name += " after another ID" + } + t.Run(name, func(t *testing.T) { + d := dedupDet(t, h, 0) + if got := observeStatus(t, d, "same", t0); got != StatusAccepted { + t.Fatal(got) + } + if other { + if got := observeStatus(t, d, "other", at(c.after)); got != StatusAccepted { + t.Fatal(got) + } + } + if got := observeStatus(t, d, "same", at(c.after)); got != c.want { + t.Fatalf("repeat %v after the first copy: %v, want %v", c.after, got, c.want) + } + }) + } + } +} + +// An older copy arriving out of order stays a duplicate and neither deletes +// nor revives the newer remembered copy. +func TestDedupOlderCopyOutOfOrderKeepsTheNewerCopy(t *testing.T) { + const h, rd = 2 * time.Minute, 30 * time.Second + d := dedupDet(t, h, rd) + if got := observeStatus(t, d, "same", at(3*time.Minute)); got != StatusAccepted { + t.Fatal(got) + } + if got := observeStatus(t, d, "same", at(2*time.Minute+45*time.Second)); got != StatusDuplicate { + t.Fatalf("older copy within the reorder delay: %v, want duplicate", got) + } + // still measured from the newer copy at 3m + if got := observeStatus(t, d, "same", at(3*time.Minute+h)); got != StatusDuplicate { + t.Fatalf("repeat at the horizon of the newer copy: %v, want duplicate", got) + } + if got := observeStatus(t, d, "same", at(3*time.Minute+h+time.Nanosecond)); got != StatusAccepted { + t.Fatalf("repeat beyond the horizon of the newer copy: %v, want accepted", got) + } + if s := d.Snapshot(); s.Accepted != 2 || s.Duplicates != 2 { + t.Fatalf("accepted=%d duplicates=%d, want 2 and 2", s.Accepted, s.Duplicates) + } +} + +// ---- automatic history start (finding 3) ---- + +func historyCfg(reorder time.Duration, buffer int) Config { + lim := experimentalLimits() + lim.ReorderDelay, lim.MaxReorderBuffer = reorder, buffer + return Config{Limits: lim, NewStream: []NewStreamRule{{Name: "ns", Scope: ScopeStream, Window: time.Minute, MinCount: 1, + CensorWindow: 10 * time.Second, QuietPeriod: 10 * time.Second, State: quickState()}}} +} + +// censoredByTrigger returns, per trigger ID, whether its new-stream +// activation was left-censored. +func censoredByTrigger(cs []Candidate) map[TxID]bool { + out := map[TxID]bool{} + for _, c := range filter(cs, "ns", StateActive) { + out[c.TriggerID] = c.NewStream.Censored + } + return out +} + +// Whatever releases the first event from the reorder buffer (a later +// Observe, Advance, Flush or a forced release), the automatic history start +// is that event's time, so the arrival order within the delay does not +// change the result. +func TestAutoHistoryStartIsTheFirstProcessedEventOnEveryReleasePath(t *testing.T) { + early := func(t *testing.T) Event { return relayEvent(t, "early", t0, 1, 2) } + later := func(t *testing.T) Event { return relayEvent(t, "later", at(20*time.Second), 1, 3) } + want := map[TxID]bool{"early": true, "later": false} + for _, c := range []struct { + name string + feed func(t *testing.T, d *Detector) []Candidate + }{ + {"watermark", func(t *testing.T, d *Detector) []Candidate { + out := d.Observe(later(t)).Candidates + out = append(out, d.Observe(early(t)).Candidates...) + return append(out, d.Advance(at(20*time.Second+30*time.Second)).Candidates...) + }}, + {"flush", func(t *testing.T, d *Detector) []Candidate { + out := d.Observe(later(t)).Candidates + out = append(out, d.Observe(early(t)).Candidates...) + return append(out, d.Flush().Candidates...) + }}, + {"later observe", func(t *testing.T, d *Detector) []Candidate { + out := d.Observe(later(t)).Candidates + out = append(out, d.Observe(early(t)).Candidates...) + return append(out, d.Observe(relayEvent(t, "far", at(10*time.Minute), 1, 4)).Candidates...) + }}, + } { + t.Run(c.name, func(t *testing.T) { + got := censoredByTrigger(c.feed(t, newDet(t, historyCfg(30*time.Second, 1024)))) + for id, cens := range want { + if g, ok := got[id]; !ok || g != cens { + t.Fatalf("%s: censored=%v present=%v, want censored=%v (all: %v)", id, g, ok, cens, got) + } + } + }) + } + // forced release: the buffer holds one event, so the second Observe forces + // the oldest (early) out first + t.Run("forced release", func(t *testing.T) { + d := newDet(t, historyCfg(30*time.Second, 1)) + out := d.Observe(later(t)).Candidates + out = append(out, d.Observe(early(t)).Candidates...) + out = append(out, d.Flush().Candidates...) + if got := censoredByTrigger(out); got["later"] { + t.Fatalf("later stream censored after a forced release: %v", got) + } + }) +} + +// An explicit HistoryStart is used as given, never replaced by the first +// processed event. +func TestExplicitHistoryStartIsKept(t *testing.T) { + for _, c := range []struct { + name string + start time.Time + want map[TxID]bool + }{ + {"well before the data", at(-time.Hour), map[TxID]bool{"early": false, "later": false}}, + {"between the events", at(15 * time.Second), map[TxID]bool{"later": true}}, + } { + t.Run(c.name, func(t *testing.T) { + cfg := historyCfg(30*time.Second, 1024) + cfg.HistoryStart = c.start + evs := []Event{relayEvent(t, "later", at(20*time.Second), 1, 3)} + if c.start.Before(t0) { + evs = append(evs, relayEvent(t, "early", t0, 1, 2)) + } + got := censoredByTrigger(run(t, newDet(t, cfg), evs)) + if !reflect.DeepEqual(got, c.want) { + t.Fatalf("censored %v, want %v", got, c.want) + } + }) + } +} + +// End of replay input: sorted and reversed lines within the reorder delay +// give the same report. Stats.EffectiveDedupHorizon is left out: it is +// measured from the dedup log in acceptance order and so reads up to +// ReorderDelay low for out-of-order input (unchanged since ad3ab30f). +func TestReplayResultDoesNotDependOnOrderWithinTheDelay(t *testing.T) { + cfg := historyCfg(30*time.Second, 1024) + cfg.Expected.Mode = ExpectedInclude + lines := []string{replayLineAt("early", "2030-01-07T12:00:00Z", "02"), replayLineAt("later", "2030-01-07T12:00:20Z", "03")} + opt := ReplayOptions{Start: t0, End: at(time.Hour), MaxLineBytes: 4096} + var reports []string + for _, in := range [][]string{lines, {lines[1], lines[0]}} { + rep, err := Replay(strings.NewReader(strings.Join(in, "\n")), cfg, opt) + if err != nil { + t.Fatal(err) + } + if len(rep.Candidates) == 0 { + t.Fatal("fixture: no candidates") + } + rep.Stats.EffectiveDedupHorizon = 0 + b, err := json.Marshal(rep) + if err != nil { + t.Fatal(err) + } + reports = append(reports, string(b)) + } + if reports[0] != reports[1] { + t.Fatalf("sorted and reversed input differ:\n%s\n%s", reports[0], reports[1]) + } +} + +// ---- config ownership (finding 4) ---- + +func payloadCfg(types []PayloadType) Config { + per := experimentalPeriodic() + per.PayloadTypes = types + return Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}, + HistoryStart: at(-time.Hour), + Rate: []RateRule{{Name: "r", Scope: ScopeGlobal, Window: time.Minute, Threshold: 1, PayloadTypes: types, State: quickState()}}, + NewStream: []NewStreamRule{{Name: "ns", Scope: ScopeStream, Window: time.Minute, MinCount: 1, + CensorWindow: 10 * time.Second, QuietPeriod: 10 * time.Second, PayloadTypes: types, State: quickState()}}, + Periodic: []PeriodicRule{per}} +} + +// New copies every PayloadTypes slice: nil stays nil, empty stays empty, and +// a non-empty filter never shares its backing array with the caller. +func TestNewCopiesEveryPayloadFilter(t *testing.T) { + for _, c := range []struct { + name string + types []PayloadType + }{{"nil", nil}, {"empty", make([]PayloadType, 0, 4)}, {"nonempty", []PayloadType{5, 6}}} { + t.Run(c.name, func(t *testing.T) { + cfg := payloadCfg(c.types) + d, err := New(cfg) + if err != nil { + t.Fatal(err) + } + got := map[string][2][]PayloadType{ + "rate": {cfg.Rate[0].PayloadTypes, d.cfg.Rate[0].PayloadTypes}, + "newstream": {cfg.NewStream[0].PayloadTypes, d.cfg.NewStream[0].PayloadTypes}, + "periodic": {cfg.Periodic[0].PayloadTypes, d.cfg.Periodic[0].PayloadTypes}, + } + for kind, p := range got { + caller, own := p[0], p[1] + if (caller == nil) != (own == nil) || !reflect.DeepEqual(caller, own) { + t.Errorf("%s: copy %v (nil=%v), caller %v (nil=%v)", kind, own, own == nil, caller, caller == nil) + } + if cap(caller) > 0 && cap(own) > 0 && &caller[:1][0] == &own[:1][0] { + t.Errorf("%s: PayloadTypes shares its backing array with the caller", kind) + } + } + }) + } +} + +// After New, changing the caller's filters changes nothing for any rule +// kind, and running the detector leaves the caller's config as it was. +func TestCallerConfigAndDetectorAreIndependent(t *testing.T) { + cfg := payloadCfg([]PayloadType{5}) + before := payloadCfg([]PayloadType{5}) + d, err := New(cfg) + if err != nil { + t.Fatal(err) + } + cfg.Rate[0].PayloadTypes[0] = 4 + cfg.NewStream[0].PayloadTypes[0] = 4 + cfg.Periodic[0].PayloadTypes[0] = 4 + var evs []Event + for i := 0; i < experimentalPeriodic().MinPulses; i++ { + evs = append(evs, relayEvent(t, "p"+string(rune('a'+i)), at(time.Duration(i)*5*time.Minute), 1, 2)) + } + cs := run(t, d, evs) + for _, rule := range []string{"r", "ns", "per"} { + if len(filter(cs, rule, StateActive)) == 0 { + t.Errorf("rule %s: no activation after the caller changed its filter:\n%s", rule, describe(cs)) + } + } + cfg.Rate[0].PayloadTypes[0] = 5 + cfg.NewStream[0].PayloadTypes[0] = 5 + cfg.Periodic[0].PayloadTypes[0] = 5 + if !reflect.DeepEqual(cfg, before) { + t.Fatalf("the detector changed the caller's config:\n got %+v\nwant %+v", cfg, before) + } +} + +// ---- replay route invariants (finding 5) ---- + +// Route and first hop parse with NewRoute's invariants: width 0 only with no +// bytes, the width matching the bytes, hex only; nothing is dropped silently. +func TestReplayRouteWidthInvariants(t *testing.T) { + for _, c := range []struct { + name string + kind, hop, full string + ok bool + hopW, fullW int + }{ + {"route width 0 with bytes", "relay", "1:5c", "0:deadbeef", false, 0, 0}, + {"hop width 0 with bytes (relay)", "relay", "0:5c", "", false, 0, 0}, + {"hop width 0 with bytes (nopath)", "nopath", "0:5c", "", false, 0, 0}, + {"hop width without bytes", "relay", "1:", "", false, 0, 0}, + {"route width without bytes", "relay", "1:5c", "2:", false, 0, 0}, + {"hop bytes without width", "relay", ":5c", "", false, 0, 0}, + {"route bytes without width", "relay", "1:5c", ":5c00", false, 0, 0}, + {"hop invalid hex", "relay", "1:zz", "", false, 0, 0}, + {"route invalid hex", "relay", "1:5c", "1:5cz0", false, 0, 0}, + {"hop length does not match width", "relay", "2:5c", "", false, 0, 0}, + {"route length not a multiple of width", "relay", "2:5c01", "2:5c01aa", false, 0, 0}, + {"hop wider than MaxHopWidth", "relay", "4:5c010203", "", false, 0, 0}, + {"empty route (\"\")", "relay", "1:5c", "", true, 1, 0}, + {"empty route (\"-\")", "relay", "1:5c", "-", true, 1, 0}, + {"empty route (\"0:\")", "relay", "1:5c", "0:", true, 1, 0}, + {"empty hop (\"0:\", nopath)", "nopath", "0:", "", true, 0, 0}, + {"1-byte route", "relay", "1:5c", "1:5c11", true, 1, 1}, + {"2-byte route", "relay", "2:5c01", "2:5c01aa02", true, 2, 2}, + {"3-byte route", "relay", "3:5c0102", "3:5c0102aabbcc", true, 3, 3}, + } { + t.Run(c.name, func(t *testing.T) { + var rec ReplayRecord + if err := json.Unmarshal([]byte(replayLine("x")), &rec); err != nil { + t.Fatal(err) + } + rec.RouteKind, rec.FirstHop, rec.FullRoute = c.kind, c.hop, c.full + e, err := rec.Event() + if c.ok != (err == nil) { + t.Fatalf("first_hop %q full_route %q: err %v, want valid=%v", c.hop, c.full, err, c.ok) + } + if c.ok && (e.FirstHop.Width() != c.hopW || e.FullRoute.Width() != c.fullW) { + t.Fatalf("widths hop=%d route=%d, want %d and %d", e.FirstHop.Width(), e.FullRoute.Width(), c.hopW, c.fullW) + } + }) + } +} + +// A replay line whose first hop has width 0 but bytes is counted invalid. +func TestReplayRejectsZeroWidthFirstHopWithBytes(t *testing.T) { + var rec ReplayRecord + if err := json.Unmarshal([]byte(replayLine("x")), &rec); err != nil { + t.Fatal(err) + } + rec.RouteKind, rec.FirstHop = "nopath", "0:5c" + line, err := json.Marshal(rec) + if err != nil { + t.Fatal(err) + } + rep, err := Replay(strings.NewReader(string(line)), replayFixtureConfig(t), ReplayOptions{Start: t0, End: at(time.Hour), MaxLineBytes: 4096}) + if err != nil { + t.Fatal(err) + } + if rep.Input.Accepted != 0 || rep.Input.Invalid != 1 { + t.Fatalf("first hop with width 0 and bytes: %+v", rep.Input) + } +} + +// ---- BurstGap validation (finding 6) ---- + +// BurstGap must stay below MinPeriod/2 for every value, with no overflow. +func TestBurstGapValidationBoundaries(t *testing.T) { + const m = time.Minute + for _, c := range []struct { + name string + minPeriod time.Duration + gap time.Duration + ok bool + }{ + {"zero", m, 0, true}, + {"largest allowed (even MinPeriod)", m, m/2 - 1, true}, + {"half (even MinPeriod)", m, m / 2, false}, + {"half plus 1ns (even MinPeriod)", m, m/2 + 1, false}, + {"largest allowed (odd MinPeriod)", m + 1, m / 2, true}, + {"one ns over (odd MinPeriod)", m + 1, m/2 + 1, false}, + {"MaxInt64/2 (2x fits)", m, math.MaxInt64 / 2, false}, + {"MaxInt64/2+1 (2x overflows)", m, math.MaxInt64/2 + 1, false}, + {"MaxInt64", m, math.MaxInt64, false}, + {"negative", m, -1, false}, + } { + t.Run(c.name, func(t *testing.T) { + r := experimentalPeriodic() + r.MinPeriod, r.BurstGap = c.minPeriod, c.gap + cfg := Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}, Periodic: []PeriodicRule{r}} + if err := cfg.Validate(); (err == nil) != c.ok { + t.Fatalf("BurstGap %d with MinPeriod %d: err %v, want valid=%v", c.gap, c.minPeriod, err, c.ok) + } + }) + } +} + +// ---- fuzz targets for the changed parsing and search ---- + +// Any first_hop / full_route text either is rejected or yields exactly the +// width and bytes written: nothing is dropped or normalised silently. +func FuzzReplayRouteParsing(f *testing.F) { + for _, s := range [][3]string{{"relay", "1:5c", "1:5c11"}, {"relay", "1:5c", "0:deadbeef"}, + {"nopath", "0:5c", ""}, {"relay", "2:5c01", "2:5c01aa"}, {"relay", "3:5c0102", "-"}, {"nopath", ":", "0:"}} { + f.Add(s[0], s[1], s[2]) + } + f.Fuzz(func(t *testing.T, kind, hop, full string) { + var rec ReplayRecord + if err := json.Unmarshal([]byte(replayLine("x")), &rec); err != nil { + t.Fatal(err) + } + rec.RouteKind, rec.FirstHop, rec.FullRoute = kind, hop, full + e, err := rec.Event() + if err != nil { + return + } + for _, c := range []struct { + text string + width int + bytes []byte + }{{hop, e.FirstHop.Width(), e.FirstHop.Bytes()}, {full, e.FullRoute.Width(), e.FullRoute.Bytes()}} { + w, b, perr := parseWidthHex(c.text) + if perr != nil { + t.Fatalf("%q parses with error %v but the event was accepted", c.text, perr) + } + if len(b) == 0 { + if c.width != 0 || w != 0 { + t.Fatalf("%q: empty bytes, width %d (parsed %d)", c.text, c.width, w) + } + } else if c.width != w || string(c.bytes) != string(b) { + t.Fatalf("%q: event has width %d bytes %x, text says %d %x", c.text, c.width, c.bytes, w, b) + } + } + }) +} + +// Arbitrary pulse gaps and rule parameters: every pulse stays within +// periodicWorkBound, the fused pass equals chainFor, a refined cell never +// explains fewer of the newest gaps than its seed, and the search never +// ranks below the seeds alone. +func FuzzPeriodicSearch(f *testing.F) { + f.Add(uint8(2), uint8(6), uint8(0), []byte{0, 12, 0, 12, 0, 12, 0, 12, 0, 12, 0, 12, 0, 12, 0, 12, 0, 12}) + f.Add(uint8(8), uint8(0), uint8(8), []byte{5, 200, 40, 7, 90, 3, 250, 17, 64, 64, 64, 1, 2, 3}) + f.Fuzz(func(t *testing.T, missing, jitAbs, jitRel uint8, gaps []byte) { + r := experimentalPeriodic() + r.MaxMissing = int(missing % (maxMissingLimit + 1)) + r.JitterAbs = time.Duration(jitAbs%30) * time.Second + r.JitterRel = float64(jitRel%9) / 100 + r.MaxJitterFraction = 1 + cfg := Config{Limits: experimentalLimits(), Expected: ExpectedPolicy{Mode: ExpectedInclude}, Periodic: []PeriodicRule{r}} + if cfg.Validate() != nil { + return + } + maxC, maxS := periodicWorkBound(&r) + p := newPeriodicState(r.HistoryLen) + var sc periodicScratch + now := t0.UnixNano() + for i, b := range gaps { + if i >= 4*r.HistoryLen { + break + } + // gaps around 5 minutes: base 240s plus 0..127s in 1s steps, or up + // to 4x for missing pulses + now += int64(240*time.Second) + int64(b&0x7f)*int64(time.Second) + if b&0x80 != 0 { + now += int64(b&0x3) * int64(5*time.Minute) + } + c0, s0 := sc.chains, sc.steps + p.observe(now, TrafficUnclassified, &r, &sc) + if dc, ds := sc.chains-c0, sc.steps-s0; dc > maxC || ds > maxS { + t.Fatalf("pulse %d: %d chains, %d steps, bound %d and %d", i, dc, ds, maxC, maxS) + } + } + for i := p.n - 1; i >= 1 && i >= p.n-recentGaps; i-- { + g := p.at(i) - p.at(i-1) + for k := 1; k <= r.MaxMissing+1; k++ { + seed := g / int64(k) + if seed < int64(r.MinPeriod) || seed > int64(r.MaxPeriod) { + continue + } + c, ref := p.chainRefined(seed, &r, &sc) + if want := p.chainFor(seed, &r, &sc); c != want { + t.Fatalf("seed %v: fused %+v, chainFor %+v", time.Duration(seed), c, want) + } + if ref != seed && p.recentExplained(ref, &r, &sc) <= p.recentExplained(seed, &r, &sc) { + t.Fatalf("seed %v refined to %v without explaining more newest gaps", time.Duration(seed), time.Duration(ref)) + } + } + } + if p.n >= 2 { + want, got := estimateSeedsOnly(p, &r, &sc), p.estimate(&r, &sc) + if got != want && (!got.meets(&r) || p.chance(got, &r) > r.MaxChance || (want.gaps > 0 && !got.outranks(want, &r))) { + t.Fatalf("estimate %+v replaced the seeds' %+v without being able to signal and outranking it", got, want) + } + } + }) +} From 0d5995d40e4b0ad9e6732407ec33dc875c3657d8 Mon Sep 17 00:00:00 2001 From: dborup Date: Sat, 26 Sep 2026 10:42:16 +0200 Subject: [PATCH 8/9] fix(anomaly): charge refined candidates in the union bound; tests for surviving mutants searchCells now counts the refined periods as cells of their own: 2*recentGaps*(MaxMissing+1)+1 (seeds, at most one refined period per seed, the phase refinement). A common period of m gaps exists exactly when the intersection's highest lower end (g_j-tol)/k_j explains the other m-1 gaps, so the refined fits are covered by as many cells as there are seeds, each spending one gap. The earlier comment claiming the seeds' cells covered them is removed. Every Chance value doubles (x97/49 for the fixture rule); the four golden Chance values change, nothing else in the golden file does. Tests: the cells cover every period estimate measures; null false alarms on 10,000 fixed Poisson trains are no higher than ad3ab30f's; the range rounds the multiple of a missing-pulse gap (M12) and ends at a gap beyond MaxMissing (M13); a refined period equal to a seed is not measured again (M14); an older copy beyond the horizon stays a duplicate (M18). The work-bound comment no longer claims to catch CPU regressions it cannot count; the single Chance gate for refined candidates is documented as deliberate. Under nulls more regular than Poisson the refined periods still raise false alarms (documented). Co-Authored-By: Claude Opus 5.5 --- internal/anomaly/chance_test.go | 14 +- internal/anomaly/periodic.go | 48 ++++-- internal/anomaly/periodic_bench_test.go | 6 +- internal/anomaly/review_fixes_test.go | 161 +++++++++++++++++++ internal/anomaly/testdata/replay_golden.json | 8 +- 5 files changed, 214 insertions(+), 23 deletions(-) diff --git a/internal/anomaly/chance_test.go b/internal/anomaly/chance_test.go index d32c85cd9..9ab1bd6ac 100644 --- a/internal/anomaly/chance_test.go +++ b/internal/anomaly/chance_test.go @@ -121,9 +121,9 @@ func TestChanceIsAccurateForSmallTolerances(t *testing.T) { // value, far outside the measured cross-architecture spread (about 5e-13). func TestMaxChanceDecisionAroundTheBoundary(t *testing.T) { const ( - canonical = 4.59387e-08 // the value rounded to ChanceDigits - justBelow = 4.5938728007910717e-08 // value * (1 - 1e-10) - justAbove = 4.5938728017098456e-08 // value * (1 + 1e-10) + canonical = 9.09399e-08 // the value rounded to ChanceDigits + justBelow = 9.0939930954435484e-08 // value * (1 - 1e-10) + justAbove = 9.0939930972623478e-08 // value * (1 + 1e-10) ) signalAt := func(maxChance float64) (PeriodicEvidence, bool) { r := experimentalPeriodic() @@ -240,7 +240,9 @@ func TestPeriodicEvidenceJSONIsCanonical(t *testing.T) { // "Chance" member removed (numbers kept verbatim, keys sorted), taken from // the golden file before Chance was canonicalized. Timestamps, transitions, // reason codes, coverage, counters and the number of candidates are therefore -// unchanged; only the documented Chance values changed. +// unchanged; only the documented Chance values changed: rounded to +// ChanceDigits, then multiplied by 97/49 (searchCells of the fixture rule) +// when the union bound began to charge the refined candidates. const goldenWithoutChance = "2538014b83e1976a614da59c3db2eeb8d403ba76de63f3f58d55aad221eaf959" func stripChance(v interface{}) interface{} { @@ -294,8 +296,8 @@ func TestGoldenChangedOnlyInTheDocumentedChanceValues(t *testing.T) { for _, c := range chances { n[c]++ } - if len(chances) != 4 || n["4.59387e-8"] != 2 || n["0.000241965"] != 2 { - t.Fatalf("golden Chance values %v, want 4.59387e-8 x2 and 0.000241965 x2", chances) + if len(chances) != 4 || n["9.09399e-8"] != 2 || n["0.000478992"] != 2 { + t.Fatalf("golden Chance values %v, want 9.09399e-8 x2 and 0.000478992 x2", chances) } b, err := json.Marshal(stripChance(decode())) if err != nil { diff --git a/internal/anomaly/periodic.go b/internal/anomaly/periodic.go index 74e00df4f..7f4a9f0e1 100644 --- a/internal/anomaly/periodic.go +++ b/internal/anomaly/periodic.go @@ -42,8 +42,9 @@ import ( // where p_gap is the probability that one exponential gap falls within tol // of some k*P. The exponent drops one gap because P is fitted to the chain // (the gap it was derived from matches by construction). cells is the -// number of candidate periods the search may try per pulse and evaluated is -// the number of pulses tested on the key so far: a union bound over the +// number of candidate periods the search may report per pulse (seeds, +// refined periods and the phase refinement; see searchCells) and evaluated +// is the number of pulses tested on the key so far: a union bound over the // period search and over every pulse that was another chance to fire. // recentGaps is how many of the newest gaps seed the period search. @@ -405,7 +406,10 @@ func (p *periodicState) estimate(r *PeriodicRule, sc *periodicScratch) chainInfo // Chance is within MaxChance): until one does, the hypothesis kept from // pulse to pulse stays the one the seeds give, and a kept hypothesis // that meets the criteria stops the search, so it must not be one that - // cannot signal yet + // cannot signal yet. Only the best-ranked refined candidate is tried; a + // lower-ranked one that would pass MaxChance is deliberately not taken + // instead, so the estimate is always the best-ranked chain and never the + // one picked because it is the most likely to fire. if bestRef.meets(r) && p.chance(bestRef, r) <= r.MaxChance && (best.gaps == 0 || bestRef.outranks(best, r)) { best = bestRef } @@ -427,11 +431,9 @@ func (p *periodicState) estimate(r *PeriodicRule, sc *periodicScratch) chainInfo // measures as a refined candidate. // // A refined period is fitted to up to recentGaps gaps, not one, so a chance -// chain fits it more often than pg^(gaps-1) alone says. The union bound in -// chance() still covers it: a period explaining n of the newest gaps exists -// roughly as often as one of those n gaps' own seeds explains the others, -// and searchCells already charges each of those seeds. So searchCells, and -// the Chance values, are unchanged. +// chain fits it up to that many times more often than one seed; it is a +// hypothesis of its own, and searchCells charges a cell for it (see there +// for why one gap per cell is still what the fit costs). // // Cost: the chain pass as before, the range adds O(1) per gap in the window; // a refined cell adds one walk of recentGaps gaps and one chainFor. No @@ -553,6 +555,30 @@ func (p *periodicState) chance(c chainInfo, r *PeriodicRule) float64 { return float64(p.evaluated) * float64(searchCells(r)) * math.Pow(pg, float64(c.gaps-1)) } -// searchCells is the number of candidate periods estimate may try per pulse: -// recentGaps gaps times MaxMissing+1 divisors, plus the refinement. -func searchCells(r *PeriodicRule) int { return recentGaps*(r.MaxMissing+1) + 1 } +// searchCells is the number of candidate periods the search may report per +// pulse, each one cell of the union bound: a seed g/k for each of the +// recentGaps newest gaps and k = 1..MaxMissing+1, at most one refined period +// per seed, and the phase refinement of the best seed. (A kept hypothesis +// reports only when the search does not run, with at most MaxMissing+1 +// periods: itself and its multiples.) +// +// Each cell spends one gap on its fit: chance() uses pg^(gaps-1). For a seed +// that is the gap it is taken from. A refined period lies in the common +// range of the m newest gaps it explains, the intersection of their ranges +// [(g-tol)/k, (g+tol)/k]. That intersection is non-empty exactly when its +// highest lower end, L = (g_j-tol)/k_j of one gap j, lies in every other +// range, that is when the period L explains each of the other m-1 gaps. So +// "some period explains these m gaps" is the union, over the +// recentGaps*(MaxMissing+1) pairs (j, k_j), of "(g_j-tol)/k_j explains the +// other m-1 gaps", each with one gap spent: as many cells as there are +// seeds. For one set of multiples, a common period exists up to m times as +// often as one given seed explains the other gaps, so the refined periods +// cannot share the seeds' cells. Gaps older than the window are tested +// against a period the window fixes, one independent chance pg each. As +// everywhere in this bound, pg is taken at the reported period. +// +// All of this assumes the Poisson null. For traffic more regular than +// Poisson (gaps spread evenly around their mean) Chance is not a bound, and +// the fitted refined periods find more of its chance chains than the seeds +// alone. +func searchCells(r *PeriodicRule) int { return 2*recentGaps*(r.MaxMissing+1) + 1 } diff --git a/internal/anomaly/periodic_bench_test.go b/internal/anomaly/periodic_bench_test.go index e3a4a46f9..ebeff832e 100644 --- a/internal/anomaly/periodic_bench_test.go +++ b/internal/anomaly/periodic_bench_test.go @@ -119,8 +119,10 @@ func periodicWorkBound(r *PeriodicRule) (chains, steps uint64) { // measuredSteps is the total number of gaps visited by the search over the // deterministic runs of TestPeriodicSearchWorkIsBounded (4*maxHistoryLen // pulses per shape). The test allows 25% above it, so a change that makes -// the search markedly more expensive fails here even though it stays -// within the loose analytic periodicWorkBound. +// the search visit markedly more chains or gaps fails here even though it +// stays within the loose analytic periodicWorkBound. Work these counters do +// not count (more CPU per visited gap, say) is not caught; only +// BenchmarkPeriodicSearch measures it. var measuredSteps = map[string]uint64{ "periodic/0.6": 236631, "alternating/0.6": 1434346, "jitter/0.6": 246825, "missing/0.6": 243414, "bursty/0.6": 196561, "noise/0.6": 144778, diff --git a/internal/anomaly/review_fixes_test.go b/internal/anomaly/review_fixes_test.go index c42cb1fb6..e2d2928de 100644 --- a/internal/anomaly/review_fixes_test.go +++ b/internal/anomaly/review_fixes_test.go @@ -6,6 +6,7 @@ import ( "math/rand" "reflect" "slices" + "strconv" "strings" "testing" "time" @@ -474,6 +475,148 @@ func TestChainRefinedMatchesChainForAndNeverLosesGaps(t *testing.T) { t.Logf("%d refined cells", refined) } +// ---- union bound over the refined candidates (cloud review of 5b816f66) ---- + +// scenarioA is the loose rule of the cloud review's scenario a): a wide +// absolute tolerance, no missing pulses and weak shape criteria, so under a +// null only the Chance condition stops chance chains. +func scenarioA(maxChance float64) PeriodicRule { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel, r.MaxJitterFraction, r.MinPulses, r.MinCoverage, r.MaxMissing = 20*time.Second, 0, 1, 4, 0.1, 0 + r.MaxChance = maxChance + return r +} + +// poissonTrain returns 64 pulse offsets of a Poisson train (mean gap 2m to +// 22m, pulses at least 10s apart), generated in full from its seed. +func poissonTrain(seed int64) []time.Duration { + rng := rand.New(rand.NewSource(seed)) + mean := time.Duration(120+rng.Intn(1200)) * time.Second + out := make([]time.Duration, 0, 64) + ts := time.Duration(0) + for len(out) < 64 { + ts += 10*time.Second + time.Duration(rng.ExpFloat64()*float64(mean)) + out = append(out, ts) + } + return out +} + +// Chance is a union bound over every period the search can report, so the +// cells it charges must cover every period estimate measures: up to +// recentGaps*(MaxMissing+1) seeds, as many refined periods and the phase +// refinement. Some of these trains need more than the seeds and the phase +// refinement alone. +func TestSearchCellsCoverEveryPeriodTheSearchMeasures(t *testing.T) { + b := experimentalPeriodic() + b.JitterRel, b.MaxJitterFraction, b.MinPulses, b.MinCoverage = 0.1, 1, 4, 0.1 + beyondSeeds := false + for _, r := range []PeriodicRule{experimentalPeriodic(), scenarioA(0.1), b} { + for _, shape := range benchShapes { + p := newPeriodicState(r.HistoryLen) + var sc periodicScratch + for i, ts := range pulseTimes(shape, 4*r.HistoryLen) { + p.observe(ts, TrafficUnclassified, &r, &sc) + if p.n < 2 { + continue + } + var own periodicScratch + p.estimate(&r, &own) + if own.chains > uint64(searchCells(&r)) { + t.Fatalf("%s MaxMissing=%d pulse %d: estimate measured %d periods, Chance charges %d cells", + shape, r.MaxMissing, i, own.chains, searchCells(&r)) + } + beyondSeeds = beyondSeeds || own.chains > uint64(recentGaps*(r.MaxMissing+1)+1) + } + } + } + if !beyondSeeds { + t.Fatal("fixture: no search measured more periods than the seeds and the phase refinement") + } +} + +// Refined candidates are hypotheses the search tries, so they must not make +// chance chains fire more often than the seeds alone. On these fixed Poisson +// trains, with the rule of scenario a) and MaxChance 0.1, ad3ab30f (seeds +// only) fires on 60; 5b816f66, whose Chance did not charge the refined +// candidates, fired on 107. +func TestRefinedCandidatesDoNotRaiseTheNullFalseAlarmRate(t *testing.T) { + const trains, seedsOnly = 10000, 60 + fired := 0 + for trial := 0; trial < trains; trial++ { + d := newDet(t, Config{Periodic: []PeriodicRule{scenarioA(0.1)}}) + var evs []Event + for i, off := range poissonTrain(int64(trial)) { + evs = append(evs, relayEvent(t, "n"+strconv.Itoa(i), at(off), 1, 2)) + } + if len(run(t, d, evs)) > 0 { + fired++ + } + } + if fired > seedsOnly+seedsOnly/10 { + t.Fatalf("%d of %d null trains fired, seeds alone %d (+10%% allowed)", fired, trains, seedsOnly) + } + t.Logf("%d of %d null trains fired, seeds alone %d", fired, trains, seedsOnly) +} + +// A gap that bridges a missing pulse takes the nearest multiple of the +// seed, not the one below: with a seed of 303s (from the 606s gap), 594s is +// two periods although 594/303 < 2. The range then runs through both +// missing-pulse gaps to exactly 300s, which explains four newest gaps where +// the seed explains two. +func TestRefinementRoundsTheMultipleOfAGap(t *testing.T) { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel, r.MaxMissing = 6*time.Second, 0, 1 + p := ringOf(&r, 0, 1000*time.Second, 1606*time.Second, 2200*time.Second, 2502*time.Second, 2800*time.Second) + var sc periodicScratch + c, ref := p.chainRefined(int64(303*time.Second), &r, &sc) + if c.gaps != 2 { + t.Fatalf("fixture: the seed explains %d gaps, want 2", c.gaps) + } + if ref != int64(300*time.Second) { + t.Fatalf("refined period %v, want 5m0s", time.Duration(ref)) + } + if got := p.recentExplained(ref, &r, &sc); got != 4 { + t.Fatalf("refined period explains %d newest gaps, want 4", got) + } +} + +// A gap that needs more than MaxMissing+1 periods ends the common range: +// with MaxMissing 0, the oldest gap (590s, two periods) must neither narrow +// the range nor enter the phase estimate, which stays 902s/3 over the three +// newest gaps. +func TestRefinementRangeEndsAtAGapBeyondMaxMissing(t *testing.T) { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel, r.MaxMissing = 6*time.Second, 0, 0 + p := ringOf(&r, 0, 590*time.Second, 890*time.Second, 1192*time.Second, 1492*time.Second) + var sc periodicScratch + c, ref := p.chainRefined(int64(294*time.Second), &r, &sc) + if c.gaps != 1 { + t.Fatalf("fixture: the seed explains %d gaps, want 1", c.gaps) + } + if want := int64(902*time.Second) / 3; ref != want { + t.Fatalf("refined period %v, want %v", time.Duration(ref), time.Duration(want)) + } +} + +// A refined period equal to a seed already measured is not measured again. +// Newest first, the gaps are 300s, 306s and 294s: the seed 300s explains all +// three, and the seeds 306s and 294s both refine to exactly 300s. The search +// measures three chains: one pass of three gaps per seed, plus a +// verification walk of three gaps for each of the two refined cells. +func TestRefinedPeriodEqualToASeedIsNotMeasuredAgain(t *testing.T) { + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel, r.MaxMissing = 6*time.Second, 0, 0 + p := ringOf(&r, 0, 294*time.Second, 600*time.Second, 900*time.Second) + var sc periodicScratch + best := p.estimate(&r, &sc) + if best.period != int64(300*time.Second) || best.gaps != 3 { + t.Fatalf("fixture: estimate %v over %d gaps, want 5m0s over 3", time.Duration(best.period), best.gaps) + } + if sc.chains != 3 || sc.steps != 15 { + t.Fatalf("%d chains and %d steps, want 3 and 15", sc.chains, sc.steps) + } +} + // ---- dedup horizon (finding 2) ---- func dedupDet(t *testing.T, horizon, reorder time.Duration) *Detector { @@ -547,6 +690,24 @@ func TestDedupOlderCopyOutOfOrderKeepsTheNewerCopy(t *testing.T) { } } +// An older copy more than the horizon before the remembered one is still a +// duplicate: only a copy beyond the horizon after the remembered one counts +// again. The remembered copy (10m) is still held for the reorder delay, so +// nothing has been processed and the older copy is not late. +func TestDedupOlderCopyBeyondTheHorizonIsStillADuplicate(t *testing.T) { + const h, rd = 6 * time.Minute, 5 * time.Minute + d := dedupDet(t, h, rd) + if got := observeStatus(t, d, "same", at(10*time.Minute)); got != StatusAccepted { + t.Fatal(got) + } + if got := observeStatus(t, d, "same", at(10*time.Minute-h-time.Nanosecond)); got != StatusDuplicate { + t.Fatalf("older copy beyond the horizon: %v, want duplicate", got) + } + if s := d.Snapshot(); s.Accepted != 1 || s.Duplicates != 1 { + t.Fatalf("accepted=%d duplicates=%d, want 1 and 1", s.Accepted, s.Duplicates) + } +} + // ---- automatic history start (finding 3) ---- func historyCfg(reorder time.Duration, buffer int) Config { diff --git a/internal/anomaly/testdata/replay_golden.json b/internal/anomaly/testdata/replay_golden.json index 5cf04da4d..8e47bdda3 100644 --- a/internal/anomaly/testdata/replay_golden.json +++ b/internal/anomaly/testdata/replay_golden.json @@ -87,7 +87,7 @@ "Support": 8, "Events": 8, "Coverage": 1, - "Chance": 4.59387e-8, + "Chance": 9.09399e-8, "SufficientAt": "2030-01-07T12:35:00Z", "Alternatives": [ { @@ -127,7 +127,7 @@ "Support": 8, "Events": 8, "Coverage": 1, - "Chance": 4.59387e-8, + "Chance": 9.09399e-8, "SufficientAt": "2030-01-07T12:35:00Z", "Alternatives": [ { @@ -242,7 +242,7 @@ "Support": 8, "Events": 8, "Coverage": 1, - "Chance": 0.000241965, + "Chance": 0.000478992, "SufficientAt": "2030-01-07T12:47:00Z", "Alternatives": null } @@ -271,7 +271,7 @@ "Support": 8, "Events": 8, "Coverage": 1, - "Chance": 0.000241965, + "Chance": 0.000478992, "SufficientAt": "2030-01-07T12:47:00Z", "Alternatives": null } From 4554e3146e1fafd3e804d318a6889c59dbac714b Mon Sep 17 00:00:00 2001 From: dborup Date: Sat, 26 Sep 2026 14:50:53 +0200 Subject: [PATCH 9/9] test(anomaly): lock the refined-candidate gate to the signal Chance Re-review of 0d5995d4, P3-1: mutant N7 (gate at Chance/2 <= MaxChance) survived. TestRefinedCandidateGateIsTheSignalChance builds a ring where only the refined period meets the criteria and checks that it becomes the estimate and signals at MaxChance equal to its Chance, and is not taken one ulp below or with Chance just under twice MaxChance. Nit 1: the searchCells comment states the strict union bound per pulse, searchCells + (MaxMissing+1), and why the kept-hypothesis cells are left out. Test and comment only; no change to detection behaviour or the golden file. Co-Authored-By: Claude Opus 5.5 --- internal/anomaly/periodic.go | 7 +++- internal/anomaly/review_fixes_test.go | 57 +++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/internal/anomaly/periodic.go b/internal/anomaly/periodic.go index 7f4a9f0e1..77db75578 100644 --- a/internal/anomaly/periodic.go +++ b/internal/anomaly/periodic.go @@ -558,9 +558,12 @@ func (p *periodicState) chance(c chainInfo, r *PeriodicRule) float64 { // searchCells is the number of candidate periods the search may report per // pulse, each one cell of the union bound: a seed g/k for each of the // recentGaps newest gaps and k = 1..MaxMissing+1, at most one refined period -// per seed, and the phase refinement of the best seed. (A kept hypothesis +// per seed, and the phase refinement of the best seed. A kept hypothesis // reports only when the search does not run, with at most MaxMissing+1 -// periods: itself and its multiples.) +// periods (itself and its multiples), so the strict union bound per pulse is +// searchCells + (MaxMissing+1) cells. Those MaxMissing+1 are left out: the +// kept branch predates the refined cells and was never charged, and they +// add less than 1/32 of searchCells (about 3 %) to the bound. // // Each cell spends one gap on its fit: chance() uses pg^(gaps-1). For a seed // that is the gap it is taken from. A refined period lies in the common diff --git a/internal/anomaly/review_fixes_test.go b/internal/anomaly/review_fixes_test.go index e2d2928de..93d17b225 100644 --- a/internal/anomaly/review_fixes_test.go +++ b/internal/anomaly/review_fixes_test.go @@ -147,6 +147,63 @@ func TestRefinedCandidateReplacesOnlyWhenItWouldSignal(t *testing.T) { } } +// The gate on the refined candidate is the signal decision itself: the same +// Chance, compared with MaxChance exactly as evaluate compares it. The ring +// is the alternating 310s/330s train at P=320s (tol exactly 10s), where no +// seed explains two gaps and only the refined period 320s meets the +// criteria. At MaxChance equal to its Chance the refined chain is the +// estimate and signals; one ulp below, or with Chance just under twice +// MaxChance, it is not taken and nothing signals, so the seeds' estimate +// stays (a looser gate, e.g. Chance/2, would take it as a hypothesis that +// cannot signal). +func TestRefinedCandidateGateIsTheSignalChance(t *testing.T) { + const period = 320 * time.Second + ring := func(r *PeriodicRule) *periodicState { + offsets := make([]time.Duration, 32) + for i := range offsets { + offsets[i] = time.Duration(i) * period + if i%2 == 1 { + offsets[i] -= 10 * time.Second + } + } + p := ringOf(r, offsets...) + p.evaluated = uint64(p.n) + return p + } + r := experimentalPeriodic() + r.JitterAbs, r.JitterRel, r.MaxJitterFraction = 0, 1.0/32, 1 + var sc periodicScratch + p := ring(&r) + ref := p.chainFor(int64(period), &r, &sc) + chance := p.chance(ref, &r) + if ref.tol != int64(10*time.Second) || !ref.meets(&r) || ref.gaps != 31 || !(chance > 0 && chance < 1) { + t.Fatalf("fixture: refined chain %+v chance %g", ref, chance) + } + for _, c := range []struct { + name string + maxChance float64 + want bool + }{ + {"Chance equal to MaxChance", chance, true}, + {"Chance one ulp above MaxChance", math.Nextafter(chance, 0), false}, + {"Chance just below twice MaxChance", chance * 0.51, false}, + } { + t.Run(c.name, func(t *testing.T) { + r := r + r.MaxChance = c.maxChance + p := ring(&r) + est := p.estimate(&r, &sc) + if got := est.period == int64(period); got != c.want { + t.Fatalf("estimate %+v: refined 320s taken=%v, want %v", est, got, c.want) + } + p = ring(&r) + if _, ok := p.evaluate(p.at(p.n-1), &r, &sc); ok != c.want { + t.Fatalf("evaluate signalled=%v, want %v", ok, c.want) + } + }) + } +} + // ringOf returns a periodic state holding pulses at the given offsets from t0. func ringOf(r *PeriodicRule, offsets ...time.Duration) *periodicState { p := newPeriodicState(r.HistoryLen)