From d90e661f729e905049c935c059702c2670e5779d Mon Sep 17 00:00:00 2001 From: yahiaongh Date: Thu, 23 Jul 2026 19:27:30 +0100 Subject: [PATCH] fix: ANOMALY_PROBABILITY env var was misapplied to redpanda-console instead of log-producer; corrected and verified live (error rates now 0-15%, matching expected burst frequency) --- docs/MILESTONES.md | 2 +- infra/docker-compose.yml | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/MILESTONES.md b/docs/MILESTONES.md index 3a7f11e..bd70290 100644 --- a/docs/MILESTONES.md +++ b/docs/MILESTONES.md @@ -50,7 +50,7 @@ Status and scope for each phase of SentinelOps. Dates are approximate; this is a ## Known gaps (tracked, not blocking) - No Grafana/Prometheus scrape config yet — metrics are exposed but not yet centrally collected - Fixed: log producer was assigning `level` independently of `status_code` (a 200 response could randomly be logged as ERROR) — root cause of inflated error-rate metrics. Fixed by deriving level from status_code. -- Still open: even after that fix, ANOMALY_PROBABILITY (tuned 0.02 → 0.0015 based on burst-duration math) has not brought error rates down to the expected ~5% in live testing — actual observed rates remain 25-37%. The duty-cycle calculation may be missing a factor (e.g. multiple concurrent bursts per service, or burst state not resetting as assumed). Needs a fresh look with the producer's actual runtime behavior, not just the math. +- Fixed: the ANOMALY_PROBABILITY tuning (0.02 → 0.0015) from the previous fix never actually took effect — a copy/paste error had applied the new value to `redpanda-console`'s environment block (which ignores it) instead of `log-producer`'s. Corrected and verified live: error rates dropped from ~30-50% to ~0-15%, matching expected burst-driven anomaly frequency (confirmed against actual burst log timestamps in a sample window). ## Recently closed - `anomaly-rust`: added `detect()` end-to-end tests, including a regression guard for the directional-severity bug diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index c34cc4b..9aeb6af 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -31,9 +31,6 @@ services: depends_on: [redpanda] environment: KAFKA_BROKERS: redpanda:9092 - LOG_TOPIC: service-logs - EMIT_INTERVAL_SECONDS: "0.2" - ANOMALY_PROBABILITY: "0.0015" ports: - "8080:8080" networks: [sentinelops-net] @@ -76,7 +73,7 @@ services: KAFKA_BROKERS: redpanda:9092 LOG_TOPIC: service-logs EMIT_INTERVAL_SECONDS: "0.2" - ANOMALY_PROBABILITY: "0.02" + ANOMALY_PROBABILITY: "0.0015" restart: unless-stopped networks: [sentinelops-net]