diff --git a/docs/MILESTONES.md b/docs/MILESTONES.md index 88e4e87..3a7f11e 100644 --- a/docs/MILESTONES.md +++ b/docs/MILESTONES.md @@ -49,7 +49,8 @@ Status and scope for each phase of SentinelOps. Dates are approximate; this is a ## Known gaps (tracked, not blocking) - No Grafana/Prometheus scrape config yet — metrics are exposed but not yet centrally collected -- Sustained 40-49% error rates across all services for over an hour post-recovery (not resolving over time as event counts grew 10x) — this looks like a real bug (possibly in the `/api/services` error-rate calculation, or an actual issue in the log producer/ingestion path) rather than a transient catch-up artifact. Needs investigation as the next priority after Milestone 4. +- Fixed: log producer was assigning `level` independently of `status_code` (a 200 response could randomly be logged as ERROR) — root cause of inflated error-rate metrics. Fixed by deriving level from status_code. +- Still open: even after that fix, ANOMALY_PROBABILITY (tuned 0.02 → 0.0015 based on burst-duration math) has not brought error rates down to the expected ~5% in live testing — actual observed rates remain 25-37%. The duty-cycle calculation may be missing a factor (e.g. multiple concurrent bursts per service, or burst state not resetting as assumed). Needs a fresh look with the producer's actual runtime behavior, not just the math. ## Recently closed - `anomaly-rust`: added `detect()` end-to-end tests, including a regression guard for the directional-severity bug diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 3ef8778..c34cc4b 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -31,6 +31,9 @@ services: depends_on: [redpanda] environment: KAFKA_BROKERS: redpanda:9092 + LOG_TOPIC: service-logs + EMIT_INTERVAL_SECONDS: "0.2" + ANOMALY_PROBABILITY: "0.0015" ports: - "8080:8080" networks: [sentinelops-net] diff --git a/services/log-producer/__pycache__/producer.cpython-312.pyc b/services/log-producer/__pycache__/producer.cpython-312.pyc new file mode 100644 index 0000000..f393096 Binary files /dev/null and b/services/log-producer/__pycache__/producer.cpython-312.pyc differ diff --git a/services/log-producer/producer.py b/services/log-producer/producer.py index 4ca21f0..01b590a 100644 --- a/services/log-producer/producer.py +++ b/services/log-producer/producer.py @@ -33,8 +33,14 @@ KAFKA_BROKERS = os.getenv("KAFKA_BROKERS", "localhost:9092") TOPIC = os.getenv("LOG_TOPIC", "service-logs") EMIT_INTERVAL_SECONDS = float(os.getenv("EMIT_INTERVAL_SECONDS", "0.2")) -ANOMALY_PROBABILITY = float(os.getenv("ANOMALY_PROBABILITY", "0.02")) - +# Target: anomaly bursts should occupy roughly 3-5% of total time per +# service, not be the dominant traffic pattern. With burst length averaging +# ~40 events (random.randint(20, 60)) and ~1.25 events/sec/service, a 0.02 +# probability produced bursts ~44% of the time — anomalies were effectively +# "normal," which defeats statistical anomaly detection entirely (there's no +# stable low baseline to detect deviation from). 0.0015 targets ~5% duty cycle: +# mean gap = 1/0.0015 ≈ 667 events, duty = 40/(667+40) ≈ 5.7%. +ANOMALY_PROBABILITY = float(os.getenv("ANOMALY_PROBABILITY", "0.0015")) SERVICES = ["checkout", "auth", "payments", "inventory"] ENDPOINTS = { "checkout": ["/cart/add", "/cart/checkout", "/cart/remove"], @@ -42,7 +48,6 @@ "payments": ["/charge", "/refund", "/payment-status"], "inventory": ["/stock/check", "/stock/reserve", "/stock/release"], } -LOG_LEVELS = ["INFO", "INFO", "INFO", "WARN", "ERROR"] fake = Faker() @@ -123,12 +128,18 @@ def build_event(service: str, anomaly_state: AnomalyState) -> LogEvent: anomaly_injected=True, ) - # Normal baseline traffic + # Normal baseline traffic. `level` is derived from `status_code` (not + # independently randomized) so downstream error-rate calculations are + # meaningful: a 200 response must never be logged as ERROR, and vice versa. latency_ms = round(max(5, random.gauss(120, 40)), 2) status_code = 200 if random.random() > 0.03 else random.choice([400, 404]) - level = random.choice(LOG_LEVELS) + if status_code >= 500: + level = "ERROR" + elif status_code >= 400: + level = "WARN" + else: + level = "INFO" message = f"{service} handled {endpoint} for user {fake.uuid4()[:8]}" - return LogEvent( event_id=str(uuid.uuid4()), timestamp=datetime.now(UTC).isoformat(),