proseqgo/monitoring/alerts.yml at main · BehRoooz/proseqgo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
groups:
  - name: proseqgo_simple_alerts
    interval: 30s
    rules:
      - alert: ProSeqGOServiceMetricsTargetDown
        expr: up{job=~"prometheus|embedding_api_metrics|embedding_worker_metrics|go_prediction_api_metrics|trainer_api_metrics|redis_exporter"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Service scrape target down: {{ $labels.job }}"
          description: "{{ $labels.instance }} is down for >2m"

      - alert: ProSeqGOHighHttp5xxRatio
        expr: |
          (
            sum by (service) (rate(http_requests_total{status_code=~"5.."}[5m]))
            /
            clamp_min(sum by (service) (rate(http_requests_total[5m])), 0.001)
          ) > 0.05
          and
          sum by (service) (rate(http_requests_total[5m])) > 0.1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High 5xx ratio on {{ $labels.service }}"
          description: "5xx ratio >5% for 10m with enough traffic"

      - alert: ProSeqGOEmbeddingQueueBacklogHigh
        expr: embedding_queue_jobs{status="queued"} > 20
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Embedding queue backlog high"
          description: "Queued embedding jobs >20 for 10m"

      - alert: ProSeqGOEmbeddingWorkerDownWithBacklog
        expr: |
          up{job="embedding_worker_metrics"} == 0
          and
          rq_queue_length{queue="embedding-jobs"} > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Embedding RQ worker down while queue has jobs"
          description: "embedding-worker scrape is down and Redis queue embedding-jobs is non-empty"

      - alert: ProSeqGORedisDown
        expr: redis_up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Redis is down"
          description: "redis_exporter reports redis_up == 0"