feat: Alert Knob

[sinapah]

Issue

Based on changes in canonical/cos-lib#205.

Solution

Context

Testing Instructions

To test, first deploy the following bundle:

bundle: kubernetes
applications:
  aval:
    charm: avalanche-k8s
    channel: dev/edge
    revision: 79
    base: ubuntu@26.04/stable
    resources:
      avalanche-image: 21
    scale: 1
    constraints: arch=amd64
  cos-config:
    charm: cos-configuration-k8s
    channel: dev/edge
    revision: 124
    base: ubuntu@26.04/stable
    resources:
      git-sync-image: 38
    scale: 1
    options:
      git_branch: main
      git_repo: https://github.com/sinapah/cos-config
      prometheus_alert_rules_path: prometheus_alert_rules
    constraints: arch=amd64
    storage:
      content-from-git: kubernetes,1,1024M
    trust: true
  loki:
    charm: loki-k8s
    channel: dev/edge
    revision: 250
    base: ubuntu@26.04/stable
    resources:
      loki-image: 105
      node-exporter-image: 4
    scale: 1
    constraints: arch=amd64
    storage:
      active-index-directory: kubernetes,1,1024M
      loki-chunks: kubernetes,1,1024M
    trust: true
  otelcol:
    charm: opentelemetry-collector-k8s
    channel: dev/edge
    revision: 238
    base: ubuntu@26.04/stable
    resources:
      opentelemetry-collector-image: 11
    scale: 1
    constraints: arch=amd64
    storage:
      persisted: kubernetes,1,1024M
    trust: true
  prom:
    charm: prometheus-k8s
    channel: dev/edge
    revision: 320
    base: ubuntu@26.04/stable
    resources:
      prometheus-image: 155
    scale: 1
    constraints: arch=amd64
    storage:
      database: kubernetes,1,1024M
    trust: true
  traefik:
    charm: traefik-k8s
    channel: latest/edge
    revision: 413
    base: ubuntu@20.04/stable
    resources:
      traefik-image: 174
    scale: 1
    constraints: arch=amd64
    storage:
      configurations: kubernetes,1,1024M
    trust: true
relations:
- - otelcol:metrics-endpoint
  - aval:metrics-endpoint
- - otelcol:send-remote-write
  - prom:receive-remote-write
- - otelcol:metrics-endpoint
  - loki:metrics-endpoint
- - prom:ingress
  - traefik:ingress-per-unit
- - cos-config:send-remote-write
  - prom:receive-remote-write

You should then expect to see in Prometheus's prometheus container at /etc/prometheus/rules, the following file: juju_test_a3ea41d9_otelcol.rules. The content of the file should be:

groups:
- name: test_a3ea41d9_otelcol_AggregatorHostHealth_rules
  rules:
  - alert: HostMetricsMissing
    annotations:
      description: '`Up` missing for unit ''{{ $labels.juju_unit }}'' of application
        {{ $labels.juju_application }} in model {{ $labels.juju_model }}. Please ensure
        the unit or the collector scraping it is up and is able to successfully reach
        the metrics backend.'
      summary: Unit '{{ $labels.juju_unit }}' of application '{{ $labels.juju_application
        }}' is down or failing to remote write.
    expr: absent(up{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3",juju_unit="otelcol/0"})
    for: 5m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      juju_unit: otelcol/0
      severity: warning
  - alert: AggregatorMetricsMissing
    annotations:
      description: '`Up` missing for ALL units of application {{ $labels.juju_application
        }} in model {{ $labels.juju_model }}. This can also mean the units or the
        collector scraping them are unable to reach the remote write endpoint of the
        metrics backend. Please ensure the correct firewall rules are applied.'
      summary: Metrics not received from application '{{ $labels.juju_application
        }}'. All units are down or failing to remote write.
    expr: absent(up{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"})
    for: 5m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
- name: test_a3ea41d9_otelcol_prometheus_alerts_rules
  rules:
  - alert: PrometheusTargetDown
    annotations:
      description: At least one scrape target is down for {{ $labels.juju_application
        }}
      summary: Prometheus target is down
    expr: sum by (juju_application) (up{juju_application="prometheus-k8s",juju_model="observability",juju_model_uuid="e9f39fb5-e6ff-4e0c-82a5-712bdd7e1912"}
      == 0) > 0
    for: 15m
    labels:
      juju_application: prometheus-k8s
      juju_charm: opentelemetry-collector-k8s
      juju_model: observability
      juju_model_uuid: e9f39fb5-e6ff-4e0c-82a5-712bdd7e1912
      severity: critical
      team: observability
- name: test_a3ea41d9_aval_AlwaysFiringDueToAbsentMetric_rules
  rules:
  - alert: AlwaysFiringDueToAbsentMetric
    annotations:
      description: '{{ $labels.instance }} of job {{ $labels.job }} is firing the
        dummy alarm.'
      summary: Instance {{ $labels.instance }} dummy alarm (always firing)
    expr: absent(some_metric_name_that_shouldnt_exist{job="non_existing_job",juju_application="aval",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"})
    for: 0m
    labels:
      juju_application: aval
      juju_charm: avalanche-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: High
- name: test_a3ea41d9_aval_AlwaysFiringDueToNumericValue_rules
  rules:
  - alert: AlwaysFiringDueToNumericValue
    annotations:
      description: '{{ $labels.instance }} of job {{ $labels.job }} is firing the
        dummy alarm.'
      summary: Instance {{ $labels.instance }} dummy alarm (always firing)
    expr: avalanche_metric_mmmmm_0_0{juju_application="aval",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3",series_id="0"}
      > -1
    for: 0m
    labels:
      juju_application: aval
      juju_charm: avalanche-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: High
- name: test_a3ea41d9_aval_HostHealth_rules
  rules:
  - alert: HostDown
    annotations:
      description: Juju application '{{ $labels.juju_application }}' in model '{{
        $labels.juju_model }}' is down. Prometheus has been unable to scrape it during
        at least the past five minutes.
      summary: Host '{{ $labels.instance }}' is down.
    expr: up{juju_application="aval",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}
      < 1
    for: 5m
    labels:
      juju_application: aval
      juju_charm: avalanche-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
  - alert: HostMetricsMissing
    annotations:
      description: '`Up` missing for unit ''{{ $labels.juju_unit }}'' of application
        {{ $labels.juju_application }} in model {{ $labels.juju_model }}. Please ensure
        the unit or the collector scraping it is up and is able to successfully reach
        the metrics backend.'
      summary: Unit '{{ $labels.juju_unit }}' of application '{{ $labels.juju_application
        }}' is down or failing to remote write.
    expr: absent(up{juju_application="aval",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"})
    for: 5m
    labels:
      juju_application: aval
      juju_charm: avalanche-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: warning
- name: test_a3ea41d9_loki_LokiDistributorFailures_rules
  rules:
  - alert: LokiDistributorRejectingLogs
    annotations:
      description: "The {{ $labels.job }} distributor is rejecting log entries on\
        \ route {{ $labels.route }} due to {{ $labels.reason }}\n  Rate = {{ printf\
        \ \"%.2f\" $value }}/s\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      summary: Loki distributor rejecting logs on route {{ $labels.route }} (job {{
        $labels.job }})
    expr: sum by (namespace, job, route, reason) (rate(loki_distributor_rejected_requests_total{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
      > 0.1
    for: 5m
    labels:
      juju_application: loki
      juju_charm: loki-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: warning
- name: test_a3ea41d9_loki_LokiProcessTooManyRestarts_rules
  rules:
  - alert: LokiProcessTooManyRestarts
    annotations:
      description: "A loki process had too many restarts (target {{ $labels.instance\
        \ }})\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      summary: Loki process too many restarts (instance {{ $labels.instance }})
    expr: changes(process_start_time_seconds{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[15m])
      > 5
    for: 0m
    labels:
      juju_application: loki
      juju_charm: loki-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: warning
- name: test_a3ea41d9_loki_LokiRequestErrors_rules
  rules:
  - alert: LokiRequestErrors
    annotations:
      description: "The {{ $labels.job }} and {{ $labels.route }} are experiencing\
        \ errors\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      summary: Loki request errors (instance {{ $labels.instance }})
    expr: 100 * sum by (namespace, job, route) (rate(loki_request_duration_seconds_count{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3",status_code=~"5.."}[1m]))
      / sum by (namespace, job, route) (rate(loki_request_duration_seconds_count{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[1m]))
      > 10
    for: 15m
    labels:
      juju_application: loki
      juju_charm: loki-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
- name: test_a3ea41d9_loki_LokiRequestLatency_rules
  rules:
  - alert: LokiRequestLatency
    annotations:
      description: "The {{ $labels.job }} on route {{ $labels.route }} is experiencing\
        \ {{ printf \"%.2f\" $value }}s 99th percentile latency\n  VALUE = {{ $value\
        \ }}\n  LABELS = {{ $labels }}"
      summary: Loki request latency for route {{ $labels.route }} (job {{ $labels.job
        }})
    expr: (histogram_quantile(0.99, sum by (namespace, job, route, le) (rate(loki_request_duration_seconds_bucket{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3",route!~"(?i).*tail.*"}[5m]))))
      > 1
    for: 5m
    labels:
      juju_application: loki
      juju_charm: loki-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: warning
- name: test_a3ea41d9_loki_LokiRequestPanic_rules
  rules:
  - alert: LokiRequestPanic
    annotations:
      description: "The {{ $labels.job }} is experiencing {{ printf \"%.2f\" $value\
        \ }}% increase of panics\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      summary: Loki request panic (instance {{ $labels.instance }})
    expr: sum by (namespace, job) (increase(loki_panic_total{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[10m]))
      > 0
    for: 5m
    labels:
      juju_application: loki
      juju_charm: loki-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
- name: test_a3ea41d9_loki_HostHealth_rules
  rules:
  - alert: HostDown
    annotations:
      description: Juju application '{{ $labels.juju_application }}' in model '{{
        $labels.juju_model }}' is down. Prometheus has been unable to scrape it during
        at least the past five minutes.
      summary: Host '{{ $labels.instance }}' is down.
    expr: up{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}
      < 1
    for: 5m
    labels:
      juju_application: loki
      juju_charm: loki-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
  - alert: HostMetricsMissing
    annotations:
      description: '`Up` missing for unit ''{{ $labels.juju_unit }}'' of application
        {{ $labels.juju_application }} in model {{ $labels.juju_model }}. Please ensure
        the unit or the collector scraping it is up and is able to successfully reach
        the metrics backend.'
      summary: Unit '{{ $labels.juju_unit }}' of application '{{ $labels.juju_application
        }}' is down or failing to remote write.
    expr: absent(up{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"})
    for: 5m
    labels:
      juju_application: loki
      juju_charm: loki-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: warning
- name: test_a3ea41d9_otelcol_Exporter_rules
  rules:
  - alert: failed-logs
    annotations:
      description: Destination may have a problem or payload is incorrect
      summary: Some log points failed to send by exporter (Unit {{ $labels.juju_unit
        }} in model {{ $labels.juju_model }})
    expr: sum(rate(otelcol_exporter_send_failed_log_records{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
      > 0
    for: 5m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
  - alert: failed-metrics
    annotations:
      description: Destination may have a problem or payload is incorrect
      summary: Some metric points failed to send by exporter (Unit {{ $labels.juju_unit
        }} in model {{ $labels.juju_model }})
    expr: sum(rate(otelcol_exporter_send_failed_metric_points{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
      > 0
    for: 5m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
  - alert: failed-traces
    annotations:
      description: Destination may have a problem or payload is incorrect
      summary: Some spans failed to send by exporter (Unit {{ $labels.juju_unit }}
        in model {{ $labels.juju_model }})
    expr: sum(rate(otelcol_exporter_send_failed_spans{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
      > 0
    for: 5m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
  - alert: queue-near-capacity
    annotations:
      description: 'The exporter''s sending queue has stayed above 80% capacity, meaning
        the destination can''t keep up with incoming telemetry. If the queue reaches
        capacity, data will be dropped. Check that the exporter''s endpoint is operational,
        or raise the ''queue_size'' config option. Note that data is only retried
        until ''max_elapsed_time_min'' elapses, after which it is dropped.

        '
      summary: Exporter sending queue is over 80% full (Unit {{ $labels.juju_unit
        }} in model {{ $labels.juju_model }})
    expr: otelcol_exporter_queue_size{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}
      / otelcol_exporter_queue_capacity{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}
      > 0.8
    for: 5m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: warning
  - alert: queue-overflow
    annotations:
      description: 'The exporter''s sending queue is at capacity and rejecting new
        telemetry at enqueue, causing data loss. Check that the exporter''s endpoint
        is operational, reduce the incoming load, or raise the ''queue_size'' config
        option.

        '
      summary: Exporter sending queue is full and dropping telemetry (Unit {{ $labels.juju_unit
        }} in model {{ $labels.juju_model }})
    expr: sum(rate(otelcol_exporter_enqueue_failed_spans{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
      > 0 or sum(rate(otelcol_exporter_enqueue_failed_log_records{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
      > 0 or sum(rate(otelcol_exporter_enqueue_failed_metric_points{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
      > 0
    for: 0m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
- name: test_a3ea41d9_otelcol_Hardware_rules
  rules:
  - alert: high-cpu-usage
    annotations:
      description: Collector needs to scale up
      summary: High max CPU usage (Unit {{ $labels.juju_unit }} in model {{ $labels.juju_model
        }})
    expr: max(rate(otelcol_process_cpu_seconds_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m])
      * 100) > 90
    for: 5m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
- name: test_a3ea41d9_otelcol_Receiver_rules
  rules:
  - alert: receiver-refused-logs
    annotations:
      description: Maybe collector has received non standard log points or it reached
        some limits
      summary: Some log points have been refused by receiver (Unit {{ $labels.juju_unit
        }} in model {{ $labels.juju_model }})
    expr: sum(rate(otelcol_receiver_refused_log_records_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
      > 0
    for: 5m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical
  - alert: receiver-refused-metrics
    annotations:
      description: Maybe collector has received non standard metric points or it reached
        some limits
      summary: Some metric points have been refused by receiver (Unit {{ $labels.juju_unit
        }} in model {{ $labels.juju_model }})
    expr: sum(rate(otelcol_receiver_refused_metric_points_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
      > 0
    for: 5m
    labels:
      juju_application: otelcol
      juju_charm: opentelemetry-collector-k8s
      juju_model: test
      juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
      severity: critical

Now, let's refresh the charm and then perform the following modifcations:

1. Remove all alerts related to Avalanche. That means remove all alerts where juju_charm is equal to avalanche-k8s.
2. Remove all alerts related to the group test_a3ea41d9_otelcol_Receiver_rules.
3. Remove all alerts where the severity is critical. This should result in the removal of the HostMetricsMissing alert.
4. Patch the alert named "PrometheusTargetDown" and set its for to 1h instead of the current 15m.
5. Patch the alert named AggregatorMetricsMissing and change its annotations.summary to be "XYZ".
6. Patch the alert named high-cpu-usage and change its expr from max(rate(otelcol_process_cpu_seconds_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]) to max(rate(otelcol_process_cpu_seconds_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[45m]).
7. Add two custom alert groups named "group-a" and "group-b" each with two custom alerts.

To apply the changes above, we need a YAML block that looks like:

remove:
  - where:
      labels:
        juju_charm: avalanche-k8s

  - where:
      group: test_a3ea41d9_otelcol_Receiver_rules

  - where:
      labels:
        severity: critical

patch:
  - where:
      alert: PrometheusTargetDown
    set:
      for: 1h

  - where:
      alert: LokiRequestLatency
    set:
      annotations:
        summary: "XYZ"

  - where:
      alert: queue-near-capacity
    set:
      expr: 'max(rate(otelcol_process_cpu_seconds_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[45m]))'

add:
  groups:
    - name: group-a
      rules:
        - alert: GroupAAlertOne
          expr: up{juju_model="prod", juju_application="myapp"} == 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Group A alert one fired"

        - alert: GroupAAlertTwo
          expr: rate(http_requests_total{juju_model="prod"}[5m]) > 100
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Group A alert two fired"

    - name: group-b
      rules:
        - alert: GroupBAlertOne
          expr: memory_usage_bytes{juju_model="prod", juju_application="myapp"} > 1e9
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Group B alert one fired"

        - alert: GroupBAlertTwo
          expr: disk_io_time_seconds_total{juju_model="prod"} > 0.9
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Group B alert two fired"

The resulting diff is:

<   - alert: AggregatorMetricsMissing
<     annotations:
<       description: '`Up` missing for ALL units of application {{ $labels.juju_application
<         }} in model {{ $labels.juju_model }}. This can also mean the units or the
<         collector scraping them are unable to reach the remote write endpoint of the
<         metrics backend. Please ensure the correct firewall rules are applied.'
<       summary: Metrics not received from application '{{ $labels.juju_application
<         }}'. All units are down or failing to remote write.
<     expr: absent(up{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"})
<     for: 5m
<     labels:
<       juju_application: otelcol
<       juju_charm: opentelemetry-collector-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
< - name: test_a3ea41d9_otelcol_prometheus_alerts_rules
<   rules:
<   - alert: PrometheusTargetDown
<     annotations:
<       description: At least one scrape target is down for {{ $labels.juju_application
<         }}
<       summary: Prometheus target is down
<     expr: sum by (juju_application) (up{juju_application="prometheus-k8s",juju_model="observability",juju_model_uuid="e9f39fb5-e6ff-4e0c-82a5-712bdd7e1912"}
<       == 0) > 0
<     for: 15m
<     labels:
<       juju_application: prometheus-k8s
<       juju_charm: opentelemetry-collector-k8s
<       juju_model: observability
<       juju_model_uuid: e9f39fb5-e6ff-4e0c-82a5-712bdd7e1912
<       severity: critical
<       team: observability
< - name: test_a3ea41d9_aval_AlwaysFiringDueToAbsentMetric_rules
<   rules:
<   - alert: AlwaysFiringDueToAbsentMetric
<     annotations:
<       description: '{{ $labels.instance }} of job {{ $labels.job }} is firing the
<         dummy alarm.'
<       summary: Instance {{ $labels.instance }} dummy alarm (always firing)
<     expr: absent(some_metric_name_that_shouldnt_exist{job="non_existing_job",juju_application="aval",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"})
<     for: 0m
<     labels:
<       juju_application: aval
<       juju_charm: avalanche-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: High
< - name: test_a3ea41d9_aval_AlwaysFiringDueToNumericValue_rules
<   rules:
<   - alert: AlwaysFiringDueToNumericValue
<     annotations:
<       description: '{{ $labels.instance }} of job {{ $labels.job }} is firing the
<         dummy alarm.'
<       summary: Instance {{ $labels.instance }} dummy alarm (always firing)
<     expr: avalanche_metric_mmmmm_0_0{juju_application="aval",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3",series_id="0"}
<       > -1
<     for: 0m
<     labels:
<       juju_application: aval
<       juju_charm: avalanche-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: High
< - name: test_a3ea41d9_aval_HostHealth_rules
<   rules:
<   - alert: HostDown
<     annotations:
<       description: Juju application '{{ $labels.juju_application }}' in model '{{
<         $labels.juju_model }}' is down. Prometheus has been unable to scrape it during
<         at least the past five minutes.
<       summary: Host '{{ $labels.instance }}' is down.
<     expr: up{juju_application="aval",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}
<       < 1
<     for: 5m
<     labels:
<       juju_application: aval
<       juju_charm: avalanche-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
<   - alert: HostMetricsMissing
<     annotations:
<       description: '`Up` missing for unit ''{{ $labels.juju_unit }}'' of application
<         {{ $labels.juju_application }} in model {{ $labels.juju_model }}. Please ensure
<         the unit or the collector scraping it is up and is able to successfully reach
<         the metrics backend.'
<       summary: Unit '{{ $labels.juju_unit }}' of application '{{ $labels.juju_application
<         }}' is down or failing to remote write.
<     expr: absent(up{juju_application="aval",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"})
<     for: 5m
<     labels:
<       juju_application: aval
<       juju_charm: avalanche-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: warning
152,168d54
< - name: test_a3ea41d9_loki_LokiRequestErrors_rules
<   rules:
<   - alert: LokiRequestErrors
<     annotations:
<       description: "The {{ $labels.job }} and {{ $labels.route }} are experiencing\
<         \ errors\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
<       summary: Loki request errors (instance {{ $labels.instance }})
<     expr: 100 * sum by (namespace, job, route) (rate(loki_request_duration_seconds_count{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3",status_code=~"5.."}[1m]))
<       / sum by (namespace, job, route) (rate(loki_request_duration_seconds_count{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[1m]))
<       > 10
<     for: 15m
<     labels:
<       juju_application: loki
<       juju_charm: loki-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
176,177c62
<       summary: Loki request latency for route {{ $labels.route }} (job {{ $labels.job
<         }})
---
>       summary: XYZ
187,202d71
< - name: test_a3ea41d9_loki_LokiRequestPanic_rules
<   rules:
<   - alert: LokiRequestPanic
<     annotations:
<       description: "The {{ $labels.job }} is experiencing {{ printf \"%.2f\" $value\
<         \ }}% increase of panics\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
<       summary: Loki request panic (instance {{ $labels.instance }})
<     expr: sum by (namespace, job) (increase(loki_panic_total{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[10m]))
<       > 0
<     for: 5m
<     labels:
<       juju_application: loki
<       juju_charm: loki-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
205,219d73
<   - alert: HostDown
<     annotations:
<       description: Juju application '{{ $labels.juju_application }}' in model '{{
<         $labels.juju_model }}' is down. Prometheus has been unable to scrape it during
<         at least the past five minutes.
<       summary: Host '{{ $labels.instance }}' is down.
<     expr: up{juju_application="loki",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}
<       < 1
<     for: 5m
<     labels:
<       juju_application: loki
<       juju_charm: loki-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
238,279d91
<   - alert: failed-logs
<     annotations:
<       description: Destination may have a problem or payload is incorrect
<       summary: Some log points failed to send by exporter (Unit {{ $labels.juju_unit
<         }} in model {{ $labels.juju_model }})
<     expr: sum(rate(otelcol_exporter_send_failed_log_records{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
<       > 0
<     for: 5m
<     labels:
<       juju_application: otelcol
<       juju_charm: opentelemetry-collector-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
<   - alert: failed-metrics
<     annotations:
<       description: Destination may have a problem or payload is incorrect
<       summary: Some metric points failed to send by exporter (Unit {{ $labels.juju_unit
<         }} in model {{ $labels.juju_model }})
<     expr: sum(rate(otelcol_exporter_send_failed_metric_points{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
<       > 0
<     for: 5m
<     labels:
<       juju_application: otelcol
<       juju_charm: opentelemetry-collector-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
<   - alert: failed-traces
<     annotations:
<       description: Destination may have a problem or payload is incorrect
<       summary: Some spans failed to send by exporter (Unit {{ $labels.juju_unit }}
<         in model {{ $labels.juju_model }})
<     expr: sum(rate(otelcol_exporter_send_failed_spans{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
<       > 0
<     for: 5m
<     labels:
<       juju_application: otelcol
<       juju_charm: opentelemetry-collector-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
291,293c103
<     expr: otelcol_exporter_queue_size{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}
<       / otelcol_exporter_queue_capacity{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}
<       > 0.8
---
>     expr: max(rate(otelcol_process_cpu_seconds_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[45m]))
301,306d110
<   - alert: queue-overflow
<     annotations:
<       description: 'The exporter''s sending queue is at capacity and rejecting new
<         telemetry at enqueue, causing data loss. Check that the exporter''s endpoint
<         is operational, reduce the incoming load, or raise the ''queue_size'' config
<         option.
308,322c112,113
<         '
<       summary: Exporter sending queue is full and dropping telemetry (Unit {{ $labels.juju_unit
<         }} in model {{ $labels.juju_model }})
<     expr: sum(rate(otelcol_exporter_enqueue_failed_spans{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
<       > 0 or sum(rate(otelcol_exporter_enqueue_failed_log_records{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
<       > 0 or sum(rate(otelcol_exporter_enqueue_failed_metric_points{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
<       > 0
<     for: 0m
<     labels:
<       juju_application: otelcol
<       juju_charm: opentelemetry-collector-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
< - name: test_a3ea41d9_otelcol_Hardware_rules
---
> groups:
> - name: group-a
324c115
<   - alert: high-cpu-usage
---
>   - alert: GroupAAlertOne
326,330c117,118
<       description: Collector needs to scale up
<       summary: High max CPU usage (Unit {{ $labels.juju_unit }} in model {{ $labels.juju_model
<         }})
<     expr: max(rate(otelcol_process_cpu_seconds_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m])
<       * 100) > 90
---
>       summary: Group A alert one fired
>     expr: up{juju_model="prod", juju_application="myapp"} == 0
333,338c121,129
<       juju_application: otelcol
<       juju_charm: opentelemetry-collector-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
< - name: test_a3ea41d9_otelcol_Receiver_rules
---
>       severity: warning
>   - alert: GroupAAlertTwo
>     annotations:
>       summary: Group A alert two fired
>     expr: rate(http_requests_total{juju_model="prod"}[5m]) > 100
>     for: 10m
>     labels:
>       severity: warning
> - name: group-b
340c131
<   - alert: receiver-refused-logs
---
>   - alert: GroupBAlertOne
342,347c133,134
<       description: Maybe collector has received non standard log points or it reached
<         some limits
<       summary: Some log points have been refused by receiver (Unit {{ $labels.juju_unit
<         }} in model {{ $labels.juju_model }})
<     expr: sum(rate(otelcol_receiver_refused_log_records_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
<       > 0
---
>       summary: Group B alert one fired
>     expr: memory_usage_bytes{juju_model="prod", juju_application="myapp"} > 1e9
350,355c137,138
<       juju_application: otelcol
<       juju_charm: opentelemetry-collector-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
<   - alert: receiver-refused-metrics
---
>       severity: warning
>   - alert: GroupBAlertTwo
357,363c140,142
<       description: Maybe collector has received non standard metric points or it reached
<         some limits
<       summary: Some metric points have been refused by receiver (Unit {{ $labels.juju_unit
<         }} in model {{ $labels.juju_model }})
<     expr: sum(rate(otelcol_receiver_refused_metric_points_total{juju_application="otelcol",juju_model="test",juju_model_uuid="a3ea41d9-8d54-4897-8098-dde7bc214ea3"}[5m]))
<       > 0
<     for: 5m
---
>       summary: Group B alert two fired
>     expr: disk_io_time_seconds_total{juju_model="prod"} > 0.9
>     for: 15m
365,369c144
<       juju_application: otelcol
<       juju_charm: opentelemetry-collector-k8s
<       juju_model: test
<       juju_model_uuid: a3ea41d9-8d54-4897-8098-dde7bc214ea3
<       severity: critical
---
>       severity: warning

Upgrade Notes

[sinapah]

When canonical/cos-lib#205 goes in, remove branch pinning and bump to latest available cosl version.

[sed-i]

The add directive is interesting: it overlaps with side-loading rules via charm resource via cos-config charm.

On one hand, having one way to do things, as well as scoped responsibilities, points us at not having an add directive. Otoh, it is quite powerful and handy. Any thoughts on this?

[sed-i]

• Would we benefit from a feature file here?
• Do we have a test that captures what happens when we attempt to patch somethings that has no match?