feat: allow configuring alert rules

charmcraft.yaml

@@ -16,11 +16,38 @@ parts:
       - astral-uv
     build-packages:
       - git
+  # Bundle cos-tool so the charm can validate the alert_rules config with
+  # Prometheus' own rule parser at runtime.
+  cos-tool:
+    plugin: dump
+    source: https://github.com/canonical/cos-tool/releases/latest/download/cos-tool-${CRAFT_ARCH_BUILD_FOR}
+    source-type: file
+    permissions:
+      - path: cos-tool-${CRAFT_ARCH_BUILD_FOR}
+        mode: "755"
 
 platforms:
   ubuntu@22.04:amd64:
   ubuntu@24.04:amd64:
 
+actions:
+  get-alert-rules:
+    description: |
+      Dump the alert rules currently in effect as a single Prometheus rules
+      document (YAML), returned in the `alert-rules` result. Use this as a starting
+      point for the `alert_rules` config option: dump the rules, edit them,
+      then set the edited document as the `alert_rules` config.
+
+      Because `juju run` wraps the result under the unit name, extract the raw
+      document before saving. For example:
+
+        juju run openstack-exporter/leader get-alert-rules --format=json \
+          | jq -r 'first(.[]).results."alert-rules"' > rules.yaml
+
+      Edit rules.yaml, then apply it:
+
+        juju config openstack-exporter alert_rules="$(cat rules.yaml)"
+
 requires:
   credentials:
     interface: keystone-admin
@@ -76,6 +103,31 @@ config:
         This option allows the selection of a different channel.
 
         If the snap file has been attached via the openstack-exporter resource, this option has no effect.
+    alert_rules:
+      default: ""
+      type: string
+      description: |
+        A full Prometheus alert rules document (YAML) that replaces the alert rules shipped
+        with the charm. Empty string (default) keeps the shipped rules.
+
+        Use this to customise severities, thresholds and `for:` durations, or to add or remove
+        alerts. The value must be a valid Prometheus rules file with a top-level `groups` key. 
+        For example:
+
+          groups:
+            - name: NovaCompute
+              rules:
+                - alert: NovaComputeDown
+                  expr: openstack_nova_agent_state{adminState="enabled"} == 0
+                  for: 2m
+                  labels:
+                    severity: critical
+                  annotations:
+                    summary: Nova Compute Agent Down
+
+        Warning: Changing this option reloads the alert rules in Prometheus. 
+        Any currently firing alerts that are removed by the new rules will be canceled 
+        and reported to Alert Manager as resolved.
 
 charm-libs:
   - lib: grafana-agent.cos_agent

src/charm.py

@@ -20,11 +20,18 @@
 from ops.model import ActiveStatus, BlockedStatus, ModelError, WaitingStatus
 
 from service import SNAP_NAME, UPSTREAM_SNAP, get_installed_snap_service, snap_install_or_refresh
-from validate_config import validate_cache_ttl, validate_port
+from validate_config import (
+    validate_alert_rules,
+    validate_cache_ttl,
+    validate_port,
+)
 
 logger = logging.getLogger(__name__)
 
 RESOURCE_NAME = "openstack-exporter"
+# Alert rules shipped with the charm, and the directory COSAgentProvider reads rendered rules from.
+SHIPPED_ALERT_RULES_DIRNAME = "prometheus_alert_rules"
+ALERT_RULES_DIRNAME = "alert_rules"
 # Snap config options global constants
 # This is to match between openstack-exporter and the entry in clouds.yaml
 CLOUD_NAME = "openstack"
@@ -42,13 +49,8 @@ def __init__(self, *args: Any) -> None:
         """Initialize the charm."""
         super().__init__(*args)
 
-        self._grafana_agent = COSAgentProvider(
-            self,
-            metrics_endpoints=[
-                {"path": "/metrics", "port": self.config["port"]},
-            ],
-        )
-
+        # Register _configure before COSAgentProvider so rendered alert rules are written
+        # to disk before COSAgentProvider reads them on config_changed.
         self.framework.observe(self.on.install, self._on_install)
         self.framework.observe(self.on.upgrade_charm, self._on_upgrade)
         self.framework.observe(self.on.config_changed, self._configure)
@@ -57,6 +59,15 @@ def __init__(self, *args: Any) -> None:
         self.framework.observe(self.on.credentials_relation_broken, self._configure)
         self.framework.observe(self.on.cos_agent_relation_changed, self._configure)
         self.framework.observe(self.on.cos_agent_relation_broken, self._configure)
+        self.framework.observe(self.on.get_alert_rules_action, self._on_get_alert_rules)
+
+        self._grafana_agent = COSAgentProvider(
+            self,
+            metrics_endpoints=[
+                {"path": "/metrics", "port": self.config["port"]},
+            ],
+            metrics_rules_dir=f"./src/{ALERT_RULES_DIRNAME}",
+        )
 
     def _is_keystone_data_ready(self, data: dict[str, str]) -> bool:
         """Check if all the data is available from keystone.
@@ -156,6 +167,7 @@ def validate_configs(self) -> Optional[str]:
         validators: list[tuple[Callable, str]] = [
             (validate_port, "port"),
             (validate_cache_ttl, "cache_ttl"),
+            (validate_alert_rules, "alert_rules"),
         ]
         for validator, config_key in validators:
             if error := validator(self.model.config[config_key]):
@@ -164,6 +176,51 @@ def validate_configs(self) -> Optional[str]:
         # All config options are valid
         return None
 
+    def _render_alert_rules(self) -> None:
+        """Render alert rules for COSAgentProvider to read.
+
+        Writes the user-provided `alert_rules` document when set, otherwise the alert rules
+        shipped with the charm. The rendered directory is rebuilt each time so unsetting the
+        option restores the shipped rules.
+        """
+        src_dir = Path(self.charm_dir) / "src"
+        rendered_dir = src_dir / ALERT_RULES_DIRNAME
+        rendered_dir.mkdir(parents=True, exist_ok=True)
+
+        # Clear previously rendered rules so unsetting the override restores the shipped rules.
+        for existing in rendered_dir.glob("*"):
+            existing.unlink()
+
+        override = str(self.model.config["alert_rules"])
+        if override.strip():
+            (rendered_dir / "custom.yaml").write_text(override)
+            return
+
+        for path in (src_dir / SHIPPED_ALERT_RULES_DIRNAME).glob("*.yaml"):
+            (rendered_dir / path.name).write_text(path.read_text())
+
+    def _effective_alert_rules(self) -> str:
+        """Return the currently effective alert rules as a single YAML document.
+
+        Returns the user-provided `alert_rules` override when set, otherwise the shipped rules
+        merged into one document with a top-level `groups` key.
+        """
+        override = str(self.model.config["alert_rules"])
+        if override.strip():
+            return override
+
+        src_dir = Path(self.charm_dir) / "src"
+        groups: list = []
+        for path in sorted((src_dir / SHIPPED_ALERT_RULES_DIRNAME).glob("*.yaml")):
+            doc = yaml.safe_load(path.read_text()) or {}
+            groups.extend(doc.get("groups", []))
+        # width avoids backslash line-folding, allow_unicode avoids \uXXXX escapes.
+        return yaml.safe_dump({"groups": groups}, sort_keys=False, allow_unicode=True, width=4096)
+
+    def _on_get_alert_rules(self, event: ops.ActionEvent) -> None:
+        """Dump the effective alert rules so they can be edited and set as config."""
+        event.set_results({"alert-rules": self._effective_alert_rules()})
+
     def install(self) -> None:
         """Install the necessary resources for the charm."""
         # If this fails, it's not recoverable.
@@ -182,6 +239,7 @@ def _configure(self, _: ops.HookEvent) -> None:
             logger.error(config_error)
             return
 
+        self._render_alert_rules()
         self.install()
 
         if self._upstream_snap_present():

src/validate_config.py

@@ -2,8 +2,15 @@
 # See LICENSE file for licensing details.
 """Configuration validation functions."""
 
+import logging
 import re
-from typing import Optional
+from typing import Optional, cast
+
+import yaml
+from cosl import CosTool
+from cosl.cos_tool import OfficialRuleFileFormat
+
+logger = logging.getLogger(__name__)
 
 MAX_PORT = 65535
 
@@ -67,3 +74,33 @@ def validate_cache_ttl(cache_ttl: str) -> Optional[str]:
             )
 
     return None
+
+
+def validate_alert_rules(config: str) -> Optional[str]:
+    """Validate the alert_rules configuration.
+
+    The value is a full Prometheus rules document that replaces the shipped alert rules.
+    Empty string (default) keeps the shipped rules and is valid.
+
+    The document is validated with the bundled `cos-tool` binary via `cosl.CosTool`, which
+    uses Prometheus' own rule parser, so invalid PromQL expressions and durations are rejected.
+
+    Return error message if invalid, None if valid.
+
+    """
+    if not config.strip():
+        return None
+
+    try:
+        data = yaml.safe_load(config)
+    except yaml.YAMLError as error:
+        return f"alert_rules is not valid YAML: {error}"
+
+    if not isinstance(data, dict) or not data.get("groups"):
+        return "alert_rules must be a Prometheus rules document with a top-level 'groups' key."
+
+    valid, errors = CosTool("promql").validate_alert_rules(cast(OfficialRuleFileFormat, data))
+    if not valid:
+        return f"alert_rules failed Prometheus validation: {errors}"
+
+    return None

tests/functional/tests/charm_tests/openstack_exporter.py

@@ -184,6 +184,41 @@ def test_configure_ssl_ca(self):
         model.set_application_config(APP_NAME, {"cache": "true"})
         model.block_until_all_units_idle()
 
+    def test_configure_alert_rules(self):
+        """Test overriding the shipped alert rules via the alert_rules config."""
+        key = "alert_rules"
+        override = (
+            "groups:\n"
+            "- name: FunctionalTest\n"
+            "  rules:\n"
+            "  - alert: FunctionalTestAlert\n"
+            "    expr: up == 0\n"
+            "    for: 2m\n"
+            "    labels:\n"
+            "      severity: critical\n"
+        )
+
+        # Rendered alert rules directory inside the charm dir on the unit.
+        unit_num = self.leader_unit_entity_id.split("/")[1]
+        rendered_dir = f"/var/lib/juju/agents/unit-{APP_NAME}-{unit_num}/charm/src/alert_rules"
+
+        # Set the override and verify the custom document replaces the shipped rules.
+        model.set_application_config(APP_NAME, {key: override})
+        model.block_until_all_units_idle()
+        model.block_until_file_has_contents(
+            APP_NAME, f"{rendered_dir}/custom.yaml", "FunctionalTestAlert"
+        )
+        results = model.run_on_leader(APP_NAME, f"ls {rendered_dir}")
+        self.assertEqual(results.get("Stdout", "").strip(), "custom.yaml")
+
+        # Reset config: alert_rules; the shipped rules should be restored.
+        model.reset_application_config(APP_NAME, [key])
+        model.block_until_file_missing(APP_NAME, f"{rendered_dir}/custom.yaml")
+        results = model.run_on_leader(APP_NAME, f"ls {rendered_dir}")
+        rendered_files = results.get("Stdout", "").strip()
+        self.assertNotIn("custom.yaml", rendered_files)
+        self.assertNotEqual(rendered_files, "")
+
 
 class OpenstackExporterStatusTest(OpenstackExporterBaseTest):
     """Test status changes for openstack exporter."""