diff --git a/charmcraft.yaml b/charmcraft.yaml index 0fa8366..df3a87d 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -16,11 +16,38 @@ parts: - astral-uv build-packages: - git + # Bundle cos-tool so the charm can validate the alert_rules config with + # Prometheus' own rule parser at runtime. + cos-tool: + plugin: dump + source: https://github.com/canonical/cos-tool/releases/latest/download/cos-tool-${CRAFT_ARCH_BUILD_FOR} + source-type: file + permissions: + - path: cos-tool-${CRAFT_ARCH_BUILD_FOR} + mode: "755" platforms: ubuntu@22.04:amd64: ubuntu@24.04:amd64: +actions: + get-alert-rules: + description: | + Dump the alert rules currently in effect as a single Prometheus rules + document (YAML), returned in the `alert-rules` result. Use this as a starting + point for the `alert_rules` config option: dump the rules, edit them, + then set the edited document as the `alert_rules` config. + + Because `juju run` wraps the result under the unit name, extract the raw + document before saving. For example: + + juju run openstack-exporter/leader get-alert-rules --format=json \ + | jq -r 'first(.[]).results."alert-rules"' > rules.yaml + + Edit rules.yaml, then apply it: + + juju config openstack-exporter alert_rules="$(cat rules.yaml)" + requires: credentials: interface: keystone-admin @@ -76,6 +103,31 @@ config: This option allows the selection of a different channel. If the snap file has been attached via the openstack-exporter resource, this option has no effect. + alert_rules: + default: "" + type: string + description: | + A full Prometheus alert rules document (YAML) that replaces the alert rules shipped + with the charm. Empty string (default) keeps the shipped rules. + + Use this to customise severities, thresholds and `for:` durations, or to add or remove + alerts. The value must be a valid Prometheus rules file with a top-level `groups` key. + For example: + + groups: + - name: NovaCompute + rules: + - alert: NovaComputeDown + expr: openstack_nova_agent_state{adminState="enabled"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: Nova Compute Agent Down + + Warning: Changing this option reloads the alert rules in Prometheus. + Any currently firing alerts that are removed by the new rules will be canceled + and reported to Alert Manager as resolved. charm-libs: - lib: grafana-agent.cos_agent diff --git a/src/charm.py b/src/charm.py index f6cc36c..986a4f1 100755 --- a/src/charm.py +++ b/src/charm.py @@ -20,11 +20,18 @@ from ops.model import ActiveStatus, BlockedStatus, ModelError, WaitingStatus from service import SNAP_NAME, UPSTREAM_SNAP, get_installed_snap_service, snap_install_or_refresh -from validate_config import validate_cache_ttl, validate_port +from validate_config import ( + validate_alert_rules, + validate_cache_ttl, + validate_port, +) logger = logging.getLogger(__name__) RESOURCE_NAME = "openstack-exporter" +# Alert rules shipped with the charm, and the directory COSAgentProvider reads rendered rules from. +SHIPPED_ALERT_RULES_DIRNAME = "prometheus_alert_rules" +ALERT_RULES_DIRNAME = "alert_rules" # Snap config options global constants # This is to match between openstack-exporter and the entry in clouds.yaml CLOUD_NAME = "openstack" @@ -42,13 +49,8 @@ def __init__(self, *args: Any) -> None: """Initialize the charm.""" super().__init__(*args) - self._grafana_agent = COSAgentProvider( - self, - metrics_endpoints=[ - {"path": "/metrics", "port": self.config["port"]}, - ], - ) - + # Register _configure before COSAgentProvider so rendered alert rules are written + # to disk before COSAgentProvider reads them on config_changed. self.framework.observe(self.on.install, self._on_install) self.framework.observe(self.on.upgrade_charm, self._on_upgrade) self.framework.observe(self.on.config_changed, self._configure) @@ -57,6 +59,15 @@ def __init__(self, *args: Any) -> None: self.framework.observe(self.on.credentials_relation_broken, self._configure) self.framework.observe(self.on.cos_agent_relation_changed, self._configure) self.framework.observe(self.on.cos_agent_relation_broken, self._configure) + self.framework.observe(self.on.get_alert_rules_action, self._on_get_alert_rules) + + self._grafana_agent = COSAgentProvider( + self, + metrics_endpoints=[ + {"path": "/metrics", "port": self.config["port"]}, + ], + metrics_rules_dir=f"./src/{ALERT_RULES_DIRNAME}", + ) def _is_keystone_data_ready(self, data: dict[str, str]) -> bool: """Check if all the data is available from keystone. @@ -156,6 +167,7 @@ def validate_configs(self) -> Optional[str]: validators: list[tuple[Callable, str]] = [ (validate_port, "port"), (validate_cache_ttl, "cache_ttl"), + (validate_alert_rules, "alert_rules"), ] for validator, config_key in validators: if error := validator(self.model.config[config_key]): @@ -164,6 +176,51 @@ def validate_configs(self) -> Optional[str]: # All config options are valid return None + def _render_alert_rules(self) -> None: + """Render alert rules for COSAgentProvider to read. + + Writes the user-provided `alert_rules` document when set, otherwise the alert rules + shipped with the charm. The rendered directory is rebuilt each time so unsetting the + option restores the shipped rules. + """ + src_dir = Path(self.charm_dir) / "src" + rendered_dir = src_dir / ALERT_RULES_DIRNAME + rendered_dir.mkdir(parents=True, exist_ok=True) + + # Clear previously rendered rules so unsetting the override restores the shipped rules. + for existing in rendered_dir.glob("*"): + existing.unlink() + + override = str(self.model.config["alert_rules"]) + if override.strip(): + (rendered_dir / "custom.yaml").write_text(override) + return + + for path in (src_dir / SHIPPED_ALERT_RULES_DIRNAME).glob("*.yaml"): + (rendered_dir / path.name).write_text(path.read_text()) + + def _effective_alert_rules(self) -> str: + """Return the currently effective alert rules as a single YAML document. + + Returns the user-provided `alert_rules` override when set, otherwise the shipped rules + merged into one document with a top-level `groups` key. + """ + override = str(self.model.config["alert_rules"]) + if override.strip(): + return override + + src_dir = Path(self.charm_dir) / "src" + groups: list = [] + for path in sorted((src_dir / SHIPPED_ALERT_RULES_DIRNAME).glob("*.yaml")): + doc = yaml.safe_load(path.read_text()) or {} + groups.extend(doc.get("groups", [])) + # width avoids backslash line-folding, allow_unicode avoids \uXXXX escapes. + return yaml.safe_dump({"groups": groups}, sort_keys=False, allow_unicode=True, width=4096) + + def _on_get_alert_rules(self, event: ops.ActionEvent) -> None: + """Dump the effective alert rules so they can be edited and set as config.""" + event.set_results({"alert-rules": self._effective_alert_rules()}) + def install(self) -> None: """Install the necessary resources for the charm.""" # If this fails, it's not recoverable. @@ -182,6 +239,7 @@ def _configure(self, _: ops.HookEvent) -> None: logger.error(config_error) return + self._render_alert_rules() self.install() if self._upstream_snap_present(): diff --git a/src/validate_config.py b/src/validate_config.py index 3e48ee1..38a8e5b 100644 --- a/src/validate_config.py +++ b/src/validate_config.py @@ -2,8 +2,15 @@ # See LICENSE file for licensing details. """Configuration validation functions.""" +import logging import re -from typing import Optional +from typing import Optional, cast + +import yaml +from cosl import CosTool +from cosl.cos_tool import OfficialRuleFileFormat + +logger = logging.getLogger(__name__) MAX_PORT = 65535 @@ -67,3 +74,33 @@ def validate_cache_ttl(cache_ttl: str) -> Optional[str]: ) return None + + +def validate_alert_rules(config: str) -> Optional[str]: + """Validate the alert_rules configuration. + + The value is a full Prometheus rules document that replaces the shipped alert rules. + Empty string (default) keeps the shipped rules and is valid. + + The document is validated with the bundled `cos-tool` binary via `cosl.CosTool`, which + uses Prometheus' own rule parser, so invalid PromQL expressions and durations are rejected. + + Return error message if invalid, None if valid. + + """ + if not config.strip(): + return None + + try: + data = yaml.safe_load(config) + except yaml.YAMLError as error: + return f"alert_rules is not valid YAML: {error}" + + if not isinstance(data, dict) or not data.get("groups"): + return "alert_rules must be a Prometheus rules document with a top-level 'groups' key." + + valid, errors = CosTool("promql").validate_alert_rules(cast(OfficialRuleFileFormat, data)) + if not valid: + return f"alert_rules failed Prometheus validation: {errors}" + + return None diff --git a/tests/functional/tests/charm_tests/openstack_exporter.py b/tests/functional/tests/charm_tests/openstack_exporter.py index 82c46f3..439ce6a 100644 --- a/tests/functional/tests/charm_tests/openstack_exporter.py +++ b/tests/functional/tests/charm_tests/openstack_exporter.py @@ -184,6 +184,41 @@ def test_configure_ssl_ca(self): model.set_application_config(APP_NAME, {"cache": "true"}) model.block_until_all_units_idle() + def test_configure_alert_rules(self): + """Test overriding the shipped alert rules via the alert_rules config.""" + key = "alert_rules" + override = ( + "groups:\n" + "- name: FunctionalTest\n" + " rules:\n" + " - alert: FunctionalTestAlert\n" + " expr: up == 0\n" + " for: 2m\n" + " labels:\n" + " severity: critical\n" + ) + + # Rendered alert rules directory inside the charm dir on the unit. + unit_num = self.leader_unit_entity_id.split("/")[1] + rendered_dir = f"/var/lib/juju/agents/unit-{APP_NAME}-{unit_num}/charm/src/alert_rules" + + # Set the override and verify the custom document replaces the shipped rules. + model.set_application_config(APP_NAME, {key: override}) + model.block_until_all_units_idle() + model.block_until_file_has_contents( + APP_NAME, f"{rendered_dir}/custom.yaml", "FunctionalTestAlert" + ) + results = model.run_on_leader(APP_NAME, f"ls {rendered_dir}") + self.assertEqual(results.get("Stdout", "").strip(), "custom.yaml") + + # Reset config: alert_rules; the shipped rules should be restored. + model.reset_application_config(APP_NAME, [key]) + model.block_until_file_missing(APP_NAME, f"{rendered_dir}/custom.yaml") + results = model.run_on_leader(APP_NAME, f"ls {rendered_dir}") + rendered_files = results.get("Stdout", "").strip() + self.assertNotIn("custom.yaml", rendered_files) + self.assertNotEqual(rendered_files, "") + class OpenstackExporterStatusTest(OpenstackExporterBaseTest): """Test status changes for openstack exporter.""" diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 04ae5f7..e074098 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -22,6 +22,15 @@ def setup_method(self, _): def teardown_method(self, _): self.harness.cleanup() + @pytest.fixture(autouse=True) + def isolate_charm_dir(self, tmp_path, mocker): + mocker.patch.object( + OpenstackExporterOperatorCharm, + "charm_dir", + new_callable=mock.PropertyMock, + return_value=tmp_path, + ) + @pytest.mark.parametrize( "config", [ @@ -606,6 +615,8 @@ def test_on_upgrade(self, mock_install): ("cache_ttl", "2m3.4s"), ("cache_ttl", "1h2m3s4ms5us6ns"), ("cache_ttl", "39h9m14s"), + ("alert_rules", "groups:\n- name: g\n rules:\n - alert: A\n expr: up == 0\n"), + ("alert_rules", ""), ], ) def test_config_change_with_valid_config(self, config_option, config_value, mocker): @@ -615,12 +626,16 @@ def test_config_change_with_valid_config(self, config_option, config_value, mock mock_get_installed_snap_service = mocker.patch("charm.get_installed_snap_service") mock_get_installed_snap_service.return_value = mocked_upstream_service mock_install = mocker.patch("charm.OpenstackExporterOperatorCharm.install") + mock_render_alert_rules = mocker.patch( + "charm.OpenstackExporterOperatorCharm._render_alert_rules" + ) mock_event = mock.MagicMock() self.harness.begin() self.harness.update_config({config_option: config_value}) # If valid config, install method can be called from _configure + mock_render_alert_rules.assert_called_once() mock_install.assert_called_once() # Status should be set to Active @@ -647,6 +662,8 @@ def test_config_change_with_valid_config(self, config_option, config_value, mock ("cache_ttl", ".s"), ("cache_ttl", "+.s"), ("cache_ttl", "1d"), + ("alert_rules", "foo: bar"), + ("alert_rules", "groups: [oops"), ], ) def test_config_change_with_invalid_config(self, config_option, config_value, mocker): @@ -660,6 +677,9 @@ def test_config_change_with_invalid_config(self, config_option, config_value, mo f"cache_ttl must be non-negative, non-zero, " f"and in correct pattern, got {config_value}" ) + elif config_option == "alert_rules": + validate_function = "charm.validate_alert_rules" + error_msg = f"alert_rules {config_value!r} is not valid" mock_event = mock.MagicMock() mock_logger = mocker.patch("charm.logger.error") @@ -672,3 +692,110 @@ def test_config_change_with_invalid_config(self, config_option, config_value, mo mock_logger.assert_called_once_with(error_msg) mock_event.add_status.assert_any_call(ops.BlockedStatus(error_msg)) + + def test_render_alert_rules_default_uses_shipped(self, tmp_path, mocker): + """Without an override, the shipped rules are rendered for COSAgentProvider.""" + mocker.patch("charm.get_installed_snap_service") + mocker.patch("charm.snap_install_or_refresh") + + src_dir = tmp_path / "src" + shipped_dir = src_dir / "prometheus_alert_rules" + shipped_dir.mkdir(parents=True) + rendered_dir = src_dir / "alert_rules" + rendered_dir.mkdir() + stale_rule = rendered_dir / "stale.yaml" + stale_rule.write_text("groups: []\n") + real_shipped_dir = Path("src/prometheus_alert_rules") + for src_file in real_shipped_dir.glob("*.yaml"): + (shipped_dir / src_file.name).write_text(src_file.read_text()) + + mocker.patch.object( + OpenstackExporterOperatorCharm, + "charm_dir", + new_callable=mock.PropertyMock, + return_value=tmp_path, + ) + + self.harness.begin() + self.harness.update_config({"cache_ttl": "100s"}) + + rendered_names = sorted(p.name for p in rendered_dir.glob("*.yaml")) + shipped_names = sorted(p.name for p in shipped_dir.glob("*.yaml")) + assert rendered_names == shipped_names + assert not stale_rule.exists() + + def test_render_alert_rules_override_replaces_shipped(self, tmp_path, mocker): + """An alert_rules override replaces the shipped rules with a single document.""" + mocker.patch("charm.get_installed_snap_service") + mocker.patch("charm.snap_install_or_refresh") + + src_dir = tmp_path / "src" + shipped_dir = src_dir / "prometheus_alert_rules" + shipped_dir.mkdir(parents=True) + real_shipped_dir = Path("src/prometheus_alert_rules") + for src_file in real_shipped_dir.glob("*.yaml"): + (shipped_dir / src_file.name).write_text(src_file.read_text()) + + mocker.patch.object( + OpenstackExporterOperatorCharm, + "charm_dir", + new_callable=mock.PropertyMock, + return_value=tmp_path, + ) + + self.harness.begin() + override = ( + "groups:\n" + "- name: Custom\n" + " rules:\n" + " - alert: NovaComputeDown\n" + " expr: openstack_nova_agent_state == 0\n" + " for: 2m\n" + ) + self.harness.update_config({"alert_rules": override}) + + rendered_dir = src_dir / "alert_rules" + rendered_files = list(rendered_dir.glob("*.yaml")) + assert [f.name for f in rendered_files] == ["custom.yaml"] + assert rendered_files[0].read_text() == override + + def test_get_alert_rules_action_default_merges_shipped(self, tmp_path): + """Without an override, the action dumps the shipped rules as one merged document.""" + import yaml + + from charm import validate_alert_rules + + src_dir = tmp_path / "src" + shipped_dir = src_dir / "prometheus_alert_rules" + shipped_dir.mkdir(parents=True) + real_shipped_dir = Path("src/prometheus_alert_rules") + expected_groups = [] + for src_file in sorted(real_shipped_dir.glob("*.yaml")): + (shipped_dir / src_file.name).write_text(src_file.read_text()) + expected_groups.extend(yaml.safe_load(src_file.read_text()).get("groups", [])) + + self.harness.begin() + results = self.harness.run_action("get-alert-rules").results + + dumped = yaml.safe_load(results["alert-rules"]) + assert dumped["groups"] == expected_groups + # The dumped document must itself be a valid value for the alert_rules config. + assert validate_alert_rules(results["alert-rules"]) is None + + def test_get_alert_rules_action_returns_override(self, mocker): + """With an override set, the action returns the override verbatim.""" + mocker.patch("charm.get_installed_snap_service") + mocker.patch("charm.snap_install_or_refresh") + override = ( + "groups:\n" + "- name: Custom\n" + " rules:\n" + " - alert: NovaComputeDown\n" + " expr: openstack_nova_agent_state == 0\n" + " for: 2m\n" + ) + self.harness.begin() + self.harness.update_config({"alert_rules": override}) + + results = self.harness.run_action("get-alert-rules").results + assert results["alert-rules"] == override diff --git a/tests/unit/test_validate_config.py b/tests/unit/test_validate_config.py index 1490f92..e335a30 100644 --- a/tests/unit/test_validate_config.py +++ b/tests/unit/test_validate_config.py @@ -2,7 +2,11 @@ # See LICENSE file for licensing details. import pytest -from validate_config import validate_cache_ttl, validate_port +from validate_config import ( + validate_alert_rules, + validate_cache_ttl, + validate_port, +) @pytest.mark.parametrize( @@ -115,3 +119,46 @@ def test_validate_cache_ttl_invalid(cache_ttl): """ assert validate_cache_ttl(cache_ttl) is not None + + +@pytest.mark.parametrize( + "config", + [ + "", + " ", + "groups:\n- name: g\n rules:\n - alert: A\n expr: up == 0\n", + ], +) +def test_validate_alert_rules_valid(config, mocker): + # Ensure the test does not depend on cos-tool being present. + mocker.patch("validate_config.CosTool.validate_alert_rules", return_value=(True, "")) + assert validate_alert_rules(config) is None + + +@pytest.mark.parametrize( + "config", + [ + "groups: [oops", # invalid YAML + "just a string", # not a mapping + "foo: bar", # no groups key + "groups: []", # empty groups + ], +) +def test_validate_alert_rules_invalid(config, mocker): + mocker.patch("validate_config.CosTool.validate_alert_rules", return_value=(True, "")) + assert validate_alert_rules(config) is not None + + +def test_validate_alert_rules_cos_tool_success(mocker): + mocker.patch("validate_config.CosTool.validate_alert_rules", return_value=(True, "")) + config = "groups:\n- name: g\n rules:\n - alert: A\n expr: up == 0\n" + assert validate_alert_rules(config) is None + + +def test_validate_alert_rules_cos_tool_failure(mocker): + mocker.patch( + "validate_config.CosTool.validate_alert_rules", + return_value=(False, "error validating: bad for"), + ) + config = "groups:\n- name: g\n rules:\n - alert: A\n expr: up == 0\n for: djdj\n" + assert validate_alert_rules(config) is not None