From bc8fe40a65212dced743f98fe07c01acedbeafa1 Mon Sep 17 00:00:00 2001 From: Michael Thamm Date: Tue, 8 Sep 2026 17:06:12 -0400 Subject: [PATCH 1/2] feat: Compressed PRW rules --- .../v1/prometheus_remote_write.py | 197 +++++++++++++++++- .../unit/test_remote_write_compression.py | 125 +++++++++++ 2 files changed, 315 insertions(+), 7 deletions(-) create mode 100644 coordinator/tests/unit/test_remote_write_compression.py diff --git a/coordinator/lib/charms/prometheus_k8s/v1/prometheus_remote_write.py b/coordinator/lib/charms/prometheus_k8s/v1/prometheus_remote_write.py index b47a2bc..2d9b8af 100644 --- a/coordinator/lib/charms/prometheus_k8s/v1/prometheus_remote_write.py +++ b/coordinator/lib/charms/prometheus_k8s/v1/prometheus_remote_write.py @@ -11,6 +11,24 @@ should use the `PrometheusRemoteWriteConsumer`. Charms that operate software that exposes the Prometheus remote_write API, that is, they can receive metrics data over remote_write, should use the `PrometheusRemoteWriteProducer`. + +## Alert rules encoding + +The consumer publishes its alert rules to the `alert_rules` key of its application +databag. Because large deployments can produce enough alert rules to exceed Juju's +relation data size limit, the rules can be stored LZMA-compressed and base64-encoded +instead of as plain JSON. + +Compression is negotiated over the relation: the provider advertises the encodings it +is able to read in the `alert_rules_encodings` key of its own application databag, and +the consumer picks the best encoding both sides support. A consumer related to a +provider running an older version of this library (which advertises nothing) keeps +writing plain JSON, so upgrades are safe in any order. + +An admin can decode compressed rules with: +```bash + | base64 -d | xz -d | jq +``` """ import copy @@ -20,9 +38,9 @@ import re import socket from pathlib import Path -from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Union, cast -from cosl import CosTool, JujuTopology +from cosl import CosTool, JujuTopology, LZMABase64 from cosl.rules import HOST_METRICS_MISSING_RULE_NAME, AlertRules, generic_alert_groups from cosl.types import OfficialRuleFileFormat from ops.charm import ( @@ -44,7 +62,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 18 +LIBPATCH = 19 PYDEPS = ["cosl"] @@ -58,6 +76,72 @@ DEFAULT_ALERT_RULES_RELATIVE_PATH = "./src/prometheus_alert_rules" +ALERT_RULES_KEY = "alert_rules" +"""Databag key holding the consumer's alert rules.""" + +ALERT_RULES_ENCODINGS_KEY = "alert_rules_encodings" +"""Databag key with which the provider advertises the encodings it can read.""" + +JSON_ENCODING = "json" +"""Plain JSON alert rules, as written by every version of this library.""" + +LZMA_ENCODING = "lzma" +"""LZMA-compressed, base64-encoded JSON alert rules.""" + +SUPPORTED_ALERT_RULES_ENCODINGS = (LZMA_ENCODING, JSON_ENCODING) +"""Alert rules encodings this library can read and write, most preferred first.""" + + +def _encode_alert_rules(rules: Mapping[str, Any], encoding: str = JSON_ENCODING) -> str: + """Serialize alert rules for storing them in a relation databag. + + Args: + rules: alert rules in the official Prometheus rule file format. + encoding: one of `SUPPORTED_ALERT_RULES_ENCODINGS`. Anything else is treated + as `JSON_ENCODING`, because plain JSON is readable by every version of + this library. + + Returns: + The serialized alert rules. + """ + serialized = json.dumps(rules) + if encoding == LZMA_ENCODING: + return LZMABase64.compress(serialized) + return serialized + + +def _decode_alert_rules(raw: str) -> OfficialRuleFileFormat: + """Deserialize alert rules read from a relation databag. + + Both plain JSON and LZMA-compressed, base64-encoded JSON are accepted, regardless + of the encodings this library advertises, so that a provider can always read the + rules of a consumer running any version of this library. + + Args: + raw: the raw databag value. + + Returns: + The alert rules in the official Prometheus rule file format. + + Raises: + Exception: if `raw` is neither valid JSON nor a valid compressed payload. + """ + if not raw: + return cast(OfficialRuleFileFormat, {}) + + try: + decoded = json.loads(raw) + except json.JSONDecodeError: + # Not JSON, so this must be a compressed payload. + decoded = raw + + if isinstance(decoded, str): + # A compressed payload, either bare or (as pydantic based libraries write it) + # JSON-encoded. + decoded = json.loads(LZMABase64.decompress(decoded)) + + return cast(OfficialRuleFileFormat, decoded) + class RelationNotFoundError(Exception): """Raised if there is no relation with the given name.""" @@ -366,6 +450,10 @@ def __init__(self, *args): If the syntax of a rule is invalid, the `MetricsEndpointProvider` logs an error and does not load the particular rule. + The alert rules are published to the `alert_rules` key of this application's databag, + LZMA-compressed and base64-encoded if the provider advertises that it can read them + that way, and as plain JSON otherwise. See the module docstring for details. + To avoid false positives and false negatives the library will inject label filters automatically in the PromQL expression. For example if the charm provides an alert rule with an `expr` like this one: @@ -459,7 +547,10 @@ def __init__( self.framework.observe(on_relation.relation_changed, self._handle_endpoints_changed) self.framework.observe(on_relation.relation_departed, self._handle_endpoints_changed) self.framework.observe(on_relation.relation_broken, self._on_relation_broken) - self.framework.observe(on_relation.relation_joined, self._push_alerts_on_relation_joined) + self.framework.observe(on_relation.relation_joined, self._push_alerts_on_relation_event) + # The provider advertises the alert rules encodings it supports over relation data, + # so alerts are (re)pushed on relation-changed to pick up the negotiated encoding. + self.framework.observe(on_relation.relation_changed, self._push_alerts_on_relation_event) self.framework.observe( self._charm.on.leader_elected, self._push_alerts_to_all_relation_databags ) @@ -490,13 +581,52 @@ def _handle_endpoints_changed(self, event: RelationEvent) -> None: self.on.endpoints_changed.emit(relation_id=event.relation.id) - def _push_alerts_on_relation_joined(self, event: RelationEvent) -> None: + def _push_alerts_on_relation_event(self, event: RelationEvent) -> None: self._push_alerts_to_relation_databag(event.relation) def _push_alerts_to_all_relation_databags(self, _: Optional[HookEvent]) -> None: for relation in self.model.relations[self._relation_name]: self._push_alerts_to_relation_databag(relation) + def _alert_rules_encoding(self, relation: Relation) -> str: + """Return the best alert rules encoding the provider on the other end can read. + + Providers advertise the encodings they support in their application databag. + Providers running an older version of this library advertise nothing, in which + case plain JSON is used for backwards compatibility. + + Args: + relation: the relation whose remote application databag to inspect. + + Returns: + One of `SUPPORTED_ALERT_RULES_ENCODINGS`. + """ + if not relation.app: + return JSON_ENCODING + + if (remote_databag := relation.data.get(relation.app)) is None: + return JSON_ENCODING + + try: + advertised = json.loads(remote_databag.get(ALERT_RULES_ENCODINGS_KEY, "[]")) + except json.JSONDecodeError: + logger.warning( + "Could not parse the '%s' advertised over relation %s; " + "falling back to plain JSON alert rules.", + ALERT_RULES_ENCODINGS_KEY, + relation.id, + ) + return JSON_ENCODING + + if not isinstance(advertised, list): + return JSON_ENCODING + + for encoding in SUPPORTED_ALERT_RULES_ENCODINGS: + if encoding in advertised: + return encoding + + return JSON_ENCODING + def _push_alerts_to_relation_databag(self, relation: Relation) -> None: if not self._charm.unit.is_leader(): return @@ -526,7 +656,9 @@ def _push_alerts_to_relation_databag(self, relation: Relation) -> None: alert_rules_as_dict, self._extra_alert_labels ) ) - relation.data[self._charm.app]["alert_rules"] = json.dumps(alert_rules_as_dict) + relation.data[self._charm.app][ALERT_RULES_KEY] = _encode_alert_rules( + alert_rules_as_dict, self._alert_rules_encoding(relation) + ) def reload_alerts(self) -> None: """Reload alert rules from disk and push to relation data.""" @@ -756,6 +888,17 @@ def __init__( on_relation.relation_changed, self._on_relation_changed, ) + # Consumers only compress their alert rules if we advertise that we can read them, + # so make sure the advertisement is (re)published after an upgrade or a leadership + # change, when no relation event may fire. + self.framework.observe( + self._charm.on.leader_elected, + self._publish_encodings_to_all_relation_databags, + ) + self.framework.observe( + self._charm.on.upgrade_charm, + self._publish_encodings_to_all_relation_databags, + ) def _on_consumers_changed(self, event: RelationEvent) -> None: if not isinstance(event, RelationBrokenEvent): @@ -766,8 +909,31 @@ def _on_consumers_changed(self, event: RelationEvent) -> None: def _on_relation_changed(self, event: RelationEvent) -> None: """Flag Providers that data has changed, so they can re-read alerts.""" + self._publish_alert_rules_encodings(event.relation) self.on.alert_rules_changed.emit(event.relation.id) + def _publish_encodings_to_all_relation_databags(self, _: HookEvent) -> None: + for relation in self.model.relations[self._relation_name]: + self._publish_alert_rules_encodings(relation) + + def _publish_alert_rules_encodings(self, relation: Relation) -> None: + """Advertise the alert rules encodings this library is able to read. + + Consumers use this to decide whether they may compress their alert rules: a + consumer related to a provider that does not advertise anything keeps writing + plain JSON, which every version of this library can read. + + Args: + relation: The relation whose data to update. + """ + if not self._charm.unit.is_leader(): + # Only the leader unit can write to app data. + return + + relation.data[self._charm.app][ALERT_RULES_ENCODINGS_KEY] = json.dumps( + list(SUPPORTED_ALERT_RULES_ENCODINGS) + ) + def update_endpoint(self, relation: Optional[Relation] = None) -> None: """Triggers programmatically the update of the relation data. @@ -777,6 +943,9 @@ def update_endpoint(self, relation: Optional[Relation] = None) -> None: host address change because the charmed operator becomes connected to an Ingress after the `prometheus_remote_write` relation is established. + The alert rules encodings this library can read are advertised at the same + time, so that consumers know they may compress their alert rules. + Args: relation: An optional instance of `class:ops.model.Relation` to update. If not provided, all instances of the `prometheus_remote_write` @@ -786,6 +955,7 @@ def update_endpoint(self, relation: Optional[Relation] = None) -> None: for relation in relations: self._set_endpoint_on_relation(relation) + self._publish_alert_rules_encodings(relation) def _set_endpoint_on_relation(self, relation: Relation) -> None: """Set the remote_write endpoint on relations. @@ -835,7 +1005,20 @@ def alerts(self) -> dict: if not relation.units or not relation.app: continue - alert_rules = json.loads(relation.data[relation.app].get("alert_rules", "{}")) + try: + alert_rules = _decode_alert_rules( + relation.data[relation.app].get(ALERT_RULES_KEY, "{}") + ) + except Exception as e: + # Never let unreadable remote data break the provider: a consumer could + # be writing rules in a format this version of the library predates. + logger.error( + "Could not read the alert rules published over relation %s: %s", + relation.id, + e, + ) + continue + if not alert_rules: continue diff --git a/coordinator/tests/unit/test_remote_write_compression.py b/coordinator/tests/unit/test_remote_write_compression.py new file mode 100644 index 0000000..1fee668 --- /dev/null +++ b/coordinator/tests/unit/test_remote_write_compression.py @@ -0,0 +1,125 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Feature: compression of the alert rules received over receive-remote-write. + +The coordinator advertises the alert rule encodings it can read, so that requirers know +they may compress large rule sets, and ingests the rules in either encoding. +""" + +import json + +import pytest +import yaml +from charms.prometheus_k8s.v1.prometheus_remote_write import ( + ALERT_RULES_ENCODINGS_KEY, + ALERT_RULES_KEY, + JSON_ENCODING, + LZMA_ENCODING, +) +from cosl.utils import LZMABase64 +from scenario import Container, Exec, Relation, State + +ALERT_RULES = { + "groups": [ + { + "name": "compressed-remote-write-group", + "rules": [ + { + "alert": "CompressedRuleFiring", + "expr": 'sum(rate({job="valid"}[5m])) > 0', + "for": "1m", + "labels": {"severity": "warning"}, + "annotations": {"summary": "compressed"}, + } + ], + } + ] +} + +METADATA = json.dumps( + { + "model": "test", + "model_uuid": "20ce8299-3634-4bef-8bd8-5ace6c8816b4", + "application": "remote-write-compressed", + "charm_name": "remote-write-compressed-charm", + } +) + + +def _nginx_container_for_alert_rules(): + return Container( + "nginx", + can_connect=True, + execs={ + Exec(["mimirtool", "rules", "sync"], return_code=0), + Exec(["update-ca-certificates", "--fresh"], return_code=0), + Exec(["nginx", "-s", "reload"], return_code=0), + }, + ) + + +def _written_group_names(context, state_out): + fs = state_out.get_container("nginx").get_filesystem(context) + rules_dir = fs.joinpath("etc", "mimir-alerts", "rules") + if not rules_dir.exists(): + return set() + + written_group_names = set() + for rule_file in sorted(path for path in rules_dir.iterdir() if path.is_file()): + written_rules = yaml.safe_load(rule_file.read_text()) + for group in written_rules["groups"]: + written_group_names.add(group["name"]) + return written_group_names + + +@pytest.mark.parametrize( + "published", + [ + pytest.param(json.dumps(ALERT_RULES), id="plain_json"), + pytest.param(LZMABase64.compress(json.dumps(ALERT_RULES)), id="compressed"), + ], +) +def test_alert_rules_are_ingested_in_either_encoding( + context, s3, all_worker, nginx_prometheus_exporter_container, published +): + # GIVEN a requirer that published its alert rules, compressed or not + remote_write_relation = Relation( + "receive-remote-write", + remote_app_name="remote-write-compressed", + remote_app_data={ALERT_RULES_KEY: published, "scrape_metadata": METADATA}, + ) + state_in = State( + leader=True, + relations=[s3, all_worker, remote_write_relation], + containers=[_nginx_container_for_alert_rules(), nginx_prometheus_exporter_container], + ) + + # WHEN the relation changed event is processed + state_out = context.run(context.on.relation_changed(remote_write_relation), state_in) + + # THEN the rules are written to disk regardless of the encoding they arrived in + assert _written_group_names(context, state_out) == {"compressed-remote-write-group"} + + +def test_coordinator_advertises_the_encodings_it_can_read( + context, s3, all_worker, nginx_prometheus_exporter_container +): + # GIVEN a requirer related over receive-remote-write + remote_write_relation = Relation( + "receive-remote-write", remote_app_name="remote-write-compressed" + ) + state_in = State( + leader=True, + relations=[s3, all_worker, remote_write_relation], + containers=[_nginx_container_for_alert_rules(), nginx_prometheus_exporter_container], + ) + + # WHEN the relation joined event is processed + state_out = context.run(context.on.relation_joined(remote_write_relation), state_in) + + # THEN requirers are told that compressed alert rules are supported + advertised = state_out.get_relation(remote_write_relation.id).local_app_data[ + ALERT_RULES_ENCODINGS_KEY + ] + assert json.loads(advertised) == [LZMA_ENCODING, JSON_ENCODING] From ee2db52a4d88328f38bb705a19bceaad2e2b7ff0 Mon Sep 17 00:00:00 2001 From: Michael Thamm Date: Thu, 10 Sep 2026 16:54:14 -0400 Subject: [PATCH 2/2] chore --- .../v1/prometheus_remote_write.py | 186 +++++++++++------- 1 file changed, 118 insertions(+), 68 deletions(-) diff --git a/coordinator/lib/charms/prometheus_k8s/v1/prometheus_remote_write.py b/coordinator/lib/charms/prometheus_k8s/v1/prometheus_remote_write.py index 2d9b8af..4e15ae4 100644 --- a/coordinator/lib/charms/prometheus_k8s/v1/prometheus_remote_write.py +++ b/coordinator/lib/charms/prometheus_k8s/v1/prometheus_remote_write.py @@ -34,11 +34,12 @@ import copy import json import logging +import lzma import os import re import socket from pathlib import Path -from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Union, cast +from typing import Any, Callable, Dict, Final, List, Mapping, Optional, Set, Tuple, Union, cast from cosl import CosTool, JujuTopology, LZMABase64 from cosl.rules import HOST_METRICS_MISSING_RULE_NAME, AlertRules, generic_alert_groups @@ -76,25 +77,37 @@ DEFAULT_ALERT_RULES_RELATIVE_PATH = "./src/prometheus_alert_rules" -ALERT_RULES_KEY = "alert_rules" +ALERT_RULES_KEY: Final[str] = "alert_rules" """Databag key holding the consumer's alert rules.""" -ALERT_RULES_ENCODINGS_KEY = "alert_rules_encodings" +ALERT_RULES_ENCODINGS_KEY: Final[str] = "alert_rules_encodings" """Databag key with which the provider advertises the encodings it can read.""" -JSON_ENCODING = "json" +JSON_ENCODING: Final[str] = "json" """Plain JSON alert rules, as written by every version of this library.""" -LZMA_ENCODING = "lzma" +LZMA_ENCODING: Final[str] = "lzma" """LZMA-compressed, base64-encoded JSON alert rules.""" -SUPPORTED_ALERT_RULES_ENCODINGS = (LZMA_ENCODING, JSON_ENCODING) -"""Alert rules encodings this library can read and write, most preferred first.""" +SUPPORTED_ALERT_RULES_ENCODINGS: Final[Tuple[str, ...]] = (LZMA_ENCODING, JSON_ENCODING) +"""Alert rules encodings this library can read and write, most preferred first. + +This is in preference order, not sorted: it is a constant, so the bytes written to the +databag are stable across hooks, which is what matters for avoiding spurious +relation-changed events. +""" + +_LZMA_BASE64_PREFIX: Final[str] = "/Td6WFoA" +"""Base64 of the xz magic bytes, b"\\xfd7zXZ\\x00", every compressed payload starts with.""" def _encode_alert_rules(rules: Mapping[str, Any], encoding: str = JSON_ENCODING) -> str: """Serialize alert rules for storing them in a relation databag. + Keys are sorted so that the same rules always serialize to the same bytes: juju + compares databag values byte for byte, so an unstable key order would trigger a + spurious relation-changed on the other side of the relation on every hook. + Args: rules: alert rules in the official Prometheus rule file format. encoding: one of `SUPPORTED_ALERT_RULES_ENCODINGS`. Anything else is treated @@ -104,12 +117,50 @@ def _encode_alert_rules(rules: Mapping[str, Any], encoding: str = JSON_ENCODING) Returns: The serialized alert rules. """ - serialized = json.dumps(rules) + serialized = json.dumps(rules, sort_keys=True) if encoding == LZMA_ENCODING: return LZMABase64.compress(serialized) return serialized +def _best_alert_rules_encoding(remote_app_databag: Optional[Mapping[str, str]]) -> str: + """Return the best alert rules encoding the remote app is able to read. + + Providers advertise the encodings they support in their application databag. + Providers running an older version of this library advertise nothing, in which + case plain JSON is used for backwards compatibility. + + Args: + remote_app_databag: the remote application databag, or None if it is not + readable yet (e.g. the relation is still being set up). + + Returns: + One of `SUPPORTED_ALERT_RULES_ENCODINGS`. + """ + raw = remote_app_databag.get(ALERT_RULES_ENCODINGS_KEY, "[]") if remote_app_databag else "[]" + + try: + advertised = json.loads(raw) + if not isinstance(advertised, list): + raise TypeError("expected a list, got {}".format(type(advertised).__name__)) + except (json.JSONDecodeError, TypeError) as e: + logger.warning( + "Ignoring malformed '%s' (%s); assuming the remote end is only able to read " + "uncompressed alert rules.", + ALERT_RULES_ENCODINGS_KEY, + e, + ) + return JSON_ENCODING + + for encoding in SUPPORTED_ALERT_RULES_ENCODINGS: + if encoding in advertised: + return encoding + + # Either nothing was advertised (an older provider), or only encodings this library + # does not know about. Plain JSON is the encoding every version can read. + return JSON_ENCODING + + def _decode_alert_rules(raw: str) -> OfficialRuleFileFormat: """Deserialize alert rules read from a relation databag. @@ -124,7 +175,7 @@ def _decode_alert_rules(raw: str) -> OfficialRuleFileFormat: The alert rules in the official Prometheus rule file format. Raises: - Exception: if `raw` is neither valid JSON nor a valid compressed payload. + ValueError: if `raw` holds neither alert rules nor a compressed payload of them. """ if not raw: return cast(OfficialRuleFileFormat, {}) @@ -132,13 +183,28 @@ def _decode_alert_rules(raw: str) -> OfficialRuleFileFormat: try: decoded = json.loads(raw) except json.JSONDecodeError: - # Not JSON, so this must be a compressed payload. + # Not JSON at all, so this can only be a bare compressed payload. decoded = raw if isinstance(decoded, str): # A compressed payload, either bare or (as pydantic based libraries write it) # JSON-encoded. - decoded = json.loads(LZMABase64.decompress(decoded)) + if not decoded.startswith(_LZMA_BASE64_PREFIX): + raise ValueError( + "Expected either alert rules or an LZMA-compressed, base64-encoded" + " payload of them, got the string {!r:.60}".format(decoded) + ) + try: + decoded = json.loads(LZMABase64.decompress(decoded)) + except (ValueError, lzma.LZMAError) as e: + # ValueError covers both a malformed base64 payload (binascii.Error) and + # compressed content that is not JSON (json.JSONDecodeError). + raise ValueError("Could not decompress the alert rules: {}".format(e)) from e + + if not isinstance(decoded, dict): + raise ValueError( + "Alert rules must be a JSON object, not {}".format(type(decoded).__name__) + ) return cast(OfficialRuleFileFormat, decoded) @@ -548,8 +614,6 @@ def __init__( self.framework.observe(on_relation.relation_departed, self._handle_endpoints_changed) self.framework.observe(on_relation.relation_broken, self._on_relation_broken) self.framework.observe(on_relation.relation_joined, self._push_alerts_on_relation_event) - # The provider advertises the alert rules encodings it supports over relation data, - # so alerts are (re)pushed on relation-changed to pick up the negotiated encoding. self.framework.observe(on_relation.relation_changed, self._push_alerts_on_relation_event) self.framework.observe( self._charm.on.leader_elected, self._push_alerts_to_all_relation_databags @@ -588,45 +652,6 @@ def _push_alerts_to_all_relation_databags(self, _: Optional[HookEvent]) -> None: for relation in self.model.relations[self._relation_name]: self._push_alerts_to_relation_databag(relation) - def _alert_rules_encoding(self, relation: Relation) -> str: - """Return the best alert rules encoding the provider on the other end can read. - - Providers advertise the encodings they support in their application databag. - Providers running an older version of this library advertise nothing, in which - case plain JSON is used for backwards compatibility. - - Args: - relation: the relation whose remote application databag to inspect. - - Returns: - One of `SUPPORTED_ALERT_RULES_ENCODINGS`. - """ - if not relation.app: - return JSON_ENCODING - - if (remote_databag := relation.data.get(relation.app)) is None: - return JSON_ENCODING - - try: - advertised = json.loads(remote_databag.get(ALERT_RULES_ENCODINGS_KEY, "[]")) - except json.JSONDecodeError: - logger.warning( - "Could not parse the '%s' advertised over relation %s; " - "falling back to plain JSON alert rules.", - ALERT_RULES_ENCODINGS_KEY, - relation.id, - ) - return JSON_ENCODING - - if not isinstance(advertised, list): - return JSON_ENCODING - - for encoding in SUPPORTED_ALERT_RULES_ENCODINGS: - if encoding in advertised: - return encoding - - return JSON_ENCODING - def _push_alerts_to_relation_databag(self, relation: Relation) -> None: if not self._charm.unit.is_leader(): return @@ -656,8 +681,9 @@ def _push_alerts_to_relation_databag(self, relation: Relation) -> None: alert_rules_as_dict, self._extra_alert_labels ) ) + remote_app_databag = relation.data.get(relation.app) if relation.app else None relation.data[self._charm.app][ALERT_RULES_KEY] = _encode_alert_rules( - alert_rules_as_dict, self._alert_rules_encoding(relation) + alert_rules_as_dict, _best_alert_rules_encoding(remote_app_databag) ) def reload_alerts(self) -> None: @@ -931,7 +957,7 @@ def _publish_alert_rules_encodings(self, relation: Relation) -> None: return relation.data[self._charm.app][ALERT_RULES_ENCODINGS_KEY] = json.dumps( - list(SUPPORTED_ALERT_RULES_ENCODINGS) + SUPPORTED_ALERT_RULES_ENCODINGS ) def update_endpoint(self, relation: Optional[Relation] = None) -> None: @@ -1001,6 +1027,7 @@ def alerts(self) -> dict: a dictionary mapping the name of an alert rule group to the group. """ alerts: Dict[str, OfficialRuleFileFormat] = {} + unreadable: Dict[int, str] = {} for relation in self._charm.model.relations[self._relation_name]: if not relation.units or not relation.app: continue @@ -1010,12 +1037,11 @@ def alerts(self) -> dict: relation.data[relation.app].get(ALERT_RULES_KEY, "{}") ) except Exception as e: - # Never let unreadable remote data break the provider: a consumer could - # be writing rules in a format this version of the library predates. - logger.error( - "Could not read the alert rules published over relation %s: %s", - relation.id, - e, + # Reported like a validation error, so that the charm blocks on it instead + # of silently dropping the consumer's alert rules. + unreadable[relation.id] = str(e) + self._report_alert_rules_error( + relation, "Could not decode the published alert rules: {}".format(e) ) continue @@ -1050,18 +1076,42 @@ def alerts(self) -> dict: logger.error(f"Invalid alert rule file: {errmsg}") if alerts[identifier]: del alerts[identifier] - if self._charm.unit.is_leader(): - data = json.loads(relation.data[self._charm.app].get("event", "{}")) - data["errors"] = errmsg - relation.data[self._charm.app]["event"] = json.dumps(data) + self._report_alert_rules_error(relation, errmsg) continue - if self._charm.unit.is_leader(): - data = json.loads(relation.data[self._charm.app].get("event", "{}")) - data.pop("errors", None) - relation.data[self._charm.app]["event"] = json.dumps(data) + self._report_alert_rules_error(relation, None) + + if unreadable: + logger.error( + "Could not read the alert rules published over relation(s): %s", + "; ".join("{} ({})".format(rel_id, err) for rel_id, err in unreadable.items()), + ) return alerts + def _report_alert_rules_error(self, relation: Relation, errmsg: Optional[str]) -> None: + """Report, or clear, an alert rules error for a relation. + + The error is written to the `event` key of this application's databag, from where + `has_invalid_alert_rules` reads it back so the charm can block on it, and where the + consumer picks it up as an `alert_rule_status_changed` event. + + Args: + relation: the relation the error pertains to. + errmsg: the error to report, or None to clear a previously reported one. + """ + if not self._charm.unit.is_leader(): + return + + data = json.loads(relation.data[self._charm.app].get("event", "{}")) + if errmsg: + data["errors"] = errmsg + elif "errors" not in data: + return + else: + data.pop("errors") + + relation.data[self._charm.app]["event"] = json.dumps(data, sort_keys=True) + def _get_identifier_by_alert_rules( self, rules: OfficialRuleFileFormat ) -> Tuple[Union[str, None], Union[JujuTopology, None]]: