From f9c9f2079d760ec2be70567ff9ae9302a1bdd740 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Thu, 10 Sep 2026 19:08:40 +0200 Subject: [PATCH 1/3] feat: Alert rules compression --- .../lib/charms/loki_k8s/v1/loki_push_api.py | 209 +- .../tests/unit/test_alert_rule_filtering.py | 334 +++ .../lib/charms/loki_k8s/v1/loki_push_api.py | 2592 ----------------- 3 files changed, 534 insertions(+), 2601 deletions(-) delete mode 100644 worker/lib/charms/loki_k8s/v1/loki_push_api.py diff --git a/coordinator/lib/charms/loki_k8s/v1/loki_push_api.py b/coordinator/lib/charms/loki_k8s/v1/loki_push_api.py index 309ee12..3207d68 100644 --- a/coordinator/lib/charms/loki_k8s/v1/loki_push_api.py +++ b/coordinator/lib/charms/loki_k8s/v1/loki_push_api.py @@ -478,6 +478,24 @@ def _alert_rules_error(self, event): Units of consumer charm send their alert rules over app relation data using the `alert_rules` key. +## Alert rules encoding + +The consumer publishes its alert rules to the `alert_rules` key of its application +databag. Because large deployments can produce enough alert rules to exceed Juju's +relation data size limit, the rules can be stored LZMA-compressed and base64-encoded +instead of as plain JSON. + +Compression is negotiated over the relation: the provider advertises the encodings it +is able to read in the `alert_rules_encodings` key of its own application databag, and +the consumer picks the best encoding both sides support. A consumer related to a +provider running an older version of this library (which advertises nothing) keeps +writing plain JSON, so upgrades are safe in any order. + +An admin can decode compressed rules with: +```bash + | base64 -d | xz -d | jq +``` + ## Charm logging The `charms.loki_k8s.v0.charm_logging` library can be used in conjunction with this one to configure python's logging module to forward all logs to Loki via the loki-push-api interface. @@ -501,6 +519,7 @@ def __init__(self, ...): import copy import json import logging +import lzma import os import platform import re @@ -511,12 +530,12 @@ def __init__(self, ...): from hashlib import sha256 from io import BytesIO from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Dict, Final, List, Mapping, Optional, Tuple, Union, cast from urllib import request from urllib.error import URLError import yaml -from cosl import CosTool, JujuTopology +from cosl import CosTool, JujuTopology, LZMABase64 from cosl.rules import AlertRules from cosl.types import OfficialRuleFileFormat from ops.charm import ( @@ -544,7 +563,7 @@ def __init__(self, ...): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 32 +LIBPATCH = 34 PYDEPS = ["cosl"] @@ -593,6 +612,123 @@ def __init__(self, ...): HTTP_LISTEN_PORT_START = 9080 # even start port GRPC_LISTEN_PORT_START = 9095 # odd start port +ALERT_RULES_KEY: Final[str] = "alert_rules" +"""Databag key holding the consumer's alert rules.""" + +ALERT_RULES_ENCODINGS_KEY: Final[str] = "alert_rules_encodings" +"""Databag key with which the provider advertises the encodings it can read.""" + +JSON_ENCODING: Final[str] = "json" +"""Plain JSON alert rules, as written by every version of this library.""" + +LZMA_ENCODING: Final[str] = "lzma" +"""LZMA-compressed, base64-encoded JSON alert rules.""" + +SUPPORTED_ALERT_RULES_ENCODINGS: Final[Tuple[str, ...]] = (LZMA_ENCODING, JSON_ENCODING) +"""Alert rules encodings this library can read and write, most preferred first. + +This is in preference order, not sorted: it is a constant, so the bytes written to the +databag are stable across hooks, which is what matters for avoiding spurious +relation-changed events. +""" + + +def _encode_alert_rules(rules: Mapping[str, Any], encoding: str = JSON_ENCODING) -> str: + """Serialize alert rules for storing them in a relation databag. + + Args: + rules: alert rules in the official Loki rule file format. + encoding: one of `SUPPORTED_ALERT_RULES_ENCODINGS`. Anything else is treated + as `JSON_ENCODING`, because plain JSON is readable by every version of + this library. + + Returns: + The serialized alert rules. + """ + # Sort keys to prevent unnecessary relation-changed churn from key reordering. + serialized = json.dumps(rules, sort_keys=True) + if encoding == LZMA_ENCODING: + return LZMABase64.compress(serialized) + return serialized + + +def _decode_alert_rules(raw: str) -> OfficialRuleFileFormat: + """Deserialize alert rules read from a relation databag. + + Both plain JSON and LZMA-compressed, base64-encoded JSON are accepted, regardless + of the encodings this library advertises, so that a provider can always read the + rules of a consumer running any version of this library. + + Args: + raw: the raw databag value. + + Returns: + The alert rules in the official Loki rule file format. + + Raises: + ValueError: if `raw` is neither valid JSON nor a valid compressed payload, or if it + decodes to something other than a JSON object. + """ + if not raw: + return cast(OfficialRuleFileFormat, {}) + + try: + decoded = json.loads(raw) + except json.JSONDecodeError: + # Not JSON, so this must be a compressed payload. + decoded = raw + + if isinstance(decoded, str): + # A compressed payload, either bare or (as pydantic based libraries write it) + # JSON-encoded. + try: + decoded = json.loads(LZMABase64.decompress(decoded)) + except (ValueError, lzma.LZMAError) as e: + raise ValueError(f"Could not decompress alert rules: {e}") from e + + if not isinstance(decoded, dict): + raise ValueError(f"Alert rules must be a JSON object, not {type(decoded).__name__}") + + return cast(OfficialRuleFileFormat, decoded) + + +def _best_alert_rules_encoding(remote_app_databag: Optional[Mapping[str, str]]) -> str: + """Return the best alert rules encoding the remote application is able to read. + + Providers advertise the encodings they support in their application databag. + Providers running an older version of this library advertise nothing, in which + case plain JSON is used for backwards compatibility. + + Args: + remote_app_databag: the remote application databag, or None if it is not + readable yet (e.g. the relation is still being set up). + + Returns: + One of `SUPPORTED_ALERT_RULES_ENCODINGS`. + """ + raw = remote_app_databag.get(ALERT_RULES_ENCODINGS_KEY, "[]") if remote_app_databag else "[]" + + try: + advertised = json.loads(raw) + if not isinstance(advertised, list): + raise TypeError("expected a list, got {}".format(type(advertised).__name__)) + except (json.JSONDecodeError, TypeError) as e: + logger.warning( + "Ignoring malformed '%s' (%s); assuming the remote end is only able to read " + "uncompressed alert rules.", + ALERT_RULES_ENCODINGS_KEY, + e, + ) + return JSON_ENCODING + + for encoding in SUPPORTED_ALERT_RULES_ENCODINGS: + if encoding in advertised: + return encoding + + # Either nothing was advertised (an older provider), or only encodings this library + # does not know about. Plain JSON is the encoding every version can read. + return JSON_ENCODING + class LokiPushApiError(Exception): """Base class for errors raised by this module.""" @@ -940,6 +1076,13 @@ def __init__( self.framework.observe(events.relation_changed, self._on_logging_relation_changed) self.framework.observe(events.relation_departed, self._on_logging_relation_departed) self.framework.observe(events.relation_broken, self._on_logging_relation_broken) + # Consumers only compress their alert rules if we advertise that we can read them, + # so make sure the advertisement is (re)published after a leadership change, when no + # relation event may fire. + self.framework.observe( + self._charm.on.leader_elected, + self._publish_encodings_to_all_relation_databags, + ) def _on_lifecycle_event(self, _): # Upgrade event or other charm-level event @@ -968,6 +1111,7 @@ def _on_logging_relation_joined(self, event: RelationJoinedEvent): if self._charm.unit.is_leader(): event.relation.data[self._charm.app].update(self._promtail_binary_url) logger.debug("Saved promtail binary url: %s", self._promtail_binary_url) + self._publish_alert_rules_encodings(event.relation) def _on_logging_relation_changed(self, event: HookEvent): """Handle changes in related consumers. @@ -1046,6 +1190,7 @@ def _process_logging_relation_changed(self, relation: Relation) -> bool: """ relation.data[self._charm.unit]["public_address"] = socket.getfqdn() or "" self.update_endpoint(relation=relation) + self._publish_alert_rules_encodings(relation) # Ensure promtail binary URL is set in app data. This is normally done on # relation_joined, but charms using the reconcile pattern may miss that event @@ -1056,6 +1201,28 @@ def _process_logging_relation_changed(self, relation: Relation) -> bool: return self._should_update_alert_rules(relation) + def _publish_encodings_to_all_relation_databags(self, _: Optional[HookEvent]) -> None: + for relation in self._charm.model.relations[self._relation_name]: + self._publish_alert_rules_encodings(relation) + + def _publish_alert_rules_encodings(self, relation: Relation) -> None: + """Advertise the alert rules encodings this library is able to read. + + Consumers use this to decide whether they may compress their alert rules: a + consumer related to a provider that does not advertise anything keeps writing + plain JSON, which every version of this library can read. + + Args: + relation: The relation whose data to update. + """ + if not self._charm.unit.is_leader(): + # Only the leader unit can write to app data. + return + + relation.data[self._charm.app][ALERT_RULES_ENCODINGS_KEY] = json.dumps( + list(SUPPORTED_ALERT_RULES_ENCODINGS) + ) + @property def _promtail_binary_url(self) -> dict: """URL from which Promtail binary can be downloaded.""" @@ -1103,6 +1270,7 @@ def update_endpoint(self, url: str = "", relation: Optional[Relation] = None) -> for relation in relations_list: relation.data[self._charm.unit].update({"endpoint": json.dumps(endpoint)}) + self._publish_alert_rules_encodings(relation) logger.debug("Saved endpoint in unit relation data") @@ -1159,15 +1327,25 @@ def alerts(self) -> dict: # noqa: C901 metadata indexed by relation ID. """ alerts = {} # type: Dict[str, dict] # mapping b/w juju identifiers and alert rule files + unreadable: Dict[int, str] = {} for relation in self._charm.model.relations[self._relation_name]: if not relation.units or not relation.app: continue - alert_rules = json.loads(relation.data[relation.app].get("alert_rules", "{}")) + try: + alert_rules = _decode_alert_rules( + relation.data[relation.app].get(ALERT_RULES_KEY, "{}") + ) + except Exception as e: + # Never let unreadable remote data break the provider: a consumer could + # be writing rules in a format this version of the library predates. + unreadable[relation.id] = str(e) + continue + if not alert_rules: continue - alert_rules = self._inject_alert_expr_labels(alert_rules) + alert_rules = self._inject_alert_expr_labels(cast(Dict[str, Any], alert_rules)) identifier, topology = self._get_identifier_by_alert_rules(alert_rules) if not topology: @@ -1208,6 +1386,12 @@ def alerts(self) -> dict: # noqa: C901 alerts[identifier] = alert_rules + if unreadable: + logger.error( + "Could not read the alert rules published over relation(s): %s", + "; ".join("{} ({})".format(rel_id, err) for rel_id, err in unreadable.items()), + ) + return alerts def has_invalid_alert_rules(self) -> bool: @@ -1404,9 +1588,9 @@ def _handle_alert_rules(self, relation): ) relation.data[self._charm.app]["metadata"] = json.dumps(self.topology.as_dict()) - relation.data[self._charm.app]["alert_rules"] = json.dumps( - alert_rules_as_dict, - sort_keys=True, # sort, to prevent unnecessary relation_changed events + remote_app_databag = relation.data.get(relation.app) if relation.app else None + relation.data[self._charm.app][ALERT_RULES_KEY] = _encode_alert_rules( + alert_rules_as_dict, _best_alert_rules_encoding(remote_app_databag) ) @property @@ -1424,7 +1608,9 @@ def loki_endpoints(self) -> List[dict]: seen_urls = set() for relation in self._charm.model.relations[self._relation_name]: - for unit in relation.units: + # Sort the units so the endpoints list order is stable across runs, + # otherwise the generated promtail config flaps. + for unit in sorted(relation.units, key=lambda u: u.name): if unit.app == self._charm.app: continue @@ -1589,6 +1775,11 @@ def _on_logging_relation_changed(self, event: RelationEvent): loki_push_api_alert_rules_error: This event is emitted when an invalid alert rules file is encountered or if `alert_rules_path` is empty. """ + # The provider advertises the alert rules encodings it supports over relation data, + # which may only become known after relation_joined; (re)send alert rules here so the + # negotiated encoding is picked up. + self._handle_alert_rules(event.relation) # pyright: ignore + if self._charm.unit.is_leader(): ev = json.loads(event.relation.data[event.app].get("event", "{}")) diff --git a/coordinator/tests/unit/test_alert_rule_filtering.py b/coordinator/tests/unit/test_alert_rule_filtering.py index f5c4cfa..3bdacbf 100644 --- a/coordinator/tests/unit/test_alert_rule_filtering.py +++ b/coordinator/tests/unit/test_alert_rule_filtering.py @@ -6,7 +6,16 @@ import socket from unittest.mock import patch +import pytest import yaml +from charms.loki_k8s.v1.loki_push_api import ( + ALERT_RULES_ENCODINGS_KEY, + JSON_ENCODING, + LZMA_ENCODING, + SUPPORTED_ALERT_RULES_ENCODINGS, + _best_alert_rules_encoding, + _encode_alert_rules, +) from cosl import JujuTopology from ops.model import ActiveStatus, BlockedStatus from scenario import Container, Exec, Relation, State @@ -237,3 +246,328 @@ def test_invalid_relation_becoming_valid_recovers_to_active( assert not _relation_errors(recovered_relation) assert _written_group_names(context, recovered_state) == {"valid-group"} assert isinstance(recovered_state.unit_status, ActiveStatus) + + +COMPRESSED_ALERT_RULES_RELATION = Relation( + "logging", + remote_app_name="app-compressed", + remote_app_data={ + "alert_rules": _encode_alert_rules( + json.loads(_alert_rules("compressed-group")), LZMA_ENCODING + ), + "metadata": _metadata("app-compressed"), + }, +) + + +def test_alerts_decodes_lzma_compressed_payload( + context, s3, all_worker, nginx_prometheus_exporter_container +): + # GIVEN a relation whose alert_rules payload is LZMA-compressed + nginx_container = _nginx_container_with_lokitool_sync(_rule_path("app-compressed")) + state_in = State( + relations=[s3, all_worker, COMPRESSED_ALERT_RULES_RELATION], + containers=[nginx_container, nginx_prometheus_exporter_container], + leader=True, + ) + + # WHEN the relation changed event is processed + state_out = context.run( + context.on.relation_changed(COMPRESSED_ALERT_RULES_RELATION), state_in + ) + + # THEN the compressed rules are decoded and written, and the unit remains active + assert _written_group_names(context, state_out) == {"compressed-group"} + assert isinstance(state_out.unit_status, ActiveStatus) + + +def test_alerts_skips_corrupt_alert_rules_and_logs_error( + context, s3, all_worker, nginx_prometheus_exporter_container, caplog +): + # GIVEN a relation with a corrupt/garbage alert_rules value + corrupt_relation = Relation( + "logging", + remote_app_name="app-corrupt", + remote_app_data={ + "alert_rules": "not valid json, nor valid lzma/base64", + "metadata": _metadata("app-corrupt"), + }, + ) + nginx_container = _nginx_container_with_lokitool_sync() + state_in = State( + relations=[s3, all_worker, corrupt_relation], + containers=[nginx_container, nginx_prometheus_exporter_container], + leader=True, + ) + + # WHEN the relation changed event is processed + # THEN it must not raise, and no rules are written + state_out = context.run(context.on.relation_changed(corrupt_relation), state_in) + assert _written_group_names(context, state_out) == set() + assert "Could not read the alert rules published over relation" in caplog.text + + +def test_alerts_skips_unreadable_relation_but_returns_others( + context, s3, all_worker, nginx_prometheus_exporter_container, caplog +): + """One relation with unreadable alert_rules doesn't prevent others from being read. + + Regression test for the log-aggregation change: a single malformed relation is + collected and logged once, without affecting unrelated, healthy relations. + """ + # GIVEN one relation with corrupt alert_rules and one with valid alert_rules + corrupt_relation = Relation( + "logging", + remote_app_name="app-corrupt", + remote_app_data={ + "alert_rules": "not valid json, nor valid lzma/base64", + "metadata": _metadata("app-corrupt"), + }, + ) + nginx_container = _nginx_container_with_lokitool_sync(_rule_path("app-valid")) + state_in = State( + relations=[s3, all_worker, corrupt_relation, VALID_RELATION], + containers=[nginx_container, nginx_prometheus_exporter_container], + leader=True, + ) + + # WHEN the relation changed event is processed for the valid relation + state_out = context.run(context.on.relation_changed(VALID_RELATION), state_in) + + # THEN the valid relation's rules are still written... + assert _written_group_names(context, state_out) == {"valid-group"} + assert isinstance(state_out.unit_status, ActiveStatus) + # ...and the corrupt one is reported, not silently dropped. + assert "Could not read the alert rules published over relation" in caplog.text + + +def test_alerts_skips_double_json_encoded_payload_and_logs_error( + context, s3, all_worker, nginx_prometheus_exporter_container, caplog +): + """A plain JSON *string* (not a compressed payload) is rejected with a clear error. + + Regression test for https://github.com/canonical/prometheus-k8s-operator/pull/864 + review feedback from @Abuelodelanada: ``json.loads('"foo"')`` returns the Python + string ``"foo"``, which then falls into the "this must be a compressed payload" + branch and fails to decompress. This must not raise an unhandled/opaque exception. + """ + # GIVEN a relation whose alert_rules value is a JSON string literal + double_encoded_relation = Relation( + "logging", + remote_app_name="app-double-encoded", + remote_app_data={ + "alert_rules": json.dumps("foo"), + "metadata": _metadata("app-double-encoded"), + }, + ) + nginx_container = _nginx_container_with_lokitool_sync() + state_in = State( + relations=[s3, all_worker, double_encoded_relation], + containers=[nginx_container, nginx_prometheus_exporter_container], + leader=True, + ) + + # WHEN the relation changed event is processed + # THEN it must not raise, and no rules are written + state_out = context.run(context.on.relation_changed(double_encoded_relation), state_in) + assert _written_group_names(context, state_out) == set() + assert "Could not read the alert rules published over relation" in caplog.text + assert "Could not decompress alert rules" in caplog.text + + +def test_alerts_skips_non_object_payload_and_logs_error( + context, s3, all_worker, nginx_prometheus_exporter_container, caplog +): + """A syntactically valid JSON payload that isn't an object is rejected clearly.""" + # GIVEN a relation whose alert_rules value decodes to a JSON list, not an object + non_object_relation = Relation( + "logging", + remote_app_name="app-non-object", + remote_app_data={ + "alert_rules": json.dumps([1, 2, 3]), + "metadata": _metadata("app-non-object"), + }, + ) + nginx_container = _nginx_container_with_lokitool_sync() + state_in = State( + relations=[s3, all_worker, non_object_relation], + containers=[nginx_container, nginx_prometheus_exporter_container], + leader=True, + ) + + # WHEN the relation changed event is processed + # THEN it must not raise, and no rules are written + state_out = context.run(context.on.relation_changed(non_object_relation), state_in) + assert _written_group_names(context, state_out) == set() + assert "Could not read the alert rules published over relation" in caplog.text + assert "Alert rules must be a JSON object" in caplog.text + + +def test_provider_advertises_alert_rules_encodings_on_relation_joined( + context, s3, all_worker, nginx_prometheus_exporter_container +): + # GIVEN a fresh logging relation with no rules yet + logging_relation = Relation("logging", remote_app_name="app-fresh") + nginx_container = _nginx_container_with_lokitool_sync() + state_in = State( + relations=[s3, all_worker, logging_relation], + containers=[nginx_container, nginx_prometheus_exporter_container], + leader=True, + ) + + # WHEN the relation joined event is processed + state_out = context.run(context.on.relation_joined(logging_relation), state_in) + + # THEN the provider advertises the supported alert rules encodings + relation = state_out.get_relation(logging_relation.id) + advertised = relation.local_app_data[ALERT_RULES_ENCODINGS_KEY] + assert json.loads(advertised) == list(SUPPORTED_ALERT_RULES_ENCODINGS) + + +def test_provider_advertises_alert_rules_encodings_on_relation_changed( + context, s3, all_worker, nginx_prometheus_exporter_container +): + # GIVEN a relation already carrying valid rules + nginx_container = _nginx_container_with_lokitool_sync(_rule_path("app-valid")) + state_in = State( + relations=[s3, all_worker, VALID_RELATION], + containers=[nginx_container, nginx_prometheus_exporter_container], + leader=True, + ) + + # WHEN the relation changed event is processed + state_out = context.run(context.on.relation_changed(VALID_RELATION), state_in) + + # THEN the provider (re)advertises the supported alert rules encodings + relation = state_out.get_relation(VALID_RELATION.id) + advertised = relation.local_app_data[ALERT_RULES_ENCODINGS_KEY] + assert json.loads(advertised) == list(SUPPORTED_ALERT_RULES_ENCODINGS) + + +def test_provider_advertises_alert_rules_encodings_on_leader_elected( + context, s3, all_worker, nginx_prometheus_exporter_container +): + # GIVEN a fresh logging relation and this unit just became leader + logging_relation = Relation("logging", remote_app_name="app-fresh") + nginx_container = _nginx_container_with_lokitool_sync() + state_in = State( + relations=[s3, all_worker, logging_relation], + containers=[nginx_container, nginx_prometheus_exporter_container], + leader=True, + ) + + # WHEN the leader elected event is processed + state_out = context.run(context.on.leader_elected(), state_in) + + # THEN the provider advertises the supported alert rules encodings + relation = state_out.get_relation(logging_relation.id) + advertised = relation.local_app_data[ALERT_RULES_ENCODINGS_KEY] + assert json.loads(advertised) == list(SUPPORTED_ALERT_RULES_ENCODINGS) + + +def test_provider_advertises_alert_rules_encodings_on_upgrade_charm( + context, s3, all_worker, nginx_prometheus_exporter_container +): + """The provider (re)advertises supported alert rules encodings on upgrade-charm. + + Unlike `prometheus_remote_write`'s `MetricsEndpointProvider` (which has no + equivalent lifecycle wiring and had to gain a brand new `upgrade_charm` observer + for this), this is already covered here for free by the existing + `_on_lifecycle_event` handler, which every relation already runs through on + `upgrade_charm` and calls `_publish_alert_rules_encodings` as part of + `_process_logging_relation_changed`. This test pins that behavior down + explicitly, so a future refactor of the lifecycle-event plumbing doesn't + silently drop it. + """ + # GIVEN an existing logging relation + logging_relation = Relation("logging", remote_app_name="app-fresh") + nginx_container = _nginx_container_with_lokitool_sync() + state_in = State( + relations=[s3, all_worker, logging_relation], + containers=[nginx_container, nginx_prometheus_exporter_container], + leader=True, + ) + + # WHEN the charm is upgraded + state_out = context.run(context.on.upgrade_charm(), state_in) + + # THEN the provider (re)advertises the supported alert rules encodings + relation = state_out.get_relation(logging_relation.id) + advertised = relation.local_app_data[ALERT_RULES_ENCODINGS_KEY] + assert json.loads(advertised) == list(SUPPORTED_ALERT_RULES_ENCODINGS) + + +def test_encode_alert_rules_json_default(): + """Default encoding is plain, sorted-keys JSON (legacy-compatible).""" + rules = json.loads(_alert_rules("encode-test")) + encoded = _encode_alert_rules(rules, JSON_ENCODING) + assert json.loads(encoded) == rules + # Not compressed: readable directly as JSON. + assert encoded.startswith("{") + + +def test_unknown_encoding_falls_back_to_json(): + """An encoding this library doesn't know about is treated as plain JSON.""" + rules = json.loads(_alert_rules("unknown-encoding-test")) + encoded = _encode_alert_rules(rules, "brotli") + assert json.loads(encoded) == rules + + +@pytest.mark.parametrize("encoding", SUPPORTED_ALERT_RULES_ENCODINGS) +def test_encoding_is_deterministic(encoding): + """The same rules, with keys in a different order, encode to identical bytes. + + Juju compares relation-databag values byte for byte to decide whether to emit + relation-changed; an unstable key order would trigger spurious relation-changed + events on every hook. + """ + rules = json.loads(_alert_rules("determinism-test")) + reordered = json.loads(json.dumps(rules)) + rule = reordered["groups"][0]["rules"][0] + reordered["groups"][0]["rules"][0] = dict(reversed(list(rule.items()))) + assert list(reordered["groups"][0]["rules"][0]) != list(rules["groups"][0]["rules"][0]) + + assert _encode_alert_rules(reordered, encoding) == _encode_alert_rules(rules, encoding) + + +@pytest.mark.parametrize( + "remote_app_databag, expected", + [ + pytest.param(None, JSON_ENCODING, id="unreadable_databag"), + pytest.param({}, JSON_ENCODING, id="no_advertisement"), + pytest.param({ALERT_RULES_ENCODINGS_KEY: "[]"}, JSON_ENCODING, id="nothing_advertised"), + pytest.param( + {ALERT_RULES_ENCODINGS_KEY: json.dumps([JSON_ENCODING])}, + JSON_ENCODING, + id="json_only", + ), + pytest.param( + {ALERT_RULES_ENCODINGS_KEY: json.dumps(["brotli"])}, + JSON_ENCODING, + id="unknown_encoding", + ), + pytest.param( + {ALERT_RULES_ENCODINGS_KEY: "not json"}, + JSON_ENCODING, + id="malformed_advertisement", + ), + pytest.param( + {ALERT_RULES_ENCODINGS_KEY: json.dumps({"lzma": True})}, + JSON_ENCODING, + id="not_a_list", + ), + pytest.param( + {ALERT_RULES_ENCODINGS_KEY: json.dumps([LZMA_ENCODING, JSON_ENCODING])}, + LZMA_ENCODING, + id="lzma_advertised", + ), + pytest.param( + {ALERT_RULES_ENCODINGS_KEY: json.dumps(["brotli", LZMA_ENCODING])}, + LZMA_ENCODING, + id="lzma_among_unknown_encodings", + ), + ], +) +def test_encoding_negotiation(remote_app_databag, expected): + """Exhaustive matrix for `_best_alert_rules_encoding`, direct against the pure function.""" + assert _best_alert_rules_encoding(remote_app_databag) == expected diff --git a/worker/lib/charms/loki_k8s/v1/loki_push_api.py b/worker/lib/charms/loki_k8s/v1/loki_push_api.py deleted file mode 100644 index e005222..0000000 --- a/worker/lib/charms/loki_k8s/v1/loki_push_api.py +++ /dev/null @@ -1,2592 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. -# -# Learn more at: https://juju.is/docs/sdk - -r"""## Overview. - -This document explains how to use the two principal objects this library provides: - -- `LokiPushApiProvider`: This object is meant to be used by any Charmed Operator that needs to -implement the provider side of the `loki_push_api` relation interface. For instance, a Loki charm. -The provider side of the relation represents the server side, to which logs are being pushed. - -- `LokiPushApiConsumer`: This object is meant to be used by any Charmed Operator that needs to -send log to Loki by implementing the consumer side of the `loki_push_api` relation interface. -For instance, a Promtail or Grafana agent charm which needs to send logs to Loki. - -- `LogProxyConsumer`: DEPRECATED. -This object can be used by any Charmed Operator which needs to send telemetry, such as logs, to -Loki through a Log Proxy by implementing the consumer side of the `loki_push_api` relation -interface. -In order to be able to control the labels on the logs pushed this object adds a Pebble layer -that runs Promtail in the workload container, injecting Juju topology labels into the -logs on the fly. -This object is deprecated. Consider migrating to LogForwarder with the release of Juju 3.6 LTS. - -- `LogForwarder`: This object can be used by any Charmed Operator which needs to send the workload -standard output (stdout) through Pebble's log forwarding mechanism, to Loki endpoints through the -`loki_push_api` relation interface. -In order to be able to control the labels on the logs pushed this object updates the pebble layer's -"log-targets" section with Juju topology. - -Filtering logs in Loki is largely performed on the basis of labels. In the Juju ecosystem, Juju -topology labels are used to uniquely identify the workload which generates telemetry like logs. - - -## LokiPushApiProvider Library Usage - -This object may be used by any Charmed Operator which implements the `loki_push_api` interface. -For instance, Loki or Grafana Agent. - -For this purpose a charm needs to instantiate the `LokiPushApiProvider` object with one mandatory -and three optional arguments. - -- `charm`: A reference to the parent (Loki) charm. - -- `relation_name`: The name of the relation that the charm uses to interact - with its clients, which implement `LokiPushApiConsumer` `LogForwarder`, or `LogProxyConsumer` - (note that LogProxyConsumer is deprecated). - - If provided, this relation name must match a provided relation in metadata.yaml with the - `loki_push_api` interface. - - The default relation name is "logging" for `LokiPushApiConsumer` and `LogForwarder`, and - "log-proxy" for `LogProxyConsumer` (note that LogProxyConsumer is deprecated). - - For example, a provider's `metadata.yaml` file may look as follows: - - ```yaml - provides: - logging: - interface: loki_push_api - ``` - - Subsequently, a Loki charm may instantiate the `LokiPushApiProvider` in its constructor as - follows: - - from charms.loki_k8s.v1.loki_push_api import LokiPushApiProvider - from loki_server import LokiServer - ... - - class LokiOperatorCharm(CharmBase): - ... - - def __init__(self, *args): - super().__init__(*args) - ... - external_url = urlparse(self._external_url) - self.loki_provider = LokiPushApiProvider( - self, - port=external_url.port or 80, - scheme=external_url.scheme, - path=f"{external_url.path}/loki/api/v1/push", - ) - ... - - - `port`: Loki Push Api endpoint port. Default value: `3100`. - - `scheme`: Loki Push Api endpoint scheme (`HTTP` or `HTTPS`). Default value: `HTTP` - - `address`: Loki Push Api endpoint address. Default value: `localhost` - - `path`: Loki Push Api endpoint path. Default value: `loki/api/v1/push` - - -The `LokiPushApiProvider` object has several responsibilities: - -1. Set the URL of the Loki Push API in the relation application data bag; the URL - must be unique to all instances (e.g. using a load balancer). - The default URL is the FQDN, but this can be overridden by calling `update_endpoint()`. - -2. Set the Promtail binary URL (`promtail_binary_zip_url`) so clients that use - `LogProxyConsumer` object could download and configure it. - -3. Process the metadata of the consumer application, provided via the - "metadata" field of the consumer data bag, which are used to annotate the - alert rules (see next point). An example for "metadata" is the following: - - {'model': 'loki', - 'model_uuid': '0b7d1071-ded2-4bf5-80a3-10a81aeb1386', - 'application': 'promtail-k8s' - } - -4. Process alert rules set into the relation by the `LokiPushApiConsumer` - objects, e.g.: - - '{ - "groups": [{ - "name": "loki_0b7d1071-ded2-4bf5-80a3-10a81aeb1386_promtail-k8s_alerts", - "rules": [{ - "alert": "HighPercentageError", - "expr": "sum(rate({app=\\"foo\\", env=\\"production\\"} |= \\"error\\" [5m])) - by (job) \\n /\\nsum(rate({app=\\"foo\\", env=\\"production\\"}[5m])) - by (job)\\n > 0.05 - \\n", "for": "10m", - "labels": { - "severity": "page", - "juju_model": "loki", - "juju_model_uuid": "0b7d1071-ded2-4bf5-80a3-10a81aeb1386", - "juju_application": "promtail-k8s" - }, - "annotations": { - "summary": "High request latency" - } - }] - }] - }' - - -Once these alert rules are sent over relation data, the `LokiPushApiProvider` object -stores these files in the directory `/loki/rules` inside the Loki charm container. After -storing alert rules files, the object will check alert rules by querying Loki API -endpoint: [`loki/api/v1/rules`](https://grafana.com/docs/loki/latest/api/#list-rule-groups). -If there are changes in the alert rules a `loki_push_api_alert_rules_changed` event will -be emitted with details about the `RelationEvent` which triggered it. - -This events should be observed in the charm that uses `LokiPushApiProvider`: - -```python - def __init__(self, *args): - super().__init__(*args) - ... - self.loki_provider = LokiPushApiProvider(self) - self.framework.observe( - self.loki_provider.on.loki_push_api_alert_rules_changed, - self._loki_push_api_alert_rules_changed, - ) -``` - - -## LokiPushApiConsumer Library Usage - -This Loki charm interacts with its clients using the Loki charm library. Charms -seeking to send log to Loki, must do so using the `LokiPushApiConsumer` object from -this charm library. - -> **NOTE**: `LokiPushApiConsumer` also depends on an additional charm library. -> -> Ensure sure you `charmcraft fetch-lib charms.observability_libs.v0.juju_topology` -> when using this library. - -For the simplest use cases, using the `LokiPushApiConsumer` object only requires -instantiating it, typically in the constructor of your charm (the one which -sends logs). - -```python -from charms.loki_k8s.v1.loki_push_api import LokiPushApiConsumer - -class LokiClientCharm(CharmBase): - - def __init__(self, *args): - super().__init__(*args) - ... - self._loki_consumer = LokiPushApiConsumer(self) -``` - -The `LokiPushApiConsumer` constructor requires two things: - -- A reference to the parent (LokiClientCharm) charm. - -- Optionally, the name of the relation that the Loki charm uses to interact - with its clients. If provided, this relation name must match a required - relation in metadata.yaml with the `loki_push_api` interface. - - If not provided, the relation name defaults to `logging`. - -Any time the relation between a Loki provider charm and a Loki consumer charm is -established, a `LokiPushApiEndpointJoined` event is fired. In the consumer side -is it possible to observe this event with: - -```python - -self.framework.observe( - self._loki_consumer.on.loki_push_api_endpoint_joined, - self._on_loki_push_api_endpoint_joined, -) -``` - -Any time there are departures in relations between the consumer charm and Loki -the consumer charm is informed, through a `LokiPushApiEndpointDeparted` event, for instance: - -```python -self.framework.observe( - self._loki_consumer.on.loki_push_api_endpoint_departed, - self._on_loki_push_api_endpoint_departed, -) -``` - -The consumer charm can then choose to update its configuration in both situations. - -Note that LokiPushApiConsumer does not add any labels automatically on its own. In -order to better integrate with the Canonical Observability Stack, you may want to configure your -software to add Juju topology labels. The -[observability-libs](https://charmhub.io/observability-libs) library can be used to get topology -labels in charm code. See :func:`LogProxyConsumer._scrape_configs` for an example of how -to do this with promtail. - -## LogProxyConsumer Library Usage - -> Note: This object is deprecated. Consider migrating to LogForwarder with the release of Juju 3.6 -> LTS. - -Let's say that we have a workload charm that produces logs, and we need to send those logs to a -workload implementing the `loki_push_api` interface, such as `Loki` or `Grafana Agent`. - -Adopting this object in a Charmed Operator consist of two steps: - -1. Use the `LogProxyConsumer` class by instantiating it in the `__init__` method of the charmed - operator. There are two ways to get logs in to promtail. You can give it a list of files to - read, or you can write to it using the syslog protocol. - - For example: - - ```python - from charms.loki_k8s.v1.loki_push_api import LogProxyConsumer - - ... - - def __init__(self, *args): - ... - self._log_proxy = LogProxyConsumer( - self, - logs_scheme={ - "workload-a": { - "log-files": ["/tmp/worload-a-1.log", "/tmp/worload-a-2.log"], - "syslog-port": 1514, - }, - "workload-b": {"log-files": ["/tmp/worload-b.log"], "syslog-port": 1515}, - }, - relation_name="log-proxy", - ) - self.framework.observe( - self._log_proxy.on.promtail_digest_error, - self._promtail_error, - ) - - def _promtail_error(self, event): - logger.error(event.message) - self.unit.status = BlockedStatus(event.message) - ``` - - Any time the relation between a provider charm and a LogProxy consumer charm is - established, a `LogProxyEndpointJoined` event is fired. In the consumer side is it - possible to observe this event with: - - ```python - - self.framework.observe( - self._log_proxy.on.log_proxy_endpoint_joined, - self._on_log_proxy_endpoint_joined, - ) - ``` - - Any time there are departures in relations between the consumer charm and the provider - the consumer charm is informed, through a `LogProxyEndpointDeparted` event, for instance: - - ```python - self.framework.observe( - self._log_proxy.on.log_proxy_endpoint_departed, - self._on_log_proxy_endpoint_departed, - ) - ``` - - The consumer charm can then choose to update its configuration in both situations. - - Note that: - - - You can configure your syslog software using `localhost` as the address and the method - `LogProxyConsumer.syslog_port("container_name")` to get the port, or, alternatively, if you are using rsyslog - you may use the method `LogProxyConsumer.rsyslog_config("container_name")`. - -2. Modify the `metadata.yaml` file to add: - - - The `log-proxy` relation in the `requires` section: - ```yaml - requires: - log-proxy: - interface: loki_push_api - optional: true - ``` - -Once the library is implemented in a Charmed Operator and a relation is established with -the charm that implements the `loki_push_api` interface, the library will inject a -Pebble layer that runs Promtail in the workload container to send logs. - -By default, the promtail binary injected into the container will be downloaded from the internet. -If, for any reason, the container has limited network access, you may allow charm administrators -to provide their own promtail binary at runtime by adding the following snippet to your charm -metadata: - -```yaml -resources: - promtail-bin: - type: file - description: Promtail binary for logging - filename: promtail-linux -``` - -Which would then allow operators to deploy the charm this way: - -``` -juju deploy \ - ./your_charm.charm \ - --resource promtail-bin=/tmp/promtail-linux-amd64 -``` - -If a different resource name is used, it can be specified with the `promtail_resource_name` -argument to the `LogProxyConsumer` constructor. - -The object can emit a `PromtailDigestError` event: - -- Promtail binary cannot be downloaded. -- The sha256 sum mismatch for promtail binary. - -The object can raise a `ContainerNotFoundError` event: - -- No `container_name` parameter has been specified and the Pod has more than 1 container. - -These can be monitored via the PromtailDigestError events via: - -```python - self.framework.observe( - self._loki_consumer.on.promtail_digest_error, - self._promtail_error, - ) - - def _promtail_error(self, event): - logger.error(msg) - self.unit.status = BlockedStatus(event.message) - ) -``` - -## LogForwarder class Usage - -Let's say that we have a charm's workload that writes logs to the standard output (stdout), -and we need to send those logs to a workload implementing the `loki_push_api` interface, -such as `Loki` or `Grafana Agent`. To know how to reach a Loki instance, a charm would -typically use the `loki_push_api` interface. - -Use the `LogForwarder` class by instantiating it in the `__init__` method of the charm: - -```python -from charms.loki_k8s.v1.loki_push_api import LogForwarder - -... - - def __init__(self, *args): - ... - self._log_forwarder = LogForwarder( - self, - relation_name="logging" # optional, defaults to `logging` - ) -``` - -The `LogForwarder` by default will observe relation events on the `logging` endpoint and -enable/disable log forwarding automatically. -Next, modify the `metadata.yaml` file to add: - -The `log-forwarding` relation in the `requires` section: -```yaml -requires: - logging: - interface: loki_push_api - optional: true -``` - -Once the LogForwader class is implemented in your charm and the relation (implementing the -`loki_push_api` interface) is active and healthy, the library will inject a Pebble layer in -each workload container the charm has access to, to configure Pebble's log forwarding -feature and start sending logs to Loki. - -## Alerting Rules - -This charm library also supports gathering alerting rules from all related Loki client -charms and enabling corresponding alerts within the Loki charm. Alert rules are -automatically gathered by `LokiPushApiConsumer` object from a directory conventionally -named `loki_alert_rules`. - -This directory must reside at the top level in the `src` folder of the -consumer charm. Each file in this directory is assumed to be a single alert rule -in YAML format. The file name must have one of the following extensions: `.yaml`, `.yml`, `.rule`, or `.rules`. -The format of this alert rule conforms to the -[Loki docs](https://grafana.com/docs/loki/latest/rules/#alerting-rules). - -An example of the contents of one such file is shown below. - -```yaml -alert: HighPercentageError -expr: | - sum(rate({%%juju_topology%%} |= "error" [5m])) by (job) - / - sum(rate({%%juju_topology%%}[5m])) by (job) - > 0.05 -for: 10m -labels: - severity: page -annotations: - summary: High request latency - -``` - -It is **critical** to use the `%%juju_topology%%` filter in the expression for the alert -rule shown above. This filter is a stub that is automatically replaced by the -`LokiPushApiConsumer` following Loki Client's Juju topology (application, model and its -UUID). Such a topology filter is essential to ensure that alert rules submitted by one -provider charm generates alerts only for that same charm. - -The Loki charm may be related to multiple Loki client charms. Without this, filter -rules submitted by one provider charm will also result in corresponding alerts for other -provider charms. Hence, every alert rule expression must include such a topology filter stub. - -Gathering alert rules and generating rule files within the Loki charm is easily done using -the `alerts()` method of `LokiPushApiProvider`. Alerts generated by Loki will automatically -include Juju topology labels in the alerts. These labels indicate the source of the alert. - -The following labels are automatically added to every alert - -- `juju_model` -- `juju_model_uuid` -- `juju_application` - - -Whether alert rules files does not contain the keys `alert` or `expr` or there is no alert -rules file in `alert_rules_path` a `loki_push_api_alert_rules_error` event is emitted. - -To handle these situations the event must be observed in the `LokiClientCharm` charm.py file: - -```python -class LokiClientCharm(CharmBase): - - def __init__(self, *args): - super().__init__(*args) - ... - self._loki_consumer = LokiPushApiConsumer(self) - - self.framework.observe( - self._loki_consumer.on.loki_push_api_alert_rules_error, - self._alert_rules_error - ) - - def _alert_rules_error(self, event): - self.unit.status = BlockedStatus(event.message) -``` - -## Relation Data - -The Loki charm uses both application and unit relation data to obtain information regarding -Loki Push API and alert rules. - -Units of consumer charm send their alert rules over app relation data using the `alert_rules` -key. - -## Charm logging -The `charms.loki_k8s.v0.charm_logging` library can be used in conjunction with this one to configure python's -logging module to forward all logs to Loki via the loki-push-api interface. - -```python -from lib.charms.loki_k8s.v0.charm_logging import log_charm -from lib.charms.loki_k8s.v1.loki_push_api import charm_logging_config, LokiPushApiConsumer - -@log_charm(logging_endpoint="my_endpoints", server_cert="cert_path") -class MyCharm(...): - _cert_path = "/path/to/cert/on/charm/container.crt" - def __init__(self, ...): - self.logging = LokiPushApiConsumer(...) - self.my_endpoints, self.cert_path = charm_logging_config( - self.logging, self._cert_path) -``` - -Do this, and all charm logs will be forwarded to Loki as soon as a relation is formed. -""" - -import copy -import json -import logging -import os -import platform -import re -import socket -import warnings -from copy import deepcopy -from gzip import GzipFile -from hashlib import sha256 -from io import BytesIO -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union, cast -from urllib import request -from urllib.error import URLError - -import yaml -from cosl import CosTool, JujuTopology -from cosl.rules import AlertRules -from cosl.types import OfficialRuleFileFormat -from ops.charm import ( - CharmBase, - HookEvent, - PebbleReadyEvent, - RelationBrokenEvent, - RelationCreatedEvent, - RelationDepartedEvent, - RelationEvent, - RelationJoinedEvent, - RelationRole, - WorkloadEvent, -) -from ops.framework import BoundEvent, EventBase, EventSource, Object, ObjectEvents -from ops.jujuversion import JujuVersion -from ops.model import Container, ModelError, Relation -from ops.pebble import APIError, ChangeError, Layer, PathError, ProtocolError - -# The unique Charmhub library identifier, never change it -LIBID = "bf76f23cdd03464b877c52bd1d2f563e" - -# Increment this major API version when introducing breaking changes -LIBAPI = 1 - -# Increment this PATCH version before using `charmcraft publish-lib` or reset -# to 0 if you are raising the major API version -LIBPATCH = 31 - -PYDEPS = ["cosl"] - -logger = logging.getLogger(__name__) - -RELATION_INTERFACE_NAME = "loki_push_api" -DEFAULT_RELATION_NAME = "logging" -DEFAULT_ALERT_RULES_RELATIVE_PATH = "./src/loki_alert_rules" -DEFAULT_LOG_PROXY_RELATION_NAME = "log-proxy" - -PROMTAIL_BASE_URL = "https://github.com/canonical/loki-k8s-operator/releases/download" -# To update Promtail version you only need to change the PROMTAIL_VERSION and -# update all sha256 sums in PROMTAIL_BINARIES. To support a new architecture -# you only need to add a new key value pair for the architecture in PROMTAIL_BINARIES. -PROMTAIL_VERSION = "v2.9.7" -PROMTAIL_ARM_BINARY = { - "filename": "promtail-static-arm64", - "zipsha": "c083fdb45e5c794103f974eeb426489b4142438d9e10d0ae272b2aff886e249b", - "binsha": "4cd055c477a301c0bdfdbcea514e6e93f6df5d57425ce10ffc77f3e16fec1ddf", -} - -PROMTAIL_BINARIES = { - "amd64": { - "filename": "promtail-static-amd64", - "zipsha": "6873cbdabf23062aeefed6de5f00ff382710332af3ab90a48c253ea17e08f465", - "binsha": "28da9b99f81296fe297831f3bc9d92aea43b4a92826b8ff04ba433b8cb92fb50", - }, - "arm64": PROMTAIL_ARM_BINARY, - "aarch64": PROMTAIL_ARM_BINARY, -} - -# Paths in `charm` container -BINARY_DIR = "/tmp" - -# Paths in `workload` container -WORKLOAD_BINARY_DIR = "/opt/promtail" -WORKLOAD_CONFIG_DIR = "/etc/promtail" -WORKLOAD_CONFIG_FILE_NAME = "promtail_config.yaml" -WORKLOAD_CONFIG_PATH = "{}/{}".format(WORKLOAD_CONFIG_DIR, WORKLOAD_CONFIG_FILE_NAME) -WORKLOAD_POSITIONS_PATH = "{}/positions.yaml".format(WORKLOAD_BINARY_DIR) -WORKLOAD_SERVICE_NAME = "promtail" - -# These are the initial port values. As we can have more than one container, -# we use odd and even numbers to avoid collisions. -# Each new container adds 2 to the previous value. -HTTP_LISTEN_PORT_START = 9080 # even start port -GRPC_LISTEN_PORT_START = 9095 # odd start port - - -class LokiPushApiError(Exception): - """Base class for errors raised by this module.""" - - -class RelationNotFoundError(LokiPushApiError): - """Raised if there is no relation with the given name.""" - - def __init__(self, relation_name: str): - self.relation_name = relation_name - self.message = "No relation named '{}' found".format(relation_name) - - super().__init__(self.message) - - -class RelationInterfaceMismatchError(LokiPushApiError): - """Raised if the relation with the given name has a different interface.""" - - def __init__( - self, - relation_name: str, - expected_relation_interface: str, - actual_relation_interface: str, - ): - self.relation_name = relation_name - self.expected_relation_interface = expected_relation_interface - self.actual_relation_interface = actual_relation_interface - self.message = ( - "The '{}' relation has '{}' as interface rather than the expected '{}'".format( - relation_name, actual_relation_interface, expected_relation_interface - ) - ) - super().__init__(self.message) - - -class RelationRoleMismatchError(LokiPushApiError): - """Raised if the relation with the given name has a different direction.""" - - def __init__( - self, - relation_name: str, - expected_relation_role: RelationRole, - actual_relation_role: RelationRole, - ): - self.relation_name = relation_name - self.expected_relation_interface = expected_relation_role - self.actual_relation_role = actual_relation_role - self.message = "The '{}' relation has role '{}' rather than the expected '{}'".format( - relation_name, repr(actual_relation_role), repr(expected_relation_role) - ) - super().__init__(self.message) - - -def _validate_relation_by_interface_and_direction( - charm: CharmBase, - relation_name: str, - expected_relation_interface: str, - expected_relation_role: RelationRole, -): - """Verifies that a relation has the necessary characteristics. - - Verifies that the `relation_name` provided: (1) exists in metadata.yaml, - (2) declares as interface the interface name passed as `relation_interface` - and (3) has the right "direction", i.e., it is a relation that `charm` - provides or requires. - - Args: - charm: a `CharmBase` object to scan for the matching relation. - relation_name: the name of the relation to be verified. - expected_relation_interface: the interface name to be matched by the - relation named `relation_name`. - expected_relation_role: whether the `relation_name` must be either - provided or required by `charm`. - - Raises: - RelationNotFoundError: If there is no relation in the charm's metadata.yaml - with the same name as provided via `relation_name` argument. - RelationInterfaceMismatchError: The relation with the same name as provided - via `relation_name` argument does not have the same relation interface - as specified via the `expected_relation_interface` argument. - RelationRoleMismatchError: If the relation with the same name as provided - via `relation_name` argument does not have the same role as specified - via the `expected_relation_role` argument. - """ - if relation_name not in charm.meta.relations: - raise RelationNotFoundError(relation_name) - - relation = charm.meta.relations[relation_name] - - actual_relation_interface = relation.interface_name - if actual_relation_interface != expected_relation_interface: - raise RelationInterfaceMismatchError( - relation_name, - expected_relation_interface, - actual_relation_interface, # pyright: ignore - ) - - if expected_relation_role == RelationRole.provides: - if relation_name not in charm.meta.provides: - raise RelationRoleMismatchError( - relation_name, RelationRole.provides, RelationRole.requires - ) - elif expected_relation_role == RelationRole.requires: - if relation_name not in charm.meta.requires: - raise RelationRoleMismatchError( - relation_name, RelationRole.requires, RelationRole.provides - ) - else: - raise Exception("Unexpected RelationDirection: {}".format(expected_relation_role)) - - -class InvalidAlertRulePathError(Exception): - """Raised if the alert rules folder cannot be found or is otherwise invalid.""" - - def __init__( - self, - alert_rules_absolute_path: Path, - message: str, - ): - self.alert_rules_absolute_path = alert_rules_absolute_path - self.message = message - - super().__init__(self.message) - - -def _resolve_dir_against_charm_path(charm: CharmBase, *path_elements: str) -> str: - """Resolve the provided path items against the directory of the main file. - - Look up the directory of the `main.py` file being executed. This is normally - going to be the charm.py file of the charm including this library. Then, resolve - the provided path elements and, if the result path exists and is a directory, - return its absolute path; otherwise, raise en exception. - - Raises: - InvalidAlertRulePathError, if the path does not exist or is not a directory. - """ - charm_dir = Path(str(charm.charm_dir)) - if not charm_dir.exists() or not charm_dir.is_dir(): - # Operator Framework does not currently expose a robust - # way to determine the top level charm source directory - # that is consistent across deployed charms and unit tests - # Hence for unit tests the current working directory is used - # TODO: updated this logic when the following ticket is resolved - # https://github.com/canonical/operator/issues/643 - charm_dir = Path(os.getcwd()) - - alerts_dir_path = charm_dir.absolute().joinpath(*path_elements) - - if not alerts_dir_path.exists(): - raise InvalidAlertRulePathError(alerts_dir_path, "directory does not exist") - if not alerts_dir_path.is_dir(): - raise InvalidAlertRulePathError(alerts_dir_path, "is not a directory") - - return str(alerts_dir_path) - - -class NoRelationWithInterfaceFoundError(Exception): - """No relations with the given interface are found in the charm meta.""" - - def __init__(self, charm: CharmBase, relation_interface: Optional[str] = None): - self.charm = charm - self.relation_interface = relation_interface - self.message = ( - "No relations with interface '{}' found in the meta of the '{}' charm".format( - relation_interface, charm.meta.name - ) - ) - - super().__init__(self.message) - - -class MultipleRelationsWithInterfaceFoundError(Exception): - """Multiple relations with the given interface are found in the charm meta.""" - - def __init__(self, charm: CharmBase, relation_interface: str, relations: list): - self.charm = charm - self.relation_interface = relation_interface - self.relations = relations - self.message = ( - "Multiple relations with interface '{}' found in the meta of the '{}' charm.".format( - relation_interface, charm.meta.name - ) - ) - super().__init__(self.message) - - -class LokiPushApiEndpointDeparted(EventBase): - """Event emitted when Loki departed.""" - - -class LokiPushApiEndpointJoined(EventBase): - """Event emitted when Loki joined.""" - - -class LokiPushApiAlertRulesChanged(EventBase): - """Event emitted if there is a change in the alert rules.""" - - def __init__(self, handle, relation, relation_id, app=None, unit=None): - """Pretend we are almost like a RelationEvent. - - Fields to serialize: - { - "relation_name": , - "relation_id": , - "app_name": , - "unit_name": - } - - In this way, we can transparently use `RelationEvent.snapshot()` to pass - it back if we need to log it. - """ - super().__init__(handle) - self.relation = relation - self.relation_id = relation_id - self.app = app - self.unit = unit - - def snapshot(self) -> Dict: - """Save event information.""" - if not self.relation: - return {} - snapshot = {"relation_name": self.relation.name, "relation_id": self.relation.id} - if self.app: - snapshot["app_name"] = self.app.name - if self.unit: - snapshot["unit_name"] = self.unit.name - return snapshot - - def restore(self, snapshot: dict): - """Restore event information.""" - self.relation = self.framework.model.get_relation( - snapshot["relation_name"], snapshot["relation_id"] - ) - app_name = snapshot.get("app_name") - if app_name: - self.app = self.framework.model.get_app(app_name) - else: - self.app = None - unit_name = snapshot.get("unit_name") - if unit_name: - self.unit = self.framework.model.get_unit(unit_name) - else: - self.unit = None - - -class InvalidAlertRuleEvent(EventBase): - """Event emitted when alert rule files are not parsable. - - Enables us to set a clear status on the provider. - """ - - def __init__(self, handle, errors: str = "", valid: bool = False): - super().__init__(handle) - self.errors = errors - self.valid = valid - - def snapshot(self) -> Dict: - """Save alert rule information.""" - return { - "valid": self.valid, - "errors": self.errors, - } - - def restore(self, snapshot): - """Restore alert rule information.""" - self.valid = snapshot["valid"] - self.errors = snapshot["errors"] - - -class LokiPushApiEvents(ObjectEvents): - """Event descriptor for events raised by `LokiPushApiProvider`.""" - - loki_push_api_endpoint_departed = EventSource(LokiPushApiEndpointDeparted) - loki_push_api_endpoint_joined = EventSource(LokiPushApiEndpointJoined) - loki_push_api_alert_rules_changed = EventSource(LokiPushApiAlertRulesChanged) - alert_rule_status_changed = EventSource(InvalidAlertRuleEvent) - - -class LokiPushApiProvider(Object): - """A LokiPushApiProvider class.""" - - on = LokiPushApiEvents() # pyright: ignore - - def __init__( - self, - charm, - relation_name: str = DEFAULT_RELATION_NAME, - *, - port: Union[str, int] = 3100, - scheme: str = "http", - address: str = "", - path: str = "loki/api/v1/push", - ): - """A Loki service provider. - - Args: - charm: a `CharmBase` instance that manages this - instance of the Loki service. - relation_name: an optional string name of the relation between `charm` - and the Loki charmed service. The default is "logging". - It is strongly advised not to change the default, so that people - deploying your charm will have a consistent experience with all - other charms that consume metrics endpoints. - port: an optional port of the Loki service (default is "3100"). - scheme: an optional scheme of the Loki API URL (default is "http"). - address: DEPRECATED. This argument is ignored and will be removed in v2. - It is kept for backward compatibility. - Use `update_endpoint()` instead. - path: an optional path of the Loki API URL (default is "loki/api/v1/push") - - Raises: - RelationNotFoundError: If there is no relation in the charm's metadata.yaml - with the same name as provided via `relation_name` argument. - RelationInterfaceMismatchError: The relation with the same name as provided - via `relation_name` argument does not have the `loki_push_api` relation - interface. - RelationRoleMismatchError: If the relation with the same name as provided - via `relation_name` argument does not have the `RelationRole.requires` - role. - """ - _validate_relation_by_interface_and_direction( - charm, relation_name, RELATION_INTERFACE_NAME, RelationRole.provides - ) - - if address != "": - warnings.warn( - "The 'address' parameter is deprecated and will be removed in v2. " - "Use 'update_endpoint()' instead.", - DeprecationWarning, - stacklevel=2, - ) - - super().__init__(charm, relation_name) - self._charm = charm - self._relation_name = relation_name - self._tool = CosTool("logql") - self.port = int(port) - self.scheme = scheme - self.path = path - self._custom_url = None - - events = self._charm.on[relation_name] - self.framework.observe(self._charm.on.upgrade_charm, self._on_lifecycle_event) - self.framework.observe(events.relation_joined, self._on_logging_relation_joined) - self.framework.observe(events.relation_changed, self._on_logging_relation_changed) - self.framework.observe(events.relation_departed, self._on_logging_relation_departed) - self.framework.observe(events.relation_broken, self._on_logging_relation_broken) - - def _on_lifecycle_event(self, _): - # Upgrade event or other charm-level event - should_update = False - for relation in self._charm.model.relations[self._relation_name]: - # Don't accidentally flip a True result back. - should_update = should_update or self._process_logging_relation_changed(relation) - if should_update: - # We don't have a RelationEvent, so build it up by hand - first_rel = self._charm.model.relations[self._relation_name][0] - self.on.loki_push_api_alert_rules_changed.emit( - relation=first_rel, - relation_id=first_rel.id, - ) - - def _on_logging_relation_joined(self, event: RelationJoinedEvent): - """Set basic data on relation joins. - - Set the promtail binary URL location, which will not change, and anything - else which may be required, but is static.. - - Args: - event: a `CharmEvent` in response to which the consumer - charm must set its relation data. - """ - if self._charm.unit.is_leader(): - event.relation.data[self._charm.app].update(self._promtail_binary_url) - logger.debug("Saved promtail binary url: %s", self._promtail_binary_url) - - def _on_logging_relation_changed(self, event: HookEvent): - """Handle changes in related consumers. - - Anytime there are changes in the relation between Loki - and its consumers charms. - - Args: - event: a `CharmEvent` in response to which the consumer - charm must update its relation data. - """ - should_update = self._process_logging_relation_changed(event.relation) # pyright: ignore - if should_update: - self.on.loki_push_api_alert_rules_changed.emit( - relation=event.relation, # pyright: ignore - relation_id=event.relation.id, # pyright: ignore - app=self._charm.app, - unit=self._charm.unit, - ) - - def _on_logging_relation_broken(self, event: RelationBrokenEvent): - """Removes alert rules files when consumer charms left the relation with Loki. - - Args: - event: a `CharmEvent` in response to which the Loki - charm must update its relation data. - """ - self.on.loki_push_api_alert_rules_changed.emit( - relation=event.relation, - relation_id=event.relation.id, - app=self._charm.app, - unit=self._charm.unit, - ) - - def _on_logging_relation_departed(self, event: RelationDepartedEvent): - """Removes alert rules files when consumer charms left the relation with Loki. - - Args: - event: a `CharmEvent` in response to which the Loki - charm must update its relation data. - """ - self.on.loki_push_api_alert_rules_changed.emit( - relation=event.relation, - relation_id=event.relation.id, - app=self._charm.app, - unit=self._charm.unit, - ) - - def _should_update_alert_rules(self, relation) -> bool: - """Determine whether alert rules should be regenerated. - - If there are alert rules in the relation data bag, tell the charm - whether to regenerate them based on the boolean returned here. - """ - if relation.data.get(relation.app).get("alert_rules", None) is not None: - return True - return False - - def _process_logging_relation_changed(self, relation: Relation) -> bool: - """Handle changes in related consumers. - - Anytime there are changes in relations between Loki - and its consumers charms, Loki set the `loki_push_api` - into the relation data. Set the endpoint building - appropriately, and if there are alert rules present in - the relation, let the caller know. - Besides Loki generates alert rules files based what - consumer charms forwards, - - Args: - relation: the `Relation` instance to update. - - Returns: - A boolean indicating whether an event should be emitted, so we - only emit one on lifecycle events - """ - relation.data[self._charm.unit]["public_address"] = socket.getfqdn() or "" - self.update_endpoint(relation=relation) - - # Ensure promtail binary URL is set in app data. This is normally done on - # relation_joined, but charms using the reconcile pattern may miss that event - # if the workload container is not yet ready when the relation is first established. - if self._charm.unit.is_leader(): - if not relation.data[self._charm.app].get("promtail_binary_zip_url"): - relation.data[self._charm.app].update(self._promtail_binary_url) - - return self._should_update_alert_rules(relation) - - @property - def _promtail_binary_url(self) -> dict: - """URL from which Promtail binary can be downloaded.""" - # construct promtail binary url paths from parts - promtail_binaries = {} - for arch, info in PROMTAIL_BINARIES.items(): - info["url"] = "{}/promtail-{}/{}.gz".format( - PROMTAIL_BASE_URL, PROMTAIL_VERSION, info["filename"] - ) - promtail_binaries[arch] = info - - return {"promtail_binary_zip_url": json.dumps(promtail_binaries)} - - def update_endpoint(self, url: str = "", relation: Optional[Relation] = None) -> None: - """Triggers programmatically the update of endpoint in unit relation data. - - This method should be used when the charm relying on this library needs - to update the relation data in response to something occurring outside - the `logging` relation lifecycle, e.g., in case of a - host address change because the charmed operator becomes connected to an - Ingress after the `logging` relation is established. - - To make this library reconciler-friendly, the endpoint URL was made sticky i.e., once the - endpoint is updated with a custom URL, using the public method, it cannot be unset. Users - of this method should set the "url" arg to an internal URL if the charms ingress is no - longer available. - - Args: - url: An optional url value to update relation data. - relation: An optional instance of `class:ops.model.Relation` to update. - """ - # if no relation is specified update all of them - if not relation: - if not self._charm.model.relations.get(self._relation_name): - return - - relations_list = self._charm.model.relations.get(self._relation_name) - else: - relations_list = [relation] - - if url: - self._custom_url = url - - endpoint = self._endpoint(self._custom_url or self._url) - - for relation in relations_list: - relation.data[self._charm.unit].update({"endpoint": json.dumps(endpoint)}) - - logger.debug("Saved endpoint in unit relation data") - - @property - def _url(self) -> str: - """Get local Loki Push API url. - - Return url to loki, including port number, but without the endpoint subpath. - """ - return f"{self.scheme}://{socket.getfqdn()}:{self.port}" - - def _endpoint(self, url) -> dict: - """Get Loki push API endpoint for a given url. - - Args: - url: A loki unit URL. - - Returns: str - """ - endpoint = "/loki/api/v1/push" - return {"url": url.rstrip("/") + endpoint} - - @property - def alerts(self) -> dict: # noqa: C901 - """Fetch alerts for all relations. - - A Loki alert rules file consists of a list of "groups". Each - group consists of a list of alerts (`rules`) that are sequentially - executed. This method returns all the alert rules provided by each - related metrics provider charm. These rules may be used to generate a - separate alert rules file for each relation since the returned list - of alert groups are indexed by relation ID. Also for each relation ID - associated scrape metadata such as Juju model, UUID and application - name are provided so a unique name may be generated for the rules - file. For each relation the structure of data returned is a dictionary - with four keys - - - groups - - model - - model_uuid - - application - - The value of the `groups` key is such that it may be used to generate - a Loki alert rules file directly using `yaml.dump` but the - `groups` key itself must be included as this is required by Loki, - for example as in `yaml.dump({"groups": alerts["groups"]})`. - - Currently only accepts a list of rules and these - rules are all placed into a single group, even though Loki itself - allows for multiple groups within a single alert rules file. - - Returns: - a dictionary of alert rule groups and associated scrape - metadata indexed by relation ID. - """ - alerts = {} # type: Dict[str, dict] # mapping b/w juju identifiers and alert rule files - for relation in self._charm.model.relations[self._relation_name]: - if not relation.units or not relation.app: - continue - - alert_rules = json.loads(relation.data[relation.app].get("alert_rules", "{}")) - if not alert_rules: - continue - - alert_rules = self._inject_alert_expr_labels(alert_rules) - - identifier, topology = self._get_identifier_by_alert_rules(alert_rules) - if not topology: - try: - metadata = json.loads(relation.data[relation.app]["metadata"]) - identifier = JujuTopology.from_dict(metadata).identifier - - except KeyError as e: - logger.debug( - "Relation %s has no 'metadata': %s", - relation.id, - e, - ) - - if not identifier: - logger.error( - "Alert rules were found but no usable group or identifier was present." - ) - continue - - # Topology labels are already injected by _inject_alert_expr_labels using - # alert_expression_dict, which intentionally excludes juju_charm and juju_unit. - # Don't call apply_label_matchers here as it would re-inject juju_charm. - alerts[identifier] = alert_rules - - _, errmsg = self._tool.validate_alert_rules(cast(OfficialRuleFileFormat, alert_rules)) - if errmsg: - logger.error(f"Invalid alert rule file: {errmsg}") - if alerts[identifier]: - del alerts[identifier] - if self._charm.unit.is_leader(): - relation.data[self._charm.app]["event"] = json.dumps({"errors": errmsg}) - continue - if self._charm.unit.is_leader(): - event_data = json.loads(relation.data[self._charm.app].get("event", "{}")) - event_data.pop("errors", None) - relation.data[self._charm.app]["event"] = json.dumps(event_data) - - alerts[identifier] = alert_rules - - return alerts - - def has_invalid_alert_rules(self) -> bool: - """Check whether any relation reported invalid alert rules. - - Validation errors, written to relation app data by the :attr:`alerts` - property, are read back to determine whether the relation currently - carries invalid alert rules. Non-leader units never write the app data - that holds these errors, so they always report no errors. - - Returns: - True if any related consumer reported alert rule validation - errors, False otherwise. - """ - if not self._charm.unit.is_leader(): - return False - - for relation in self._charm.model.relations.get(self._relation_name, []): - app_data = relation.data.get(self._charm.app) - if not app_data: - continue - - event_raw = app_data.get("event", "{}") - try: - event_data = json.loads(event_raw) - except (json.JSONDecodeError, TypeError): - continue - - if error_msg := event_data.get("errors"): - logger.error( - "Alert rule validation error on relation %s: %s", - relation.id, - error_msg, - ) - return True - - return False - - def _get_identifier_by_alert_rules( - self, rules: dict - ) -> Tuple[Union[str, None], Union[JujuTopology, None]]: - """Determine an appropriate dict key for alert rules. - - The key is used as the filename when writing alerts to disk, so the structure - and uniqueness is important. - - Args: - rules: a dict of alert rules - Returns: - A tuple containing an identifier, if found, and a JujuTopology, if it could - be constructed. - """ - if "groups" not in rules: - logger.debug("No alert groups were found in relation data") - return None, None - - # Construct an ID based on what's in the alert rules if they have labels - for group in rules["groups"]: - try: - labels = group["rules"][0]["labels"] - topology = JujuTopology( - # Don't try to safely get required constructor fields. There's already - # a handler for KeyErrors - model_uuid=labels["juju_model_uuid"], - model=labels["juju_model"], - application=labels["juju_application"], - unit=labels.get("juju_unit", ""), - charm_name=labels.get("juju_charm", ""), - ) - return topology.identifier, topology - except KeyError: - logger.debug("Alert rules were found but no usable labels were present") - continue - - logger.warning( - "No labeled alert rules were found, and no 'scrape_metadata' " - "was available. Using the alert group name as filename." - ) - try: - for group in rules["groups"]: - return group["name"], None - except KeyError: - logger.debug("No group name was found to use as identifier") - - return None, None - - def _inject_alert_expr_labels(self, rules: Dict[str, Any]) -> Dict[str, Any]: - """Iterate through alert rules and inject topology into expressions. - - Args: - rules: a dict of alert rules - """ - if "groups" not in rules: - return rules - - modified_groups = [] - for group in rules["groups"]: - # Copy off rules, so we don't modify an object we're iterating over - rules_copy = group["rules"] - for idx, rule in enumerate(rules_copy): - labels = rule.get("labels") - - if labels: - try: - topology = JujuTopology( - # Don't try to safely get required constructor fields. There's already - # a handler for KeyErrors - model_uuid=labels["juju_model_uuid"], - model=labels["juju_model"], - application=labels["juju_application"], - unit=labels.get("juju_unit", ""), - charm_name=labels.get("juju_charm", ""), - ) - - # Inject topology and put it back in the list. - # Use alert_expression_dict (excludes juju_charm) instead of - # label_matcher_dict because subordinate charms (e.g. otelcol) - # label logs with their own charm name, not the principal's. - rule["expr"] = self._tool.inject_label_matchers( - re.sub(r"%%juju_topology%%,?", "", rule["expr"]), - topology.alert_expression_dict, - ) - except KeyError: - # Some required JujuTopology key is missing. Just move on. - pass - - group["rules"][idx] = rule - - modified_groups.append(group) - - rules["groups"] = modified_groups - return rules - - -class ConsumerBase(Object): - """Consumer's base class.""" - - def __init__( - self, - charm: CharmBase, - relation_name: str = DEFAULT_RELATION_NAME, - alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, - recursive: bool = False, - skip_alert_topology_labeling: bool = False, - *, - forward_alert_rules: bool = True, - extra_alert_labels: Dict = {}, - ): - super().__init__(charm, relation_name) - self._charm = charm - self._relation_name = relation_name - self._forward_alert_rules = forward_alert_rules - self._extra_alert_labels = extra_alert_labels - self.topology = JujuTopology.from_charm(charm) - - try: - alert_rules_path = _resolve_dir_against_charm_path(charm, alert_rules_path) - except InvalidAlertRulePathError as e: - logger.debug( - "Invalid Loki alert rules folder at %s: %s", - e.alert_rules_absolute_path, - e.message, - ) - self._alert_rules_path = alert_rules_path - self._skip_alert_topology_labeling = skip_alert_topology_labeling - - self._recursive = recursive - - @staticmethod - def _inject_extra_labels_to_alert_rules(rules: Dict, extra_alert_labels: Dict) -> Dict: - """Return a copy of the rules dict with extra labels injected.""" - result = copy.deepcopy(rules) - for group in result.get("groups", []): - for rule in group.get("rules", []): - rule.setdefault("labels", {}).update(extra_alert_labels) - return result - - def _handle_alert_rules(self, relation): - if not self._charm.unit.is_leader(): - return - - alert_rules = ( - AlertRules(query_type="logql") - if self._skip_alert_topology_labeling - else AlertRules(query_type="logql", topology=self.topology) - ) - if self._forward_alert_rules: - alert_rules.add_path(self._alert_rules_path, recursive=self._recursive) - alert_rules_as_dict = alert_rules.as_dict() - - if self._extra_alert_labels: - alert_rules_as_dict = ConsumerBase._inject_extra_labels_to_alert_rules( - alert_rules_as_dict, self._extra_alert_labels - ) - - relation.data[self._charm.app]["metadata"] = json.dumps(self.topology.as_dict()) - relation.data[self._charm.app]["alert_rules"] = json.dumps( - alert_rules_as_dict, - sort_keys=True, # sort, to prevent unnecessary relation_changed events - ) - - @property - def loki_endpoints(self) -> List[dict]: - """Fetch Loki Push API endpoints sent from LokiPushApiProvider through relation data. - - Returns: - A list of unique dictionaries with Loki Push API endpoints, for instance: - [ - {"url": "http://loki1:3100/loki/api/v1/push"}, - {"url": "http://loki2:3100/loki/api/v1/push"}, - ] - """ - endpoints = [] - seen_urls = set() - - for relation in self._charm.model.relations[self._relation_name]: - for unit in relation.units: - if unit.app == self._charm.app: - continue - - if not (endpoint := relation.data[unit].get("endpoint")): - continue - - deserialized_endpoint = json.loads(endpoint) - url = deserialized_endpoint.get("url") - - # Deduplicate by URL. - # With loki-k8s we have ingress-per-unit, so in that case - # we do want to collect the URLs of all the units. - # With loki-coordinator-k8s, even when the coordinator - # is scaled, we want to advertise only one URL. - # Without deduplication, we'd end up with the same - # tls config section in the promtail config file, in which - # case promtail immediately exits with the following error: - # [promtail] level=error ts= msg="error creating promtail" error="failed to create client manager: duplicate client configs are not allowed, found duplicate for name: " - - if not url or url in seen_urls: - continue - - seen_urls.add(url) - endpoints.append(deserialized_endpoint) - - return endpoints - - -class LokiPushApiConsumer(ConsumerBase): - """Loki Consumer class.""" - - on = LokiPushApiEvents() # pyright: ignore - - def __init__( - self, - charm: CharmBase, - relation_name: str = DEFAULT_RELATION_NAME, - alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, - recursive: bool = True, - skip_alert_topology_labeling: bool = False, - *, - refresh_event: Optional[Union[BoundEvent, List[BoundEvent]]] = None, - forward_alert_rules: bool = True, - extra_alert_labels: Dict = {}, - ): - """Construct a Loki charm client. - - The `LokiPushApiConsumer` object provides configurations to a Loki client charm, such as - the Loki API endpoint to push logs. It is intended for workloads that can speak - loki_push_api (https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki), such - as grafana-agent. - (If you need to forward workload stdout logs, then use LogForwarder; if you need to forward - log files, then use LogProxyConsumer.) - - `LokiPushApiConsumer` can be instantiated as follows: - - self._loki_consumer = LokiPushApiConsumer(self) - - Args: - charm: a `CharmBase` object that manages this `LokiPushApiConsumer` object. - Typically, this is `self` in the instantiating class. - relation_name: the string name of the relation interface to look up. - If `charm` has exactly one relation with this interface, the relation's - name is returned. If none or multiple relations with the provided interface - are found, this method will raise either a NoRelationWithInterfaceFoundError or - MultipleRelationsWithInterfaceFoundError exception, respectively. - alert_rules_path: a string indicating a path where alert rules can be found - recursive: Whether to scan for rule files recursively. - skip_alert_topology_labeling: whether to skip the alert topology labeling. - forward_alert_rules: a boolean flag to toggle forwarding of charmed alert rules. - extra_alert_labels: Dict of extra labels to inject alert rules with. - refresh_event: an optional bound event or list of bound events which - will be observed to re-set scrape job data (IP address and others) - - Raises: - RelationNotFoundError: If there is no relation in the charm's metadata.yaml - with the same name as provided via `relation_name` argument. - RelationInterfaceMismatchError: The relation with the same name as provided - via `relation_name` argument does not have the `loki_push_api` relation - interface. - RelationRoleMismatchError: If the relation with the same name as provided - via `relation_name` argument does not have the `RelationRole.provides` - role. - - Emits: - loki_push_api_endpoint_joined: This event is emitted when the relation between the - Charmed Operator that instantiates `LokiPushApiProvider` (Loki charm for instance) - and the Charmed Operator that instantiates `LokiPushApiConsumer` is established. - loki_push_api_endpoint_departed: This event is emitted when the relation between the - Charmed Operator that implements `LokiPushApiProvider` (Loki charm for instance) - and the Charmed Operator that implements `LokiPushApiConsumer` is removed. - loki_push_api_alert_rules_error: This event is emitted when an invalid alert rules - file is encountered or if `alert_rules_path` is empty. - """ - _validate_relation_by_interface_and_direction( - charm, relation_name, RELATION_INTERFACE_NAME, RelationRole.requires - ) - super().__init__( - charm, - relation_name, - alert_rules_path, - recursive, - skip_alert_topology_labeling, - forward_alert_rules=forward_alert_rules, - extra_alert_labels=extra_alert_labels, - ) - events = self._charm.on[relation_name] - self.framework.observe(self._charm.on.upgrade_charm, self._on_lifecycle_event) - self.framework.observe(self._charm.on.config_changed, self._on_lifecycle_event) - self.framework.observe(events.relation_joined, self._on_logging_relation_joined) - self.framework.observe(events.relation_changed, self._on_logging_relation_changed) - self.framework.observe(events.relation_departed, self._on_logging_relation_departed) - - if refresh_event: - if not isinstance(refresh_event, list): - refresh_event = [refresh_event] - for ev in refresh_event: - self.framework.observe(ev, self._on_lifecycle_event) - - def _on_lifecycle_event(self, _: HookEvent): - """Update require relation data on charm upgrades and other lifecycle events. - - Args: - event: a `CharmEvent` in response to which the consumer - charm must update its relation data. - """ - # Upgrade event or other charm-level event - self._reinitialize_alert_rules() - self.on.loki_push_api_endpoint_joined.emit() - - def _on_logging_relation_joined(self, event: RelationJoinedEvent): - """Handle changes in related consumers. - - Update relation data and emit events when a relation is established. - - Args: - event: a `CharmEvent` in response to which the consumer - charm must update its relation data. - - Emits: - loki_push_api_endpoint_joined: Once the relation is established, this event is emitted. - loki_push_api_alert_rules_error: This event is emitted when an invalid alert rules - file is encountered or if `alert_rules_path` is empty. - """ - # Alert rules will not change over the lifecycle of a charm, and do not need to be - # constantly set on every relation_changed event. Leave them here. - self._handle_alert_rules(event.relation) - self.on.loki_push_api_endpoint_joined.emit() - - def _on_logging_relation_changed(self, event: RelationEvent): - """Handle changes in related consumers. - - Anytime there are changes in the relation between Loki - and its consumers charms. - - Args: - event: a `CharmEvent` in response to which the consumer - charm must update its relation data. - - Emits: - loki_push_api_endpoint_joined: Once the relation is established, this event is emitted. - loki_push_api_alert_rules_error: This event is emitted when an invalid alert rules - file is encountered or if `alert_rules_path` is empty. - """ - if self._charm.unit.is_leader(): - ev = json.loads(event.relation.data[event.app].get("event", "{}")) - - if ev: - valid = bool(ev.get("valid", True)) - errors = ev.get("errors", "") - - if valid and not errors: - self.on.alert_rule_status_changed.emit(valid=valid) - else: - self.on.alert_rule_status_changed.emit(valid=valid, errors=errors) - - self.on.loki_push_api_endpoint_joined.emit() - - def reload_alerts(self) -> None: - """Reloads alert rules and updates all relations.""" - self._reinitialize_alert_rules() - - def _reinitialize_alert_rules(self): - for relation in self._charm.model.relations[self._relation_name]: - self._handle_alert_rules(relation) - - def _process_logging_relation_changed(self, relation: Relation): - self._handle_alert_rules(relation) - self.on.loki_push_api_endpoint_joined.emit() - - def _on_logging_relation_departed(self, _: RelationEvent): - """Handle departures in related providers. - - Anytime there are departures in relations between the consumer charm and Loki - the consumer charm is informed, through a `LokiPushApiEndpointDeparted` event. - The consumer charm can then choose to update its configuration. - """ - # Provide default to avoid throwing, as in some complicated scenarios with - # upgrades and hook failures we might not have data in the storage - self.on.loki_push_api_endpoint_departed.emit() - - -class ContainerNotFoundError(Exception): - """Raised if the specified container does not exist.""" - - def __init__(self): - msg = "The specified container does not exist." - self.message = msg - - super().__init__(self.message) - - -class PromtailDigestError(EventBase): - """Event emitted when there is an error with Promtail initialization.""" - - def __init__(self, handle, message): - super().__init__(handle) - self.message = message - - def snapshot(self): - """Save message information.""" - return {"message": self.message} - - def restore(self, snapshot): - """Restore message information.""" - self.message = snapshot["message"] - - -class LogProxyEndpointDeparted(EventBase): - """Event emitted when a Log Proxy has departed.""" - - -class LogProxyEndpointJoined(EventBase): - """Event emitted when a Log Proxy joins.""" - - -class LogProxyEvents(ObjectEvents): - """Event descriptor for events raised by `LogProxyConsumer`.""" - - promtail_digest_error = EventSource(PromtailDigestError) - log_proxy_endpoint_departed = EventSource(LogProxyEndpointDeparted) - log_proxy_endpoint_joined = EventSource(LogProxyEndpointJoined) - - -class LogProxyConsumer(ConsumerBase): - """LogProxyConsumer class. - - > Note: This object is deprecated. Consider migrating to LogForwarder with the release of Juju - > 3.6 LTS. - - The `LogProxyConsumer` object provides a method for attaching `promtail` to - a workload in order to generate structured logging data from applications - which traditionally log to syslog or do not have native Loki integration. - The `LogProxyConsumer` can be instantiated as follows: - - self._log_proxy = LogProxyConsumer( - self, - logs_scheme={ - "workload-a": { - "log-files": ["/tmp/worload-a-1.log", "/tmp/worload-a-2.log"], - "syslog-port": 1514, - }, - "workload-b": {"log-files": ["/tmp/worload-b.log"], "syslog-port": 1515}, - }, - relation_name="log-proxy", - ) - - Args: - charm: a `CharmBase` object that manages this `LokiPushApiConsumer` object. - Typically, this is `self` in the instantiating class. - logs_scheme: a dict which maps containers and a list of log files and syslog port. - relation_name: the string name of the relation interface to look up. - If `charm` has exactly one relation with this interface, the relation's - name is returned. If none or multiple relations with the provided interface - are found, this method will raise either a NoRelationWithInterfaceFoundError or - MultipleRelationsWithInterfaceFoundError exception, respectively. - containers_syslog_port: a dict which maps (and enable) containers and syslog port. - alert_rules_path: an optional path for the location of alert rules - files. Defaults to "./src/loki_alert_rules", - resolved from the directory hosting the charm entry file. - The alert rules are automatically updated on charm upgrade. - recursive: Whether to scan for rule files recursively. - promtail_resource_name: An optional promtail resource name from metadata - if it has been modified and attached - insecure_skip_verify: skip SSL verification. - - Raises: - RelationNotFoundError: If there is no relation in the charm's metadata.yaml - with the same name as provided via `relation_name` argument. - RelationInterfaceMismatchError: The relation with the same name as provided - via `relation_name` argument does not have the `loki_push_api` relation - interface. - RelationRoleMismatchError: If the relation with the same name as provided - via `relation_name` argument does not have the `RelationRole.provides` - role. - """ - - on = LogProxyEvents() # pyright: ignore - - def __init__( - self, - charm, - *, - logs_scheme=None, - relation_name: str = DEFAULT_LOG_PROXY_RELATION_NAME, - alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, - recursive: bool = False, - promtail_resource_name: Optional[str] = None, - insecure_skip_verify: bool = False, - ): - super().__init__(charm, relation_name, alert_rules_path, recursive) - self._charm = charm - self._logs_scheme = logs_scheme or {} - self._relation_name = relation_name - self.topology = JujuTopology.from_charm(charm) - self._promtail_resource_name = promtail_resource_name or "promtail-bin" - self.insecure_skip_verify = insecure_skip_verify - self._promtails_ports = self._generate_promtails_ports(logs_scheme) - - # architecture used for promtail binary - arch = platform.machine() - if arch in ["x86_64", "amd64"]: - self._arch = "amd64" - elif arch in ["aarch64", "arm64", "armv8b", "armv8l"]: - self._arch = "arm64" - else: - self._arch = arch - - events = self._charm.on[relation_name] - self.framework.observe(events.relation_created, self._on_relation_created) - self.framework.observe(events.relation_changed, self._on_relation_changed) - self.framework.observe(events.relation_departed, self._on_relation_departed) - self._observe_pebble_ready() - - def _observe_pebble_ready(self): - for container in self._containers.keys(): - snake_case_container_name = container.replace("-", "_") - self.framework.observe( - getattr(self._charm.on, f"{snake_case_container_name}_pebble_ready"), - self._on_pebble_ready, - ) - - def _on_pebble_ready(self, event: WorkloadEvent): - """Event handler for `pebble_ready`.""" - if self.model.relations[self._relation_name]: - self._setup_promtail(event.workload) - - def _on_relation_created(self, _: RelationCreatedEvent) -> None: - """Event handler for `relation_created`.""" - for container in self._containers.values(): - if container.can_connect(): - self._setup_promtail(container) - - def _on_relation_changed(self, event: RelationEvent) -> None: - """Event handler for `relation_changed`. - - Args: - event: The event object `RelationChangedEvent`. - """ - self._handle_alert_rules(event.relation) - - if self._charm.unit.is_leader(): - ev = json.loads(event.relation.data[event.app].get("event", "{}")) - - if ev: - valid = bool(ev.get("valid", True)) - errors = ev.get("errors", "") - - if valid and not errors: - self.on.alert_rule_status_changed.emit(valid=valid) - else: - self.on.alert_rule_status_changed.emit(valid=valid, errors=errors) - - for container in self._containers.values(): - if not container.can_connect(): - continue - if self.model.relations[self._relation_name]: - if "promtail" not in container.get_plan().services: - self._setup_promtail(container) - continue - - new_config = self._promtail_config(container.name) - if new_config != self._current_config(container): - container.push( - WORKLOAD_CONFIG_PATH, yaml.safe_dump(new_config), make_dirs=True - ) - - # Loki may send endpoints late. Don't necessarily start, there may be - # no clients - if new_config["clients"]: - container.restart(WORKLOAD_SERVICE_NAME) - self.on.log_proxy_endpoint_joined.emit() - else: - self.on.promtail_digest_error.emit("No promtail client endpoints available!") - - def _on_relation_departed(self, _: RelationEvent) -> None: - """Event handler for `relation_departed`. - - Args: - event: The event object `RelationDepartedEvent`. - """ - for container in self._containers.values(): - if not container.can_connect(): - continue - if not self._charm.model.relations[self._relation_name]: - container.stop(WORKLOAD_SERVICE_NAME) - continue - - new_config = self._promtail_config(container.name) - if new_config != self._current_config(container): - container.push(WORKLOAD_CONFIG_PATH, yaml.safe_dump(new_config), make_dirs=True) - - if new_config["clients"]: - container.restart(WORKLOAD_SERVICE_NAME) - else: - container.stop(WORKLOAD_SERVICE_NAME) - self.on.log_proxy_endpoint_departed.emit() - - def _add_pebble_layer(self, workload_binary_path: str, container: Container) -> None: - """Adds Pebble layer that manages Promtail service in Workload container. - - Args: - workload_binary_path: string providing path to promtail binary in workload container. - container: container into which the layer is to be added. - """ - pebble_layer = Layer( - { - "summary": "promtail layer", - "description": "pebble config layer for promtail", - "services": { - WORKLOAD_SERVICE_NAME: { - "override": "replace", - "summary": WORKLOAD_SERVICE_NAME, - "command": f"{workload_binary_path} {self._cli_args}", - "startup": "disabled", - } - }, - } - ) - container.add_layer(container.name, pebble_layer, combine=True) - - def _create_directories(self, container: Container) -> None: - """Creates the directories for Promtail binary and config file.""" - container.make_dir(path=WORKLOAD_BINARY_DIR, make_parents=True) - container.make_dir(path=WORKLOAD_CONFIG_DIR, make_parents=True) - - def _obtain_promtail(self, promtail_info: dict, container: Container) -> None: - """Obtain promtail binary from an attached resource or download it. - - Args: - promtail_info: dictionary containing information about promtail binary - that must be used. The dictionary must have three keys - - "filename": filename of promtail binary - - "zipsha": sha256 sum of zip file of promtail binary - - "binsha": sha256 sum of unpacked promtail binary - container: container into which promtail is to be obtained. - """ - workload_binary_path = os.path.join(WORKLOAD_BINARY_DIR, promtail_info["filename"]) - if self._promtail_attached_as_resource: - self._push_promtail_if_attached(container, workload_binary_path) - return - - if self._promtail_must_be_downloaded(promtail_info): - self._download_and_push_promtail_to_workload(container, promtail_info) - else: - binary_path = os.path.join(BINARY_DIR, promtail_info["filename"]) - self._push_binary_to_workload(container, binary_path, workload_binary_path) - - def _push_binary_to_workload( - self, container: Container, binary_path: str, workload_binary_path: str - ) -> None: - """Push promtail binary into workload container. - - Args: - binary_path: path in charm container from which promtail binary is read. - workload_binary_path: path in workload container to which promtail binary is pushed. - container: container into which promtail is to be uploaded. - """ - with open(binary_path, "rb") as f: - container.push(workload_binary_path, f, permissions=0o755, make_dirs=True) - logger.debug("The promtail binary file has been pushed to the workload container.") - - @property - def _promtail_attached_as_resource(self) -> bool: - """Checks whether Promtail binary is attached to the charm or not. - - Returns: - a boolean representing whether Promtail binary is attached as a resource or not. - """ - try: - self._charm.model.resources.fetch(self._promtail_resource_name) - return True - except ModelError: - return False - except NameError as e: - if "invalid resource name" in str(e): - return False - raise - - def _push_promtail_if_attached(self, container: Container, workload_binary_path: str) -> bool: - """Checks whether Promtail binary is attached to the charm or not. - - Args: - workload_binary_path: string specifying expected path of promtail - in workload container - container: container into which promtail is to be pushed. - - Returns: - a boolean representing whether Promtail binary is attached or not. - """ - logger.info("Promtail binary file has been obtained from an attached resource.") - resource_path = self._charm.model.resources.fetch(self._promtail_resource_name) - self._push_binary_to_workload(container, resource_path, workload_binary_path) - return True - - def _promtail_must_be_downloaded(self, promtail_info: dict) -> bool: - """Checks whether promtail binary must be downloaded or not. - - Args: - promtail_info: dictionary containing information about promtail binary - that must be used. The dictionary must have three keys - - "filename": filename of promtail binary - - "zipsha": sha256 sum of zip file of promtail binary - - "binsha": sha256 sum of unpacked promtail binary - - Returns: - a boolean representing whether Promtail binary must be downloaded or not. - """ - binary_path = os.path.join(BINARY_DIR, promtail_info["filename"]) - if not self._is_promtail_binary_in_charm(binary_path): - return True - - if not self._sha256sums_matches(binary_path, promtail_info["binsha"]): - return True - - logger.debug("Promtail binary file is already in the the charm container.") - return False - - def _sha256sums_matches(self, file_path: str, sha256sum: str) -> bool: - """Checks whether a file's sha256sum matches or not with a specific sha256sum. - - Args: - file_path: A string representing the files' patch. - sha256sum: The sha256sum against which we want to verify. - - Returns: - a boolean representing whether a file's sha256sum matches or not with - a specific sha256sum. - """ - try: - with open(file_path, "rb") as f: - file_bytes = f.read() - result = sha256(file_bytes).hexdigest() - - if result != sha256sum: - msg = "File sha256sum mismatch, expected:'{}' but got '{}'".format( - sha256sum, result - ) - logger.debug(msg) - return False - - return True - except (APIError, FileNotFoundError): - msg = "File: '{}' could not be opened".format(file_path) - logger.error(msg) - return False - - def _is_promtail_binary_in_charm(self, binary_path: str) -> bool: - """Check if Promtail binary is already stored in charm container. - - Args: - binary_path: string path of promtail binary to check - - Returns: - a boolean representing whether Promtail is present or not. - """ - return True if Path(binary_path).is_file() else False - - def _download_and_push_promtail_to_workload( - self, container: Container, promtail_info: dict - ) -> None: - """Downloads a Promtail zip file and pushes the binary to the workload. - - Args: - promtail_info: dictionary containing information about promtail binary - that must be used. The dictionary must have three keys - - "filename": filename of promtail binary - - "zipsha": sha256 sum of zip file of promtail binary - - "binsha": sha256 sum of unpacked promtail binary - container: container into which promtail is to be uploaded. - """ - # Check for Juju proxy variables and fall back to standard ones if not set - # If no Juju proxy variable was set, we set proxies to None to let the ProxyHandler get - # the proxy env variables from the environment - proxies = { - # The ProxyHandler uses only the protocol names as keys - # https://docs.python.org/3/library/urllib.request.html#urllib.request.ProxyHandler - "https": os.environ.get("JUJU_CHARM_HTTPS_PROXY", ""), - "http": os.environ.get("JUJU_CHARM_HTTP_PROXY", ""), - # The ProxyHandler uses `no` for the no_proxy key - # https://github.com/python/cpython/blob/3.12/Lib/urllib/request.py#L2553 - "no": os.environ.get("JUJU_CHARM_NO_PROXY", ""), - } - proxies = {k: v for k, v in proxies.items() if v != ""} or None - - proxy_handler = request.ProxyHandler(proxies) - opener = request.build_opener(proxy_handler) - - with opener.open(promtail_info["url"]) as r: - file_bytes = r.read() - file_path = os.path.join(BINARY_DIR, promtail_info["filename"] + ".gz") - with open(file_path, "wb") as f: - f.write(file_bytes) - logger.info( - "Promtail binary zip file has been downloaded and stored in: %s", - file_path, - ) - - decompressed_file = GzipFile(fileobj=BytesIO(file_bytes)) - binary_path = os.path.join(BINARY_DIR, promtail_info["filename"]) - with open(binary_path, "wb") as outfile: - outfile.write(decompressed_file.read()) - logger.debug("Promtail binary file has been downloaded.") - - workload_binary_path = os.path.join(WORKLOAD_BINARY_DIR, promtail_info["filename"]) - self._push_binary_to_workload(container, binary_path, workload_binary_path) - - @property - def _cli_args(self) -> str: - """Return the cli arguments to pass to promtail. - - Returns: - The arguments as a string - """ - return "-config.file={}".format(WORKLOAD_CONFIG_PATH) - - def _current_config(self, container) -> dict: - """Property that returns the current Promtail configuration. - - Returns: - A dict containing Promtail configuration. - """ - if not container.can_connect(): - logger.debug("Could not connect to promtail container!") - return {} - try: - raw_current = container.pull(WORKLOAD_CONFIG_PATH).read() - return yaml.safe_load(raw_current) - except (ProtocolError, PathError) as e: - logger.warning( - "Could not check the current promtail configuration due to " - "a failure in retrieving the file: %s", - e, - ) - return {} - - def _promtail_config(self, container_name: str) -> dict: - """Generates the config file for Promtail. - - Reference: https://grafana.com/docs/loki/latest/send-data/promtail/configuration - """ - config = {"clients": self._clients_list()} - if self.insecure_skip_verify: - for client in config["clients"]: - client["tls_config"] = {"insecure_skip_verify": True} - - config.update(self._server_config(container_name)) - config.update(self._positions) - config.update(self._scrape_configs(container_name)) - return config - - def _clients_list(self) -> list: - """Generates a list of clients for use in the promtail config. - - Returns: - A list of endpoints - """ - return self.loki_endpoints - - def _server_config(self, container_name: str) -> dict: - """Generates the server section of the Promtail config file. - - Returns: - A dict representing the `server` section. - """ - return { - "server": { - "http_listen_port": self._promtails_ports[container_name]["http_listen_port"], - "grpc_listen_port": self._promtails_ports[container_name]["grpc_listen_port"], - } - } - - @property - def _positions(self) -> dict: - """Generates the positions section of the Promtail config file. - - Returns: - A dict representing the `positions` section. - """ - return {"positions": {"filename": WORKLOAD_POSITIONS_PATH}} - - def _scrape_configs(self, container_name: str) -> dict: - """Generates the scrape_configs section of the Promtail config file. - - Returns: - A dict representing the `scrape_configs` section. - """ - job_name = f"juju_{self.topology.identifier}" - - # The new JujuTopology doesn't include unit, but LogProxyConsumer should have it - common_labels = { - f"juju_{k}": v - for k, v in self.topology.as_dict(remapped_keys={"charm_name": "charm"}).items() - } - common_labels["container"] = container_name - scrape_configs = [] - - # Files config - labels = common_labels.copy() - labels.update( - { - "job": job_name, - "__path__": "", - } - ) - config = {"targets": ["localhost"], "labels": labels} - scrape_config = { - "job_name": "system", - "static_configs": self._generate_static_configs(config, container_name), - } - scrape_configs.append(scrape_config) - - # Syslog config - syslog_port = self._logs_scheme.get(container_name, {}).get("syslog-port") - if syslog_port: - relabel_mappings = [ - "severity", - "facility", - "hostname", - "app_name", - "proc_id", - "msg_id", - ] - syslog_labels = common_labels.copy() - syslog_labels.update({"job": f"{job_name}_syslog"}) - syslog_config = { - "job_name": "syslog", - "syslog": { - "listen_address": f"127.0.0.1:{syslog_port}", - "label_structured_data": True, - "labels": syslog_labels, - }, - "relabel_configs": [ - {"source_labels": [f"__syslog_message_{val}"], "target_label": val} - for val in relabel_mappings - ] - + [{"action": "labelmap", "regex": "__syslog_message_sd_(.+)"}], - } - scrape_configs.append(syslog_config) # type: ignore - - return {"scrape_configs": scrape_configs} - - def _generate_static_configs(self, config: dict, container_name: str) -> list: - """Generates static_configs section. - - Returns: - - a list of dictionaries representing static_configs section - """ - static_configs = [] - - for _file in self._logs_scheme.get(container_name, {}).get("log-files", []): - conf = deepcopy(config) - conf["labels"]["__path__"] = _file - static_configs.append(conf) - - return static_configs - - def _setup_promtail(self, container: Container) -> None: - # Use the first - relations = self._charm.model.relations[self._relation_name] - if len(relations) > 1: - logger.debug( - "Multiple log_proxy relations. Getting Promtail from application {}".format( - relations[0].app.name - ) - ) - relation = relations[0] - promtail_binaries = json.loads( - relation.data[relation.app].get("promtail_binary_zip_url", "{}") - ) - if not promtail_binaries: - return - - self._create_directories(container) - self._ensure_promtail_binary(promtail_binaries, container) - - container.push( - WORKLOAD_CONFIG_PATH, - yaml.safe_dump(self._promtail_config(container.name)), - make_dirs=True, - ) - - workload_binary_path = os.path.join( - WORKLOAD_BINARY_DIR, promtail_binaries[self._arch]["filename"] - ) - self._add_pebble_layer(workload_binary_path, container) - - if self._current_config(container).get("clients"): - try: - container.restart(WORKLOAD_SERVICE_NAME) - except ChangeError as e: - self.on.promtail_digest_error.emit(str(e)) - else: - self.on.log_proxy_endpoint_joined.emit() - else: - self.on.promtail_digest_error.emit("No promtail client endpoints available!") - - def _ensure_promtail_binary(self, promtail_binaries: dict, container: Container): - if self._is_promtail_installed(promtail_binaries[self._arch], container): - return - - try: - self._obtain_promtail(promtail_binaries[self._arch], container) - except URLError as e: - msg = f"Promtail binary couldn't be downloaded - {str(e)}" - logger.warning(msg) - self.on.promtail_digest_error.emit(msg) - - def _is_promtail_installed(self, promtail_info: dict, container: Container) -> bool: - """Determine if promtail has already been installed to the container. - - Args: - promtail_info: dictionary containing information about promtail binary - that must be used. The dictionary must at least contain a key - "filename" giving the name of promtail binary - container: container in which to check whether promtail is installed. - """ - workload_binary_path = f"{WORKLOAD_BINARY_DIR}/{promtail_info['filename']}" - try: - container.list_files(workload_binary_path) - except (APIError, FileNotFoundError): - return False - return True - - def _generate_promtails_ports(self, logs_scheme) -> dict: - return { - container: { - "http_listen_port": HTTP_LISTEN_PORT_START + 2 * i, - "grpc_listen_port": GRPC_LISTEN_PORT_START + 2 * i, - } - for i, container in enumerate(logs_scheme.keys()) - } - - def syslog_port(self, container_name: str) -> str: - """Gets the port on which promtail is listening for syslog in this container. - - Returns: - A str representing the port - """ - return str(self._logs_scheme.get(container_name, {}).get("syslog-port")) - - def rsyslog_config(self, container_name: str) -> str: - """Generates a config line for use with rsyslog. - - Returns: - The rsyslog config line as a string - """ - return 'action(type="omfwd" protocol="tcp" target="127.0.0.1" port="{}" Template="RSYSLOG_SyslogProtocol23Format" TCP_Framing="octet-counted")'.format( - self._logs_scheme.get(container_name, {}).get("syslog-port") - ) - - @property - def _containers(self) -> Dict[str, Container]: - return {cont: self._charm.unit.get_container(cont) for cont in self._logs_scheme.keys()} - - -class _PebbleLogClient: - @staticmethod - def check_juju_version() -> bool: - """Make sure the Juju version supports Log Forwarding.""" - juju_version = JujuVersion.from_environ() - if not juju_version > JujuVersion(version=str("3.3")): - msg = f"Juju version {juju_version} does not support Pebble log forwarding. Juju >= 3.4 is needed." - logger.warning(msg) - return False - return True - - @staticmethod - def _build_log_target( - unit_name: str, loki_endpoint: str, topology: JujuTopology, enable: bool - ) -> Dict: - """Build a log target for the log forwarding Pebble layer. - - Log target's syntax for enabling/disabling forwarding is explained here: - https://github.com/canonical/pebble?tab=readme-ov-file#log-forwarding - """ - services_value = ["all"] if enable else ["-all"] - - log_target = { - "override": "replace", - "services": services_value, - "type": "loki", - "location": loki_endpoint, - } - if enable: - log_target.update( - { - "labels": { - "product": "Juju", - "charm": topology._charm_name, - "juju_model": topology._model, - "juju_model_uuid": topology._model_uuid, - "juju_application": topology._application, - "juju_unit": topology._unit, - "job": f"juju_{topology.identifier}", - }, - } - ) - - return {unit_name: log_target} - - @staticmethod - def _build_log_targets( - loki_endpoints: Optional[Dict[str, str]], topology: JujuTopology, enable: bool - ): - """Build all the targets for the log forwarding Pebble layer.""" - targets = {} - if not loki_endpoints: - return targets - - for unit_name, endpoint in loki_endpoints.items(): - targets.update( - _PebbleLogClient._build_log_target( - unit_name=unit_name, - loki_endpoint=endpoint, - topology=topology, - enable=enable, - ) - ) - return targets - - @staticmethod - def disable_inactive_endpoints( - container: Container, active_endpoints: Dict[str, str], topology: JujuTopology - ): - """Disable forwarding for inactive endpoints by checking against the Pebble plan.""" - pebble_layer = container.get_plan().to_dict().get("log-targets", None) - if not pebble_layer: - return - - for unit_name, target in pebble_layer.items(): - # If the layer is a disabled log forwarding endpoint, skip it - if "-all" in target["services"]: # pyright: ignore - continue - - if unit_name not in active_endpoints: - layer = Layer( - { # pyright: ignore - "log-targets": _PebbleLogClient._build_log_targets( - loki_endpoints={unit_name: "(removed)"}, - topology=topology, - enable=False, - ) - } - ) - container.add_layer(f"{container.name}-log-forwarding", layer=layer, combine=True) - - @staticmethod - def enable_endpoints( - container: Container, active_endpoints: Dict[str, str], topology: JujuTopology - ): - """Enable forwarding for the specified Loki endpoints.""" - layer = Layer( - { # pyright: ignore - "log-targets": _PebbleLogClient._build_log_targets( - loki_endpoints=active_endpoints, - topology=topology, - enable=True, - ) - } - ) - container.add_layer(f"{container.name}-log-forwarding", layer, combine=True) - - -class LogForwarder(ConsumerBase): - """Forward the standard outputs of all workloads operated by a charm to one or multiple Loki endpoints. - - This class implements Pebble log forwarding. Juju >= 3.4 is needed. - """ - - def __init__( - self, - charm: CharmBase, - *, - relation_name: str = DEFAULT_RELATION_NAME, - alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, - recursive: bool = True, - skip_alert_topology_labeling: bool = False, - refresh_event: Optional[Union[BoundEvent, List[BoundEvent]]] = None, - forward_alert_rules: bool = True, - ): - _PebbleLogClient.check_juju_version() - super().__init__( - charm, - relation_name, - alert_rules_path, - recursive, - skip_alert_topology_labeling, - forward_alert_rules=forward_alert_rules, - ) - self._charm = charm - self._relation_name = relation_name - - on = self._charm.on[self._relation_name] - self.framework.observe(on.relation_joined, self._update_logging) - self.framework.observe(on.relation_changed, self._update_logging) - self.framework.observe(on.relation_departed, self._update_logging) - self.framework.observe(on.relation_broken, self._update_logging) - - if refresh_event: - if not isinstance(refresh_event, list): - refresh_event = [refresh_event] - for ev in refresh_event: - self.framework.observe(ev, self._update_logging) - - for container_name in self._charm.meta.containers.keys(): - snake_case_container_name = container_name.replace("-", "_") - self.framework.observe( - getattr(self._charm.on, f"{snake_case_container_name}_pebble_ready"), - self._on_pebble_ready, - ) - - def _on_pebble_ready(self, event: PebbleReadyEvent): - if not (loki_endpoints := self._retrieve_endpoints_from_relation()): - logger.warning("No Loki endpoints available") - return - - self._update_endpoints(event.workload, loki_endpoints) - - def _update_logging(self, event: RelationEvent): - """Update the log forwarding to match the active Loki endpoints.""" - if not (loki_endpoints := self._retrieve_endpoints_from_relation()): - logger.warning("No Loki endpoints available") - return - - for container in self._charm.unit.containers.values(): - if container.can_connect(): - self._update_endpoints(container, loki_endpoints) - # else: `_update_endpoints` will be called on pebble-ready anyway. - - self._handle_alert_rules(event.relation) - - def _retrieve_endpoints_from_relation(self) -> dict: - loki_endpoints = {} - - # Get the endpoints from relation data - for relation in self._charm.model.relations[self._relation_name]: - loki_endpoints.update(self._fetch_endpoints(relation)) - - return loki_endpoints - - def _update_endpoints(self, container: Container, loki_endpoints: dict): - _PebbleLogClient.disable_inactive_endpoints( - container=container, - active_endpoints=loki_endpoints, - topology=self.topology, - ) - _PebbleLogClient.enable_endpoints( - container=container, active_endpoints=loki_endpoints, topology=self.topology - ) - - def is_ready(self, relation: Optional[Relation] = None): - """Check if the relation is active and healthy.""" - if not relation: - relations = self._charm.model.relations[self._relation_name] - if not relations: - return False - return all(self.is_ready(relation) for relation in relations) - - try: - if self._extract_urls(relation): - return True - return False - except (KeyError, json.JSONDecodeError): - return False - - def _extract_urls(self, relation: Relation) -> Dict[str, str]: - """Default getter function to extract Loki endpoints from a relation. - - Returns: - A dictionary of remote units and the respective Loki endpoint. - { - "loki/0": "http://loki:3100/loki/api/v1/push", - "another-loki/0": "http://another-loki:3100/loki/api/v1/push", - } - """ - endpoints: Dict = {} - - for unit in relation.units: - endpoint = relation.data[unit]["endpoint"] - deserialized_endpoint = json.loads(endpoint) - url = deserialized_endpoint["url"] - endpoints[unit.name] = url - - return endpoints - - def _fetch_endpoints(self, relation: Relation) -> Dict[str, str]: - """Fetch Loki Push API endpoints from relation data using the endpoints getter.""" - endpoints: Dict = {} - - if not self.is_ready(relation): - logger.warning(f"The relation '{relation.name}' is not ready yet.") - return endpoints - - # if the code gets here, the function won't raise anymore because it's - # also called in is_ready() - endpoints = self._extract_urls(relation) - - return endpoints - - -def charm_logging_config( - endpoint_requirer: LokiPushApiConsumer, cert_path: Optional[Union[Path, str]] -) -> Tuple[Optional[List[str]], Optional[str]]: - """Utility function to determine the charm_logging config you will likely want. - - If no endpoint is provided: - disable charm logging. - If https endpoint is provided but cert_path is not found on disk: - disable charm logging. - If https endpoint is provided and cert_path is None: - ERROR - Else: - proceed with charm logging (with or without tls, as appropriate) - - Args: - endpoint_requirer: an instance of LokiPushApiConsumer. - cert_path: a path where a cert is stored. - - Returns: - A tuple with (optionally) the values of the endpoints and the certificate path. - - Raises: - LokiPushApiError: if some endpoint are http and others https. - """ - endpoints = [ep["url"] for ep in endpoint_requirer.loki_endpoints] - if not endpoints: - return None, None - - https = tuple(endpoint.startswith("https://") for endpoint in endpoints) - - if all(https): # all endpoints are https - if cert_path is None: - raise LokiPushApiError("Cannot send logs to https endpoints without a certificate.") - if not Path(cert_path).exists(): - # if endpoints is https BUT we don't have a server_cert yet: - # disable charm logging until we do to prevent tls errors - return None, None - return endpoints, str(cert_path) - - if all(not x for x in https): # all endpoints are http - return endpoints, None - - # if there's a disagreement, that's very weird: - raise LokiPushApiError("Some endpoints are http, some others are https. That's not good.") From c0d7dbb36082042dd9eb84bd8a3608bd638b3c0c Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Tue, 15 Sep 2026 12:37:16 +0200 Subject: [PATCH 2/3] fix: Workaround for charmcraft -o packing issue --- tests/integration/conftest.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6088865..373d416 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -47,11 +47,27 @@ def charm_and_channel_and_resources( except subprocess.CalledProcessError: logger.warning("Failed to build Loki %s. Trying again!", role) continue + pth = _resolve_packed_charm(pth, REPO_ROOT / role) os.environ[charm_path_key] = str(pth) return pth, None, get_resources(REPO_ROOT / role) raise subprocess.CalledProcessError(1, f"pack {role}") +def _resolve_packed_charm(packed: Path, project_dir: Path) -> Path: + # charmcraft 4.4.1 and 4.4.2 ignore the output directory passed to + # `charmcraft pack` and leave the .charm file inside the project + # directory instead: https://github.com/canonical/charmcraft/issues/2854 + # Same workaround as Juju's test helpers: https://github.com/juju/juju/pull/23174 + if packed.is_file(): + return packed + in_project_dir = project_dir / packed.name + if in_project_dir.is_file(): + return in_project_dir + raise FileNotFoundError( + f"packed charm {packed.name} not found in {packed.parent} or {project_dir}" + ) + + @fixture(scope="session") def coordinator_charm(): """Loki coordinator used for integration testing.""" From 1ab42cdcde03def9181be3a55c3d791c88128933 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Wed, 16 Sep 2026 16:50:17 +0200 Subject: [PATCH 3/3] fix: Bring back loki_push_api in worker --- .../lib/charms/loki_k8s/v1/loki_push_api.py | 2856 +++++++++++++++++ 1 file changed, 2856 insertions(+) create mode 100644 worker/lib/charms/loki_k8s/v1/loki_push_api.py diff --git a/worker/lib/charms/loki_k8s/v1/loki_push_api.py b/worker/lib/charms/loki_k8s/v1/loki_push_api.py new file mode 100644 index 0000000..3207d68 --- /dev/null +++ b/worker/lib/charms/loki_k8s/v1/loki_push_api.py @@ -0,0 +1,2856 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +# +# Learn more at: https://juju.is/docs/sdk + +r"""## Overview. + +This document explains how to use the two principal objects this library provides: + +- `LokiPushApiProvider`: This object is meant to be used by any Charmed Operator that needs to +implement the provider side of the `loki_push_api` relation interface. For instance, a Loki charm. +The provider side of the relation represents the server side, to which logs are being pushed. + +- `LokiPushApiConsumer`: This object is meant to be used by any Charmed Operator that needs to +send log to Loki by implementing the consumer side of the `loki_push_api` relation interface. +For instance, a Promtail or Grafana agent charm which needs to send logs to Loki. + +- `LogProxyConsumer`: DEPRECATED. +This object can be used by any Charmed Operator which needs to send telemetry, such as logs, to +Loki through a Log Proxy by implementing the consumer side of the `loki_push_api` relation +interface. +In order to be able to control the labels on the logs pushed this object adds a Pebble layer +that runs Promtail in the workload container, injecting Juju topology labels into the +logs on the fly. +This object is deprecated. Consider migrating to LogForwarder with the release of Juju 3.6 LTS. + +- `LogForwarder`: This object can be used by any Charmed Operator which needs to send the workload +standard output (stdout) through Pebble's log forwarding mechanism, to Loki endpoints through the +`loki_push_api` relation interface. +In order to be able to control the labels on the logs pushed this object updates the pebble layer's +"log-targets" section with Juju topology. + +Filtering logs in Loki is largely performed on the basis of labels. In the Juju ecosystem, Juju +topology labels are used to uniquely identify the workload which generates telemetry like logs. + + +## LokiPushApiProvider Library Usage + +This object may be used by any Charmed Operator which implements the `loki_push_api` interface. +For instance, Loki or Grafana Agent. + +For this purpose a charm needs to instantiate the `LokiPushApiProvider` object with one mandatory +and three optional arguments. + +- `charm`: A reference to the parent (Loki) charm. + +- `relation_name`: The name of the relation that the charm uses to interact + with its clients, which implement `LokiPushApiConsumer` `LogForwarder`, or `LogProxyConsumer` + (note that LogProxyConsumer is deprecated). + + If provided, this relation name must match a provided relation in metadata.yaml with the + `loki_push_api` interface. + + The default relation name is "logging" for `LokiPushApiConsumer` and `LogForwarder`, and + "log-proxy" for `LogProxyConsumer` (note that LogProxyConsumer is deprecated). + + For example, a provider's `metadata.yaml` file may look as follows: + + ```yaml + provides: + logging: + interface: loki_push_api + ``` + + Subsequently, a Loki charm may instantiate the `LokiPushApiProvider` in its constructor as + follows: + + from charms.loki_k8s.v1.loki_push_api import LokiPushApiProvider + from loki_server import LokiServer + ... + + class LokiOperatorCharm(CharmBase): + ... + + def __init__(self, *args): + super().__init__(*args) + ... + external_url = urlparse(self._external_url) + self.loki_provider = LokiPushApiProvider( + self, + port=external_url.port or 80, + scheme=external_url.scheme, + path=f"{external_url.path}/loki/api/v1/push", + ) + ... + + - `port`: Loki Push Api endpoint port. Default value: `3100`. + - `scheme`: Loki Push Api endpoint scheme (`HTTP` or `HTTPS`). Default value: `HTTP` + - `address`: Loki Push Api endpoint address. Default value: `localhost` + - `path`: Loki Push Api endpoint path. Default value: `loki/api/v1/push` + + +The `LokiPushApiProvider` object has several responsibilities: + +1. Set the URL of the Loki Push API in the relation application data bag; the URL + must be unique to all instances (e.g. using a load balancer). + The default URL is the FQDN, but this can be overridden by calling `update_endpoint()`. + +2. Set the Promtail binary URL (`promtail_binary_zip_url`) so clients that use + `LogProxyConsumer` object could download and configure it. + +3. Process the metadata of the consumer application, provided via the + "metadata" field of the consumer data bag, which are used to annotate the + alert rules (see next point). An example for "metadata" is the following: + + {'model': 'loki', + 'model_uuid': '0b7d1071-ded2-4bf5-80a3-10a81aeb1386', + 'application': 'promtail-k8s' + } + +4. Process alert rules set into the relation by the `LokiPushApiConsumer` + objects, e.g.: + + '{ + "groups": [{ + "name": "loki_0b7d1071-ded2-4bf5-80a3-10a81aeb1386_promtail-k8s_alerts", + "rules": [{ + "alert": "HighPercentageError", + "expr": "sum(rate({app=\\"foo\\", env=\\"production\\"} |= \\"error\\" [5m])) + by (job) \\n /\\nsum(rate({app=\\"foo\\", env=\\"production\\"}[5m])) + by (job)\\n > 0.05 + \\n", "for": "10m", + "labels": { + "severity": "page", + "juju_model": "loki", + "juju_model_uuid": "0b7d1071-ded2-4bf5-80a3-10a81aeb1386", + "juju_application": "promtail-k8s" + }, + "annotations": { + "summary": "High request latency" + } + }] + }] + }' + + +Once these alert rules are sent over relation data, the `LokiPushApiProvider` object +stores these files in the directory `/loki/rules` inside the Loki charm container. After +storing alert rules files, the object will check alert rules by querying Loki API +endpoint: [`loki/api/v1/rules`](https://grafana.com/docs/loki/latest/api/#list-rule-groups). +If there are changes in the alert rules a `loki_push_api_alert_rules_changed` event will +be emitted with details about the `RelationEvent` which triggered it. + +This events should be observed in the charm that uses `LokiPushApiProvider`: + +```python + def __init__(self, *args): + super().__init__(*args) + ... + self.loki_provider = LokiPushApiProvider(self) + self.framework.observe( + self.loki_provider.on.loki_push_api_alert_rules_changed, + self._loki_push_api_alert_rules_changed, + ) +``` + + +## LokiPushApiConsumer Library Usage + +This Loki charm interacts with its clients using the Loki charm library. Charms +seeking to send log to Loki, must do so using the `LokiPushApiConsumer` object from +this charm library. + +> **NOTE**: `LokiPushApiConsumer` also depends on an additional charm library. +> +> Ensure sure you `charmcraft fetch-lib charms.observability_libs.v0.juju_topology` +> when using this library. + +For the simplest use cases, using the `LokiPushApiConsumer` object only requires +instantiating it, typically in the constructor of your charm (the one which +sends logs). + +```python +from charms.loki_k8s.v1.loki_push_api import LokiPushApiConsumer + +class LokiClientCharm(CharmBase): + + def __init__(self, *args): + super().__init__(*args) + ... + self._loki_consumer = LokiPushApiConsumer(self) +``` + +The `LokiPushApiConsumer` constructor requires two things: + +- A reference to the parent (LokiClientCharm) charm. + +- Optionally, the name of the relation that the Loki charm uses to interact + with its clients. If provided, this relation name must match a required + relation in metadata.yaml with the `loki_push_api` interface. + + If not provided, the relation name defaults to `logging`. + +Any time the relation between a Loki provider charm and a Loki consumer charm is +established, a `LokiPushApiEndpointJoined` event is fired. In the consumer side +is it possible to observe this event with: + +```python + +self.framework.observe( + self._loki_consumer.on.loki_push_api_endpoint_joined, + self._on_loki_push_api_endpoint_joined, +) +``` + +Any time there are departures in relations between the consumer charm and Loki +the consumer charm is informed, through a `LokiPushApiEndpointDeparted` event, for instance: + +```python +self.framework.observe( + self._loki_consumer.on.loki_push_api_endpoint_departed, + self._on_loki_push_api_endpoint_departed, +) +``` + +The consumer charm can then choose to update its configuration in both situations. + +Note that LokiPushApiConsumer does not add any labels automatically on its own. In +order to better integrate with the Canonical Observability Stack, you may want to configure your +software to add Juju topology labels. The +[observability-libs](https://charmhub.io/observability-libs) library can be used to get topology +labels in charm code. See :func:`LogProxyConsumer._scrape_configs` for an example of how +to do this with promtail. + +## LogProxyConsumer Library Usage + +> Note: This object is deprecated. Consider migrating to LogForwarder with the release of Juju 3.6 +> LTS. + +Let's say that we have a workload charm that produces logs, and we need to send those logs to a +workload implementing the `loki_push_api` interface, such as `Loki` or `Grafana Agent`. + +Adopting this object in a Charmed Operator consist of two steps: + +1. Use the `LogProxyConsumer` class by instantiating it in the `__init__` method of the charmed + operator. There are two ways to get logs in to promtail. You can give it a list of files to + read, or you can write to it using the syslog protocol. + + For example: + + ```python + from charms.loki_k8s.v1.loki_push_api import LogProxyConsumer + + ... + + def __init__(self, *args): + ... + self._log_proxy = LogProxyConsumer( + self, + logs_scheme={ + "workload-a": { + "log-files": ["/tmp/worload-a-1.log", "/tmp/worload-a-2.log"], + "syslog-port": 1514, + }, + "workload-b": {"log-files": ["/tmp/worload-b.log"], "syslog-port": 1515}, + }, + relation_name="log-proxy", + ) + self.framework.observe( + self._log_proxy.on.promtail_digest_error, + self._promtail_error, + ) + + def _promtail_error(self, event): + logger.error(event.message) + self.unit.status = BlockedStatus(event.message) + ``` + + Any time the relation between a provider charm and a LogProxy consumer charm is + established, a `LogProxyEndpointJoined` event is fired. In the consumer side is it + possible to observe this event with: + + ```python + + self.framework.observe( + self._log_proxy.on.log_proxy_endpoint_joined, + self._on_log_proxy_endpoint_joined, + ) + ``` + + Any time there are departures in relations between the consumer charm and the provider + the consumer charm is informed, through a `LogProxyEndpointDeparted` event, for instance: + + ```python + self.framework.observe( + self._log_proxy.on.log_proxy_endpoint_departed, + self._on_log_proxy_endpoint_departed, + ) + ``` + + The consumer charm can then choose to update its configuration in both situations. + + Note that: + + - You can configure your syslog software using `localhost` as the address and the method + `LogProxyConsumer.syslog_port("container_name")` to get the port, or, alternatively, if you are using rsyslog + you may use the method `LogProxyConsumer.rsyslog_config("container_name")`. + +2. Modify the `metadata.yaml` file to add: + + - The `log-proxy` relation in the `requires` section: + ```yaml + requires: + log-proxy: + interface: loki_push_api + optional: true + ``` + +Once the library is implemented in a Charmed Operator and a relation is established with +the charm that implements the `loki_push_api` interface, the library will inject a +Pebble layer that runs Promtail in the workload container to send logs. + +By default, the promtail binary injected into the container will be downloaded from the internet. +If, for any reason, the container has limited network access, you may allow charm administrators +to provide their own promtail binary at runtime by adding the following snippet to your charm +metadata: + +```yaml +resources: + promtail-bin: + type: file + description: Promtail binary for logging + filename: promtail-linux +``` + +Which would then allow operators to deploy the charm this way: + +``` +juju deploy \ + ./your_charm.charm \ + --resource promtail-bin=/tmp/promtail-linux-amd64 +``` + +If a different resource name is used, it can be specified with the `promtail_resource_name` +argument to the `LogProxyConsumer` constructor. + +The object can emit a `PromtailDigestError` event: + +- Promtail binary cannot be downloaded. +- The sha256 sum mismatch for promtail binary. + +The object can raise a `ContainerNotFoundError` event: + +- No `container_name` parameter has been specified and the Pod has more than 1 container. + +These can be monitored via the PromtailDigestError events via: + +```python + self.framework.observe( + self._loki_consumer.on.promtail_digest_error, + self._promtail_error, + ) + + def _promtail_error(self, event): + logger.error(msg) + self.unit.status = BlockedStatus(event.message) + ) +``` + +## LogForwarder class Usage + +Let's say that we have a charm's workload that writes logs to the standard output (stdout), +and we need to send those logs to a workload implementing the `loki_push_api` interface, +such as `Loki` or `Grafana Agent`. To know how to reach a Loki instance, a charm would +typically use the `loki_push_api` interface. + +Use the `LogForwarder` class by instantiating it in the `__init__` method of the charm: + +```python +from charms.loki_k8s.v1.loki_push_api import LogForwarder + +... + + def __init__(self, *args): + ... + self._log_forwarder = LogForwarder( + self, + relation_name="logging" # optional, defaults to `logging` + ) +``` + +The `LogForwarder` by default will observe relation events on the `logging` endpoint and +enable/disable log forwarding automatically. +Next, modify the `metadata.yaml` file to add: + +The `log-forwarding` relation in the `requires` section: +```yaml +requires: + logging: + interface: loki_push_api + optional: true +``` + +Once the LogForwader class is implemented in your charm and the relation (implementing the +`loki_push_api` interface) is active and healthy, the library will inject a Pebble layer in +each workload container the charm has access to, to configure Pebble's log forwarding +feature and start sending logs to Loki. + +## Alerting Rules + +This charm library also supports gathering alerting rules from all related Loki client +charms and enabling corresponding alerts within the Loki charm. Alert rules are +automatically gathered by `LokiPushApiConsumer` object from a directory conventionally +named `loki_alert_rules`. + +This directory must reside at the top level in the `src` folder of the +consumer charm. Each file in this directory is assumed to be a single alert rule +in YAML format. The file name must have one of the following extensions: `.yaml`, `.yml`, `.rule`, or `.rules`. +The format of this alert rule conforms to the +[Loki docs](https://grafana.com/docs/loki/latest/rules/#alerting-rules). + +An example of the contents of one such file is shown below. + +```yaml +alert: HighPercentageError +expr: | + sum(rate({%%juju_topology%%} |= "error" [5m])) by (job) + / + sum(rate({%%juju_topology%%}[5m])) by (job) + > 0.05 +for: 10m +labels: + severity: page +annotations: + summary: High request latency + +``` + +It is **critical** to use the `%%juju_topology%%` filter in the expression for the alert +rule shown above. This filter is a stub that is automatically replaced by the +`LokiPushApiConsumer` following Loki Client's Juju topology (application, model and its +UUID). Such a topology filter is essential to ensure that alert rules submitted by one +provider charm generates alerts only for that same charm. + +The Loki charm may be related to multiple Loki client charms. Without this, filter +rules submitted by one provider charm will also result in corresponding alerts for other +provider charms. Hence, every alert rule expression must include such a topology filter stub. + +Gathering alert rules and generating rule files within the Loki charm is easily done using +the `alerts()` method of `LokiPushApiProvider`. Alerts generated by Loki will automatically +include Juju topology labels in the alerts. These labels indicate the source of the alert. + +The following labels are automatically added to every alert + +- `juju_model` +- `juju_model_uuid` +- `juju_application` + + +Whether alert rules files does not contain the keys `alert` or `expr` or there is no alert +rules file in `alert_rules_path` a `loki_push_api_alert_rules_error` event is emitted. + +To handle these situations the event must be observed in the `LokiClientCharm` charm.py file: + +```python +class LokiClientCharm(CharmBase): + + def __init__(self, *args): + super().__init__(*args) + ... + self._loki_consumer = LokiPushApiConsumer(self) + + self.framework.observe( + self._loki_consumer.on.loki_push_api_alert_rules_error, + self._alert_rules_error + ) + + def _alert_rules_error(self, event): + self.unit.status = BlockedStatus(event.message) +``` + +## Relation Data + +The Loki charm uses both application and unit relation data to obtain information regarding +Loki Push API and alert rules. + +Units of consumer charm send their alert rules over app relation data using the `alert_rules` +key. + +## Alert rules encoding + +The consumer publishes its alert rules to the `alert_rules` key of its application +databag. Because large deployments can produce enough alert rules to exceed Juju's +relation data size limit, the rules can be stored LZMA-compressed and base64-encoded +instead of as plain JSON. + +Compression is negotiated over the relation: the provider advertises the encodings it +is able to read in the `alert_rules_encodings` key of its own application databag, and +the consumer picks the best encoding both sides support. A consumer related to a +provider running an older version of this library (which advertises nothing) keeps +writing plain JSON, so upgrades are safe in any order. + +An admin can decode compressed rules with: +```bash + | base64 -d | xz -d | jq +``` + +## Charm logging +The `charms.loki_k8s.v0.charm_logging` library can be used in conjunction with this one to configure python's +logging module to forward all logs to Loki via the loki-push-api interface. + +```python +from lib.charms.loki_k8s.v0.charm_logging import log_charm +from lib.charms.loki_k8s.v1.loki_push_api import charm_logging_config, LokiPushApiConsumer + +@log_charm(logging_endpoint="my_endpoints", server_cert="cert_path") +class MyCharm(...): + _cert_path = "/path/to/cert/on/charm/container.crt" + def __init__(self, ...): + self.logging = LokiPushApiConsumer(...) + self.my_endpoints, self.cert_path = charm_logging_config( + self.logging, self._cert_path) +``` + +Do this, and all charm logs will be forwarded to Loki as soon as a relation is formed. +""" + +import copy +import json +import logging +import lzma +import os +import platform +import re +import socket +import warnings +from copy import deepcopy +from gzip import GzipFile +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from typing import Any, Dict, Final, List, Mapping, Optional, Tuple, Union, cast +from urllib import request +from urllib.error import URLError + +import yaml +from cosl import CosTool, JujuTopology, LZMABase64 +from cosl.rules import AlertRules +from cosl.types import OfficialRuleFileFormat +from ops.charm import ( + CharmBase, + HookEvent, + PebbleReadyEvent, + RelationBrokenEvent, + RelationCreatedEvent, + RelationDepartedEvent, + RelationEvent, + RelationJoinedEvent, + RelationRole, + WorkloadEvent, +) +from ops.framework import BoundEvent, EventBase, EventSource, Object, ObjectEvents +from ops.jujuversion import JujuVersion +from ops.model import Container, ModelError, Relation +from ops.pebble import APIError, ChangeError, Layer, PathError, ProtocolError + +# The unique Charmhub library identifier, never change it +LIBID = "bf76f23cdd03464b877c52bd1d2f563e" + +# Increment this major API version when introducing breaking changes +LIBAPI = 1 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 34 + +PYDEPS = ["cosl"] + +logger = logging.getLogger(__name__) + +RELATION_INTERFACE_NAME = "loki_push_api" +DEFAULT_RELATION_NAME = "logging" +DEFAULT_ALERT_RULES_RELATIVE_PATH = "./src/loki_alert_rules" +DEFAULT_LOG_PROXY_RELATION_NAME = "log-proxy" + +PROMTAIL_BASE_URL = "https://github.com/canonical/loki-k8s-operator/releases/download" +# To update Promtail version you only need to change the PROMTAIL_VERSION and +# update all sha256 sums in PROMTAIL_BINARIES. To support a new architecture +# you only need to add a new key value pair for the architecture in PROMTAIL_BINARIES. +PROMTAIL_VERSION = "v2.9.7" +PROMTAIL_ARM_BINARY = { + "filename": "promtail-static-arm64", + "zipsha": "c083fdb45e5c794103f974eeb426489b4142438d9e10d0ae272b2aff886e249b", + "binsha": "4cd055c477a301c0bdfdbcea514e6e93f6df5d57425ce10ffc77f3e16fec1ddf", +} + +PROMTAIL_BINARIES = { + "amd64": { + "filename": "promtail-static-amd64", + "zipsha": "6873cbdabf23062aeefed6de5f00ff382710332af3ab90a48c253ea17e08f465", + "binsha": "28da9b99f81296fe297831f3bc9d92aea43b4a92826b8ff04ba433b8cb92fb50", + }, + "arm64": PROMTAIL_ARM_BINARY, + "aarch64": PROMTAIL_ARM_BINARY, +} + +# Paths in `charm` container +BINARY_DIR = "/tmp" + +# Paths in `workload` container +WORKLOAD_BINARY_DIR = "/opt/promtail" +WORKLOAD_CONFIG_DIR = "/etc/promtail" +WORKLOAD_CONFIG_FILE_NAME = "promtail_config.yaml" +WORKLOAD_CONFIG_PATH = "{}/{}".format(WORKLOAD_CONFIG_DIR, WORKLOAD_CONFIG_FILE_NAME) +WORKLOAD_POSITIONS_PATH = "{}/positions.yaml".format(WORKLOAD_BINARY_DIR) +WORKLOAD_SERVICE_NAME = "promtail" + +# These are the initial port values. As we can have more than one container, +# we use odd and even numbers to avoid collisions. +# Each new container adds 2 to the previous value. +HTTP_LISTEN_PORT_START = 9080 # even start port +GRPC_LISTEN_PORT_START = 9095 # odd start port + +ALERT_RULES_KEY: Final[str] = "alert_rules" +"""Databag key holding the consumer's alert rules.""" + +ALERT_RULES_ENCODINGS_KEY: Final[str] = "alert_rules_encodings" +"""Databag key with which the provider advertises the encodings it can read.""" + +JSON_ENCODING: Final[str] = "json" +"""Plain JSON alert rules, as written by every version of this library.""" + +LZMA_ENCODING: Final[str] = "lzma" +"""LZMA-compressed, base64-encoded JSON alert rules.""" + +SUPPORTED_ALERT_RULES_ENCODINGS: Final[Tuple[str, ...]] = (LZMA_ENCODING, JSON_ENCODING) +"""Alert rules encodings this library can read and write, most preferred first. + +This is in preference order, not sorted: it is a constant, so the bytes written to the +databag are stable across hooks, which is what matters for avoiding spurious +relation-changed events. +""" + + +def _encode_alert_rules(rules: Mapping[str, Any], encoding: str = JSON_ENCODING) -> str: + """Serialize alert rules for storing them in a relation databag. + + Args: + rules: alert rules in the official Loki rule file format. + encoding: one of `SUPPORTED_ALERT_RULES_ENCODINGS`. Anything else is treated + as `JSON_ENCODING`, because plain JSON is readable by every version of + this library. + + Returns: + The serialized alert rules. + """ + # Sort keys to prevent unnecessary relation-changed churn from key reordering. + serialized = json.dumps(rules, sort_keys=True) + if encoding == LZMA_ENCODING: + return LZMABase64.compress(serialized) + return serialized + + +def _decode_alert_rules(raw: str) -> OfficialRuleFileFormat: + """Deserialize alert rules read from a relation databag. + + Both plain JSON and LZMA-compressed, base64-encoded JSON are accepted, regardless + of the encodings this library advertises, so that a provider can always read the + rules of a consumer running any version of this library. + + Args: + raw: the raw databag value. + + Returns: + The alert rules in the official Loki rule file format. + + Raises: + ValueError: if `raw` is neither valid JSON nor a valid compressed payload, or if it + decodes to something other than a JSON object. + """ + if not raw: + return cast(OfficialRuleFileFormat, {}) + + try: + decoded = json.loads(raw) + except json.JSONDecodeError: + # Not JSON, so this must be a compressed payload. + decoded = raw + + if isinstance(decoded, str): + # A compressed payload, either bare or (as pydantic based libraries write it) + # JSON-encoded. + try: + decoded = json.loads(LZMABase64.decompress(decoded)) + except (ValueError, lzma.LZMAError) as e: + raise ValueError(f"Could not decompress alert rules: {e}") from e + + if not isinstance(decoded, dict): + raise ValueError(f"Alert rules must be a JSON object, not {type(decoded).__name__}") + + return cast(OfficialRuleFileFormat, decoded) + + +def _best_alert_rules_encoding(remote_app_databag: Optional[Mapping[str, str]]) -> str: + """Return the best alert rules encoding the remote application is able to read. + + Providers advertise the encodings they support in their application databag. + Providers running an older version of this library advertise nothing, in which + case plain JSON is used for backwards compatibility. + + Args: + remote_app_databag: the remote application databag, or None if it is not + readable yet (e.g. the relation is still being set up). + + Returns: + One of `SUPPORTED_ALERT_RULES_ENCODINGS`. + """ + raw = remote_app_databag.get(ALERT_RULES_ENCODINGS_KEY, "[]") if remote_app_databag else "[]" + + try: + advertised = json.loads(raw) + if not isinstance(advertised, list): + raise TypeError("expected a list, got {}".format(type(advertised).__name__)) + except (json.JSONDecodeError, TypeError) as e: + logger.warning( + "Ignoring malformed '%s' (%s); assuming the remote end is only able to read " + "uncompressed alert rules.", + ALERT_RULES_ENCODINGS_KEY, + e, + ) + return JSON_ENCODING + + for encoding in SUPPORTED_ALERT_RULES_ENCODINGS: + if encoding in advertised: + return encoding + + # Either nothing was advertised (an older provider), or only encodings this library + # does not know about. Plain JSON is the encoding every version can read. + return JSON_ENCODING + + +class LokiPushApiError(Exception): + """Base class for errors raised by this module.""" + + +class RelationNotFoundError(LokiPushApiError): + """Raised if there is no relation with the given name.""" + + def __init__(self, relation_name: str): + self.relation_name = relation_name + self.message = "No relation named '{}' found".format(relation_name) + + super().__init__(self.message) + + +class RelationInterfaceMismatchError(LokiPushApiError): + """Raised if the relation with the given name has a different interface.""" + + def __init__( + self, + relation_name: str, + expected_relation_interface: str, + actual_relation_interface: str, + ): + self.relation_name = relation_name + self.expected_relation_interface = expected_relation_interface + self.actual_relation_interface = actual_relation_interface + self.message = ( + "The '{}' relation has '{}' as interface rather than the expected '{}'".format( + relation_name, actual_relation_interface, expected_relation_interface + ) + ) + super().__init__(self.message) + + +class RelationRoleMismatchError(LokiPushApiError): + """Raised if the relation with the given name has a different direction.""" + + def __init__( + self, + relation_name: str, + expected_relation_role: RelationRole, + actual_relation_role: RelationRole, + ): + self.relation_name = relation_name + self.expected_relation_interface = expected_relation_role + self.actual_relation_role = actual_relation_role + self.message = "The '{}' relation has role '{}' rather than the expected '{}'".format( + relation_name, repr(actual_relation_role), repr(expected_relation_role) + ) + super().__init__(self.message) + + +def _validate_relation_by_interface_and_direction( + charm: CharmBase, + relation_name: str, + expected_relation_interface: str, + expected_relation_role: RelationRole, +): + """Verifies that a relation has the necessary characteristics. + + Verifies that the `relation_name` provided: (1) exists in metadata.yaml, + (2) declares as interface the interface name passed as `relation_interface` + and (3) has the right "direction", i.e., it is a relation that `charm` + provides or requires. + + Args: + charm: a `CharmBase` object to scan for the matching relation. + relation_name: the name of the relation to be verified. + expected_relation_interface: the interface name to be matched by the + relation named `relation_name`. + expected_relation_role: whether the `relation_name` must be either + provided or required by `charm`. + + Raises: + RelationNotFoundError: If there is no relation in the charm's metadata.yaml + with the same name as provided via `relation_name` argument. + RelationInterfaceMismatchError: The relation with the same name as provided + via `relation_name` argument does not have the same relation interface + as specified via the `expected_relation_interface` argument. + RelationRoleMismatchError: If the relation with the same name as provided + via `relation_name` argument does not have the same role as specified + via the `expected_relation_role` argument. + """ + if relation_name not in charm.meta.relations: + raise RelationNotFoundError(relation_name) + + relation = charm.meta.relations[relation_name] + + actual_relation_interface = relation.interface_name + if actual_relation_interface != expected_relation_interface: + raise RelationInterfaceMismatchError( + relation_name, + expected_relation_interface, + actual_relation_interface, # pyright: ignore + ) + + if expected_relation_role == RelationRole.provides: + if relation_name not in charm.meta.provides: + raise RelationRoleMismatchError( + relation_name, RelationRole.provides, RelationRole.requires + ) + elif expected_relation_role == RelationRole.requires: + if relation_name not in charm.meta.requires: + raise RelationRoleMismatchError( + relation_name, RelationRole.requires, RelationRole.provides + ) + else: + raise Exception("Unexpected RelationDirection: {}".format(expected_relation_role)) + + +class InvalidAlertRulePathError(Exception): + """Raised if the alert rules folder cannot be found or is otherwise invalid.""" + + def __init__( + self, + alert_rules_absolute_path: Path, + message: str, + ): + self.alert_rules_absolute_path = alert_rules_absolute_path + self.message = message + + super().__init__(self.message) + + +def _resolve_dir_against_charm_path(charm: CharmBase, *path_elements: str) -> str: + """Resolve the provided path items against the directory of the main file. + + Look up the directory of the `main.py` file being executed. This is normally + going to be the charm.py file of the charm including this library. Then, resolve + the provided path elements and, if the result path exists and is a directory, + return its absolute path; otherwise, raise en exception. + + Raises: + InvalidAlertRulePathError, if the path does not exist or is not a directory. + """ + charm_dir = Path(str(charm.charm_dir)) + if not charm_dir.exists() or not charm_dir.is_dir(): + # Operator Framework does not currently expose a robust + # way to determine the top level charm source directory + # that is consistent across deployed charms and unit tests + # Hence for unit tests the current working directory is used + # TODO: updated this logic when the following ticket is resolved + # https://github.com/canonical/operator/issues/643 + charm_dir = Path(os.getcwd()) + + alerts_dir_path = charm_dir.absolute().joinpath(*path_elements) + + if not alerts_dir_path.exists(): + raise InvalidAlertRulePathError(alerts_dir_path, "directory does not exist") + if not alerts_dir_path.is_dir(): + raise InvalidAlertRulePathError(alerts_dir_path, "is not a directory") + + return str(alerts_dir_path) + + +class NoRelationWithInterfaceFoundError(Exception): + """No relations with the given interface are found in the charm meta.""" + + def __init__(self, charm: CharmBase, relation_interface: Optional[str] = None): + self.charm = charm + self.relation_interface = relation_interface + self.message = ( + "No relations with interface '{}' found in the meta of the '{}' charm".format( + relation_interface, charm.meta.name + ) + ) + + super().__init__(self.message) + + +class MultipleRelationsWithInterfaceFoundError(Exception): + """Multiple relations with the given interface are found in the charm meta.""" + + def __init__(self, charm: CharmBase, relation_interface: str, relations: list): + self.charm = charm + self.relation_interface = relation_interface + self.relations = relations + self.message = ( + "Multiple relations with interface '{}' found in the meta of the '{}' charm.".format( + relation_interface, charm.meta.name + ) + ) + super().__init__(self.message) + + +class LokiPushApiEndpointDeparted(EventBase): + """Event emitted when Loki departed.""" + + +class LokiPushApiEndpointJoined(EventBase): + """Event emitted when Loki joined.""" + + +class LokiPushApiAlertRulesChanged(EventBase): + """Event emitted if there is a change in the alert rules.""" + + def __init__(self, handle, relation, relation_id, app=None, unit=None): + """Pretend we are almost like a RelationEvent. + + Fields to serialize: + { + "relation_name": , + "relation_id": , + "app_name": , + "unit_name": + } + + In this way, we can transparently use `RelationEvent.snapshot()` to pass + it back if we need to log it. + """ + super().__init__(handle) + self.relation = relation + self.relation_id = relation_id + self.app = app + self.unit = unit + + def snapshot(self) -> Dict: + """Save event information.""" + if not self.relation: + return {} + snapshot = {"relation_name": self.relation.name, "relation_id": self.relation.id} + if self.app: + snapshot["app_name"] = self.app.name + if self.unit: + snapshot["unit_name"] = self.unit.name + return snapshot + + def restore(self, snapshot: dict): + """Restore event information.""" + self.relation = self.framework.model.get_relation( + snapshot["relation_name"], snapshot["relation_id"] + ) + app_name = snapshot.get("app_name") + if app_name: + self.app = self.framework.model.get_app(app_name) + else: + self.app = None + unit_name = snapshot.get("unit_name") + if unit_name: + self.unit = self.framework.model.get_unit(unit_name) + else: + self.unit = None + + +class InvalidAlertRuleEvent(EventBase): + """Event emitted when alert rule files are not parsable. + + Enables us to set a clear status on the provider. + """ + + def __init__(self, handle, errors: str = "", valid: bool = False): + super().__init__(handle) + self.errors = errors + self.valid = valid + + def snapshot(self) -> Dict: + """Save alert rule information.""" + return { + "valid": self.valid, + "errors": self.errors, + } + + def restore(self, snapshot): + """Restore alert rule information.""" + self.valid = snapshot["valid"] + self.errors = snapshot["errors"] + + +class LokiPushApiEvents(ObjectEvents): + """Event descriptor for events raised by `LokiPushApiProvider`.""" + + loki_push_api_endpoint_departed = EventSource(LokiPushApiEndpointDeparted) + loki_push_api_endpoint_joined = EventSource(LokiPushApiEndpointJoined) + loki_push_api_alert_rules_changed = EventSource(LokiPushApiAlertRulesChanged) + alert_rule_status_changed = EventSource(InvalidAlertRuleEvent) + + +class LokiPushApiProvider(Object): + """A LokiPushApiProvider class.""" + + on = LokiPushApiEvents() # pyright: ignore + + def __init__( + self, + charm, + relation_name: str = DEFAULT_RELATION_NAME, + *, + port: Union[str, int] = 3100, + scheme: str = "http", + address: str = "", + path: str = "loki/api/v1/push", + ): + """A Loki service provider. + + Args: + charm: a `CharmBase` instance that manages this + instance of the Loki service. + relation_name: an optional string name of the relation between `charm` + and the Loki charmed service. The default is "logging". + It is strongly advised not to change the default, so that people + deploying your charm will have a consistent experience with all + other charms that consume metrics endpoints. + port: an optional port of the Loki service (default is "3100"). + scheme: an optional scheme of the Loki API URL (default is "http"). + address: DEPRECATED. This argument is ignored and will be removed in v2. + It is kept for backward compatibility. + Use `update_endpoint()` instead. + path: an optional path of the Loki API URL (default is "loki/api/v1/push") + + Raises: + RelationNotFoundError: If there is no relation in the charm's metadata.yaml + with the same name as provided via `relation_name` argument. + RelationInterfaceMismatchError: The relation with the same name as provided + via `relation_name` argument does not have the `loki_push_api` relation + interface. + RelationRoleMismatchError: If the relation with the same name as provided + via `relation_name` argument does not have the `RelationRole.requires` + role. + """ + _validate_relation_by_interface_and_direction( + charm, relation_name, RELATION_INTERFACE_NAME, RelationRole.provides + ) + + if address != "": + warnings.warn( + "The 'address' parameter is deprecated and will be removed in v2. " + "Use 'update_endpoint()' instead.", + DeprecationWarning, + stacklevel=2, + ) + + super().__init__(charm, relation_name) + self._charm = charm + self._relation_name = relation_name + self._tool = CosTool("logql") + self.port = int(port) + self.scheme = scheme + self.path = path + self._custom_url = None + + events = self._charm.on[relation_name] + self.framework.observe(self._charm.on.upgrade_charm, self._on_lifecycle_event) + self.framework.observe(events.relation_joined, self._on_logging_relation_joined) + self.framework.observe(events.relation_changed, self._on_logging_relation_changed) + self.framework.observe(events.relation_departed, self._on_logging_relation_departed) + self.framework.observe(events.relation_broken, self._on_logging_relation_broken) + # Consumers only compress their alert rules if we advertise that we can read them, + # so make sure the advertisement is (re)published after a leadership change, when no + # relation event may fire. + self.framework.observe( + self._charm.on.leader_elected, + self._publish_encodings_to_all_relation_databags, + ) + + def _on_lifecycle_event(self, _): + # Upgrade event or other charm-level event + should_update = False + for relation in self._charm.model.relations[self._relation_name]: + # Don't accidentally flip a True result back. + should_update = should_update or self._process_logging_relation_changed(relation) + if should_update: + # We don't have a RelationEvent, so build it up by hand + first_rel = self._charm.model.relations[self._relation_name][0] + self.on.loki_push_api_alert_rules_changed.emit( + relation=first_rel, + relation_id=first_rel.id, + ) + + def _on_logging_relation_joined(self, event: RelationJoinedEvent): + """Set basic data on relation joins. + + Set the promtail binary URL location, which will not change, and anything + else which may be required, but is static.. + + Args: + event: a `CharmEvent` in response to which the consumer + charm must set its relation data. + """ + if self._charm.unit.is_leader(): + event.relation.data[self._charm.app].update(self._promtail_binary_url) + logger.debug("Saved promtail binary url: %s", self._promtail_binary_url) + self._publish_alert_rules_encodings(event.relation) + + def _on_logging_relation_changed(self, event: HookEvent): + """Handle changes in related consumers. + + Anytime there are changes in the relation between Loki + and its consumers charms. + + Args: + event: a `CharmEvent` in response to which the consumer + charm must update its relation data. + """ + should_update = self._process_logging_relation_changed(event.relation) # pyright: ignore + if should_update: + self.on.loki_push_api_alert_rules_changed.emit( + relation=event.relation, # pyright: ignore + relation_id=event.relation.id, # pyright: ignore + app=self._charm.app, + unit=self._charm.unit, + ) + + def _on_logging_relation_broken(self, event: RelationBrokenEvent): + """Removes alert rules files when consumer charms left the relation with Loki. + + Args: + event: a `CharmEvent` in response to which the Loki + charm must update its relation data. + """ + self.on.loki_push_api_alert_rules_changed.emit( + relation=event.relation, + relation_id=event.relation.id, + app=self._charm.app, + unit=self._charm.unit, + ) + + def _on_logging_relation_departed(self, event: RelationDepartedEvent): + """Removes alert rules files when consumer charms left the relation with Loki. + + Args: + event: a `CharmEvent` in response to which the Loki + charm must update its relation data. + """ + self.on.loki_push_api_alert_rules_changed.emit( + relation=event.relation, + relation_id=event.relation.id, + app=self._charm.app, + unit=self._charm.unit, + ) + + def _should_update_alert_rules(self, relation) -> bool: + """Determine whether alert rules should be regenerated. + + If there are alert rules in the relation data bag, tell the charm + whether to regenerate them based on the boolean returned here. + """ + if relation.data.get(relation.app).get("alert_rules", None) is not None: + return True + return False + + def _process_logging_relation_changed(self, relation: Relation) -> bool: + """Handle changes in related consumers. + + Anytime there are changes in relations between Loki + and its consumers charms, Loki set the `loki_push_api` + into the relation data. Set the endpoint building + appropriately, and if there are alert rules present in + the relation, let the caller know. + Besides Loki generates alert rules files based what + consumer charms forwards, + + Args: + relation: the `Relation` instance to update. + + Returns: + A boolean indicating whether an event should be emitted, so we + only emit one on lifecycle events + """ + relation.data[self._charm.unit]["public_address"] = socket.getfqdn() or "" + self.update_endpoint(relation=relation) + self._publish_alert_rules_encodings(relation) + + # Ensure promtail binary URL is set in app data. This is normally done on + # relation_joined, but charms using the reconcile pattern may miss that event + # if the workload container is not yet ready when the relation is first established. + if self._charm.unit.is_leader(): + if not relation.data[self._charm.app].get("promtail_binary_zip_url"): + relation.data[self._charm.app].update(self._promtail_binary_url) + + return self._should_update_alert_rules(relation) + + def _publish_encodings_to_all_relation_databags(self, _: Optional[HookEvent]) -> None: + for relation in self._charm.model.relations[self._relation_name]: + self._publish_alert_rules_encodings(relation) + + def _publish_alert_rules_encodings(self, relation: Relation) -> None: + """Advertise the alert rules encodings this library is able to read. + + Consumers use this to decide whether they may compress their alert rules: a + consumer related to a provider that does not advertise anything keeps writing + plain JSON, which every version of this library can read. + + Args: + relation: The relation whose data to update. + """ + if not self._charm.unit.is_leader(): + # Only the leader unit can write to app data. + return + + relation.data[self._charm.app][ALERT_RULES_ENCODINGS_KEY] = json.dumps( + list(SUPPORTED_ALERT_RULES_ENCODINGS) + ) + + @property + def _promtail_binary_url(self) -> dict: + """URL from which Promtail binary can be downloaded.""" + # construct promtail binary url paths from parts + promtail_binaries = {} + for arch, info in PROMTAIL_BINARIES.items(): + info["url"] = "{}/promtail-{}/{}.gz".format( + PROMTAIL_BASE_URL, PROMTAIL_VERSION, info["filename"] + ) + promtail_binaries[arch] = info + + return {"promtail_binary_zip_url": json.dumps(promtail_binaries)} + + def update_endpoint(self, url: str = "", relation: Optional[Relation] = None) -> None: + """Triggers programmatically the update of endpoint in unit relation data. + + This method should be used when the charm relying on this library needs + to update the relation data in response to something occurring outside + the `logging` relation lifecycle, e.g., in case of a + host address change because the charmed operator becomes connected to an + Ingress after the `logging` relation is established. + + To make this library reconciler-friendly, the endpoint URL was made sticky i.e., once the + endpoint is updated with a custom URL, using the public method, it cannot be unset. Users + of this method should set the "url" arg to an internal URL if the charms ingress is no + longer available. + + Args: + url: An optional url value to update relation data. + relation: An optional instance of `class:ops.model.Relation` to update. + """ + # if no relation is specified update all of them + if not relation: + if not self._charm.model.relations.get(self._relation_name): + return + + relations_list = self._charm.model.relations.get(self._relation_name) + else: + relations_list = [relation] + + if url: + self._custom_url = url + + endpoint = self._endpoint(self._custom_url or self._url) + + for relation in relations_list: + relation.data[self._charm.unit].update({"endpoint": json.dumps(endpoint)}) + self._publish_alert_rules_encodings(relation) + + logger.debug("Saved endpoint in unit relation data") + + @property + def _url(self) -> str: + """Get local Loki Push API url. + + Return url to loki, including port number, but without the endpoint subpath. + """ + return f"{self.scheme}://{socket.getfqdn()}:{self.port}" + + def _endpoint(self, url) -> dict: + """Get Loki push API endpoint for a given url. + + Args: + url: A loki unit URL. + + Returns: str + """ + endpoint = "/loki/api/v1/push" + return {"url": url.rstrip("/") + endpoint} + + @property + def alerts(self) -> dict: # noqa: C901 + """Fetch alerts for all relations. + + A Loki alert rules file consists of a list of "groups". Each + group consists of a list of alerts (`rules`) that are sequentially + executed. This method returns all the alert rules provided by each + related metrics provider charm. These rules may be used to generate a + separate alert rules file for each relation since the returned list + of alert groups are indexed by relation ID. Also for each relation ID + associated scrape metadata such as Juju model, UUID and application + name are provided so a unique name may be generated for the rules + file. For each relation the structure of data returned is a dictionary + with four keys + + - groups + - model + - model_uuid + - application + + The value of the `groups` key is such that it may be used to generate + a Loki alert rules file directly using `yaml.dump` but the + `groups` key itself must be included as this is required by Loki, + for example as in `yaml.dump({"groups": alerts["groups"]})`. + + Currently only accepts a list of rules and these + rules are all placed into a single group, even though Loki itself + allows for multiple groups within a single alert rules file. + + Returns: + a dictionary of alert rule groups and associated scrape + metadata indexed by relation ID. + """ + alerts = {} # type: Dict[str, dict] # mapping b/w juju identifiers and alert rule files + unreadable: Dict[int, str] = {} + for relation in self._charm.model.relations[self._relation_name]: + if not relation.units or not relation.app: + continue + + try: + alert_rules = _decode_alert_rules( + relation.data[relation.app].get(ALERT_RULES_KEY, "{}") + ) + except Exception as e: + # Never let unreadable remote data break the provider: a consumer could + # be writing rules in a format this version of the library predates. + unreadable[relation.id] = str(e) + continue + + if not alert_rules: + continue + + alert_rules = self._inject_alert_expr_labels(cast(Dict[str, Any], alert_rules)) + + identifier, topology = self._get_identifier_by_alert_rules(alert_rules) + if not topology: + try: + metadata = json.loads(relation.data[relation.app]["metadata"]) + identifier = JujuTopology.from_dict(metadata).identifier + + except KeyError as e: + logger.debug( + "Relation %s has no 'metadata': %s", + relation.id, + e, + ) + + if not identifier: + logger.error( + "Alert rules were found but no usable group or identifier was present." + ) + continue + + # Topology labels are already injected by _inject_alert_expr_labels using + # alert_expression_dict, which intentionally excludes juju_charm and juju_unit. + # Don't call apply_label_matchers here as it would re-inject juju_charm. + alerts[identifier] = alert_rules + + _, errmsg = self._tool.validate_alert_rules(cast(OfficialRuleFileFormat, alert_rules)) + if errmsg: + logger.error(f"Invalid alert rule file: {errmsg}") + if alerts[identifier]: + del alerts[identifier] + if self._charm.unit.is_leader(): + relation.data[self._charm.app]["event"] = json.dumps({"errors": errmsg}) + continue + if self._charm.unit.is_leader(): + event_data = json.loads(relation.data[self._charm.app].get("event", "{}")) + event_data.pop("errors", None) + relation.data[self._charm.app]["event"] = json.dumps(event_data) + + alerts[identifier] = alert_rules + + if unreadable: + logger.error( + "Could not read the alert rules published over relation(s): %s", + "; ".join("{} ({})".format(rel_id, err) for rel_id, err in unreadable.items()), + ) + + return alerts + + def has_invalid_alert_rules(self) -> bool: + """Check whether any relation reported invalid alert rules. + + Validation errors, written to relation app data by the :attr:`alerts` + property, are read back to determine whether the relation currently + carries invalid alert rules. Non-leader units never write the app data + that holds these errors, so they always report no errors. + + Returns: + True if any related consumer reported alert rule validation + errors, False otherwise. + """ + if not self._charm.unit.is_leader(): + return False + + for relation in self._charm.model.relations.get(self._relation_name, []): + app_data = relation.data.get(self._charm.app) + if not app_data: + continue + + event_raw = app_data.get("event", "{}") + try: + event_data = json.loads(event_raw) + except (json.JSONDecodeError, TypeError): + continue + + if error_msg := event_data.get("errors"): + logger.error( + "Alert rule validation error on relation %s: %s", + relation.id, + error_msg, + ) + return True + + return False + + def _get_identifier_by_alert_rules( + self, rules: dict + ) -> Tuple[Union[str, None], Union[JujuTopology, None]]: + """Determine an appropriate dict key for alert rules. + + The key is used as the filename when writing alerts to disk, so the structure + and uniqueness is important. + + Args: + rules: a dict of alert rules + Returns: + A tuple containing an identifier, if found, and a JujuTopology, if it could + be constructed. + """ + if "groups" not in rules: + logger.debug("No alert groups were found in relation data") + return None, None + + # Construct an ID based on what's in the alert rules if they have labels + for group in rules["groups"]: + try: + labels = group["rules"][0]["labels"] + topology = JujuTopology( + # Don't try to safely get required constructor fields. There's already + # a handler for KeyErrors + model_uuid=labels["juju_model_uuid"], + model=labels["juju_model"], + application=labels["juju_application"], + unit=labels.get("juju_unit", ""), + charm_name=labels.get("juju_charm", ""), + ) + return topology.identifier, topology + except KeyError: + logger.debug("Alert rules were found but no usable labels were present") + continue + + logger.warning( + "No labeled alert rules were found, and no 'scrape_metadata' " + "was available. Using the alert group name as filename." + ) + try: + for group in rules["groups"]: + return group["name"], None + except KeyError: + logger.debug("No group name was found to use as identifier") + + return None, None + + def _inject_alert_expr_labels(self, rules: Dict[str, Any]) -> Dict[str, Any]: + """Iterate through alert rules and inject topology into expressions. + + Args: + rules: a dict of alert rules + """ + if "groups" not in rules: + return rules + + modified_groups = [] + for group in rules["groups"]: + # Copy off rules, so we don't modify an object we're iterating over + rules_copy = group["rules"] + for idx, rule in enumerate(rules_copy): + labels = rule.get("labels") + + if labels: + try: + topology = JujuTopology( + # Don't try to safely get required constructor fields. There's already + # a handler for KeyErrors + model_uuid=labels["juju_model_uuid"], + model=labels["juju_model"], + application=labels["juju_application"], + unit=labels.get("juju_unit", ""), + charm_name=labels.get("juju_charm", ""), + ) + + # Inject topology and put it back in the list. + # Use alert_expression_dict (excludes juju_charm) instead of + # label_matcher_dict because subordinate charms (e.g. otelcol) + # label logs with their own charm name, not the principal's. + rule["expr"] = self._tool.inject_label_matchers( + re.sub(r"%%juju_topology%%,?", "", rule["expr"]), + topology.alert_expression_dict, + ) + except KeyError: + # Some required JujuTopology key is missing. Just move on. + pass + + group["rules"][idx] = rule + + modified_groups.append(group) + + rules["groups"] = modified_groups + return rules + + +class ConsumerBase(Object): + """Consumer's base class.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str = DEFAULT_RELATION_NAME, + alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, + recursive: bool = False, + skip_alert_topology_labeling: bool = False, + *, + forward_alert_rules: bool = True, + extra_alert_labels: Dict = {}, + ): + super().__init__(charm, relation_name) + self._charm = charm + self._relation_name = relation_name + self._forward_alert_rules = forward_alert_rules + self._extra_alert_labels = extra_alert_labels + self.topology = JujuTopology.from_charm(charm) + + try: + alert_rules_path = _resolve_dir_against_charm_path(charm, alert_rules_path) + except InvalidAlertRulePathError as e: + logger.debug( + "Invalid Loki alert rules folder at %s: %s", + e.alert_rules_absolute_path, + e.message, + ) + self._alert_rules_path = alert_rules_path + self._skip_alert_topology_labeling = skip_alert_topology_labeling + + self._recursive = recursive + + @staticmethod + def _inject_extra_labels_to_alert_rules(rules: Dict, extra_alert_labels: Dict) -> Dict: + """Return a copy of the rules dict with extra labels injected.""" + result = copy.deepcopy(rules) + for group in result.get("groups", []): + for rule in group.get("rules", []): + rule.setdefault("labels", {}).update(extra_alert_labels) + return result + + def _handle_alert_rules(self, relation): + if not self._charm.unit.is_leader(): + return + + alert_rules = ( + AlertRules(query_type="logql") + if self._skip_alert_topology_labeling + else AlertRules(query_type="logql", topology=self.topology) + ) + if self._forward_alert_rules: + alert_rules.add_path(self._alert_rules_path, recursive=self._recursive) + alert_rules_as_dict = alert_rules.as_dict() + + if self._extra_alert_labels: + alert_rules_as_dict = ConsumerBase._inject_extra_labels_to_alert_rules( + alert_rules_as_dict, self._extra_alert_labels + ) + + relation.data[self._charm.app]["metadata"] = json.dumps(self.topology.as_dict()) + remote_app_databag = relation.data.get(relation.app) if relation.app else None + relation.data[self._charm.app][ALERT_RULES_KEY] = _encode_alert_rules( + alert_rules_as_dict, _best_alert_rules_encoding(remote_app_databag) + ) + + @property + def loki_endpoints(self) -> List[dict]: + """Fetch Loki Push API endpoints sent from LokiPushApiProvider through relation data. + + Returns: + A list of unique dictionaries with Loki Push API endpoints, for instance: + [ + {"url": "http://loki1:3100/loki/api/v1/push"}, + {"url": "http://loki2:3100/loki/api/v1/push"}, + ] + """ + endpoints = [] + seen_urls = set() + + for relation in self._charm.model.relations[self._relation_name]: + # Sort the units so the endpoints list order is stable across runs, + # otherwise the generated promtail config flaps. + for unit in sorted(relation.units, key=lambda u: u.name): + if unit.app == self._charm.app: + continue + + if not (endpoint := relation.data[unit].get("endpoint")): + continue + + deserialized_endpoint = json.loads(endpoint) + url = deserialized_endpoint.get("url") + + # Deduplicate by URL. + # With loki-k8s we have ingress-per-unit, so in that case + # we do want to collect the URLs of all the units. + # With loki-coordinator-k8s, even when the coordinator + # is scaled, we want to advertise only one URL. + # Without deduplication, we'd end up with the same + # tls config section in the promtail config file, in which + # case promtail immediately exits with the following error: + # [promtail] level=error ts= msg="error creating promtail" error="failed to create client manager: duplicate client configs are not allowed, found duplicate for name: " + + if not url or url in seen_urls: + continue + + seen_urls.add(url) + endpoints.append(deserialized_endpoint) + + return endpoints + + +class LokiPushApiConsumer(ConsumerBase): + """Loki Consumer class.""" + + on = LokiPushApiEvents() # pyright: ignore + + def __init__( + self, + charm: CharmBase, + relation_name: str = DEFAULT_RELATION_NAME, + alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, + recursive: bool = True, + skip_alert_topology_labeling: bool = False, + *, + refresh_event: Optional[Union[BoundEvent, List[BoundEvent]]] = None, + forward_alert_rules: bool = True, + extra_alert_labels: Dict = {}, + ): + """Construct a Loki charm client. + + The `LokiPushApiConsumer` object provides configurations to a Loki client charm, such as + the Loki API endpoint to push logs. It is intended for workloads that can speak + loki_push_api (https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki), such + as grafana-agent. + (If you need to forward workload stdout logs, then use LogForwarder; if you need to forward + log files, then use LogProxyConsumer.) + + `LokiPushApiConsumer` can be instantiated as follows: + + self._loki_consumer = LokiPushApiConsumer(self) + + Args: + charm: a `CharmBase` object that manages this `LokiPushApiConsumer` object. + Typically, this is `self` in the instantiating class. + relation_name: the string name of the relation interface to look up. + If `charm` has exactly one relation with this interface, the relation's + name is returned. If none or multiple relations with the provided interface + are found, this method will raise either a NoRelationWithInterfaceFoundError or + MultipleRelationsWithInterfaceFoundError exception, respectively. + alert_rules_path: a string indicating a path where alert rules can be found + recursive: Whether to scan for rule files recursively. + skip_alert_topology_labeling: whether to skip the alert topology labeling. + forward_alert_rules: a boolean flag to toggle forwarding of charmed alert rules. + extra_alert_labels: Dict of extra labels to inject alert rules with. + refresh_event: an optional bound event or list of bound events which + will be observed to re-set scrape job data (IP address and others) + + Raises: + RelationNotFoundError: If there is no relation in the charm's metadata.yaml + with the same name as provided via `relation_name` argument. + RelationInterfaceMismatchError: The relation with the same name as provided + via `relation_name` argument does not have the `loki_push_api` relation + interface. + RelationRoleMismatchError: If the relation with the same name as provided + via `relation_name` argument does not have the `RelationRole.provides` + role. + + Emits: + loki_push_api_endpoint_joined: This event is emitted when the relation between the + Charmed Operator that instantiates `LokiPushApiProvider` (Loki charm for instance) + and the Charmed Operator that instantiates `LokiPushApiConsumer` is established. + loki_push_api_endpoint_departed: This event is emitted when the relation between the + Charmed Operator that implements `LokiPushApiProvider` (Loki charm for instance) + and the Charmed Operator that implements `LokiPushApiConsumer` is removed. + loki_push_api_alert_rules_error: This event is emitted when an invalid alert rules + file is encountered or if `alert_rules_path` is empty. + """ + _validate_relation_by_interface_and_direction( + charm, relation_name, RELATION_INTERFACE_NAME, RelationRole.requires + ) + super().__init__( + charm, + relation_name, + alert_rules_path, + recursive, + skip_alert_topology_labeling, + forward_alert_rules=forward_alert_rules, + extra_alert_labels=extra_alert_labels, + ) + events = self._charm.on[relation_name] + self.framework.observe(self._charm.on.upgrade_charm, self._on_lifecycle_event) + self.framework.observe(self._charm.on.config_changed, self._on_lifecycle_event) + self.framework.observe(events.relation_joined, self._on_logging_relation_joined) + self.framework.observe(events.relation_changed, self._on_logging_relation_changed) + self.framework.observe(events.relation_departed, self._on_logging_relation_departed) + + if refresh_event: + if not isinstance(refresh_event, list): + refresh_event = [refresh_event] + for ev in refresh_event: + self.framework.observe(ev, self._on_lifecycle_event) + + def _on_lifecycle_event(self, _: HookEvent): + """Update require relation data on charm upgrades and other lifecycle events. + + Args: + event: a `CharmEvent` in response to which the consumer + charm must update its relation data. + """ + # Upgrade event or other charm-level event + self._reinitialize_alert_rules() + self.on.loki_push_api_endpoint_joined.emit() + + def _on_logging_relation_joined(self, event: RelationJoinedEvent): + """Handle changes in related consumers. + + Update relation data and emit events when a relation is established. + + Args: + event: a `CharmEvent` in response to which the consumer + charm must update its relation data. + + Emits: + loki_push_api_endpoint_joined: Once the relation is established, this event is emitted. + loki_push_api_alert_rules_error: This event is emitted when an invalid alert rules + file is encountered or if `alert_rules_path` is empty. + """ + # Alert rules will not change over the lifecycle of a charm, and do not need to be + # constantly set on every relation_changed event. Leave them here. + self._handle_alert_rules(event.relation) + self.on.loki_push_api_endpoint_joined.emit() + + def _on_logging_relation_changed(self, event: RelationEvent): + """Handle changes in related consumers. + + Anytime there are changes in the relation between Loki + and its consumers charms. + + Args: + event: a `CharmEvent` in response to which the consumer + charm must update its relation data. + + Emits: + loki_push_api_endpoint_joined: Once the relation is established, this event is emitted. + loki_push_api_alert_rules_error: This event is emitted when an invalid alert rules + file is encountered or if `alert_rules_path` is empty. + """ + # The provider advertises the alert rules encodings it supports over relation data, + # which may only become known after relation_joined; (re)send alert rules here so the + # negotiated encoding is picked up. + self._handle_alert_rules(event.relation) # pyright: ignore + + if self._charm.unit.is_leader(): + ev = json.loads(event.relation.data[event.app].get("event", "{}")) + + if ev: + valid = bool(ev.get("valid", True)) + errors = ev.get("errors", "") + + if valid and not errors: + self.on.alert_rule_status_changed.emit(valid=valid) + else: + self.on.alert_rule_status_changed.emit(valid=valid, errors=errors) + + self.on.loki_push_api_endpoint_joined.emit() + + def reload_alerts(self) -> None: + """Reloads alert rules and updates all relations.""" + self._reinitialize_alert_rules() + + def _reinitialize_alert_rules(self): + for relation in self._charm.model.relations[self._relation_name]: + self._handle_alert_rules(relation) + + def _process_logging_relation_changed(self, relation: Relation): + self._handle_alert_rules(relation) + self.on.loki_push_api_endpoint_joined.emit() + + def _on_logging_relation_departed(self, _: RelationEvent): + """Handle departures in related providers. + + Anytime there are departures in relations between the consumer charm and Loki + the consumer charm is informed, through a `LokiPushApiEndpointDeparted` event. + The consumer charm can then choose to update its configuration. + """ + # Provide default to avoid throwing, as in some complicated scenarios with + # upgrades and hook failures we might not have data in the storage + self.on.loki_push_api_endpoint_departed.emit() + + +class ContainerNotFoundError(Exception): + """Raised if the specified container does not exist.""" + + def __init__(self): + msg = "The specified container does not exist." + self.message = msg + + super().__init__(self.message) + + +class PromtailDigestError(EventBase): + """Event emitted when there is an error with Promtail initialization.""" + + def __init__(self, handle, message): + super().__init__(handle) + self.message = message + + def snapshot(self): + """Save message information.""" + return {"message": self.message} + + def restore(self, snapshot): + """Restore message information.""" + self.message = snapshot["message"] + + +class LogProxyEndpointDeparted(EventBase): + """Event emitted when a Log Proxy has departed.""" + + +class LogProxyEndpointJoined(EventBase): + """Event emitted when a Log Proxy joins.""" + + +class LogProxyEvents(ObjectEvents): + """Event descriptor for events raised by `LogProxyConsumer`.""" + + promtail_digest_error = EventSource(PromtailDigestError) + log_proxy_endpoint_departed = EventSource(LogProxyEndpointDeparted) + log_proxy_endpoint_joined = EventSource(LogProxyEndpointJoined) + + +class LogProxyConsumer(ConsumerBase): + """LogProxyConsumer class. + + > Note: This object is deprecated. Consider migrating to LogForwarder with the release of Juju + > 3.6 LTS. + + The `LogProxyConsumer` object provides a method for attaching `promtail` to + a workload in order to generate structured logging data from applications + which traditionally log to syslog or do not have native Loki integration. + The `LogProxyConsumer` can be instantiated as follows: + + self._log_proxy = LogProxyConsumer( + self, + logs_scheme={ + "workload-a": { + "log-files": ["/tmp/worload-a-1.log", "/tmp/worload-a-2.log"], + "syslog-port": 1514, + }, + "workload-b": {"log-files": ["/tmp/worload-b.log"], "syslog-port": 1515}, + }, + relation_name="log-proxy", + ) + + Args: + charm: a `CharmBase` object that manages this `LokiPushApiConsumer` object. + Typically, this is `self` in the instantiating class. + logs_scheme: a dict which maps containers and a list of log files and syslog port. + relation_name: the string name of the relation interface to look up. + If `charm` has exactly one relation with this interface, the relation's + name is returned. If none or multiple relations with the provided interface + are found, this method will raise either a NoRelationWithInterfaceFoundError or + MultipleRelationsWithInterfaceFoundError exception, respectively. + containers_syslog_port: a dict which maps (and enable) containers and syslog port. + alert_rules_path: an optional path for the location of alert rules + files. Defaults to "./src/loki_alert_rules", + resolved from the directory hosting the charm entry file. + The alert rules are automatically updated on charm upgrade. + recursive: Whether to scan for rule files recursively. + promtail_resource_name: An optional promtail resource name from metadata + if it has been modified and attached + insecure_skip_verify: skip SSL verification. + + Raises: + RelationNotFoundError: If there is no relation in the charm's metadata.yaml + with the same name as provided via `relation_name` argument. + RelationInterfaceMismatchError: The relation with the same name as provided + via `relation_name` argument does not have the `loki_push_api` relation + interface. + RelationRoleMismatchError: If the relation with the same name as provided + via `relation_name` argument does not have the `RelationRole.provides` + role. + """ + + on = LogProxyEvents() # pyright: ignore + + def __init__( + self, + charm, + *, + logs_scheme=None, + relation_name: str = DEFAULT_LOG_PROXY_RELATION_NAME, + alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, + recursive: bool = False, + promtail_resource_name: Optional[str] = None, + insecure_skip_verify: bool = False, + ): + super().__init__(charm, relation_name, alert_rules_path, recursive) + self._charm = charm + self._logs_scheme = logs_scheme or {} + self._relation_name = relation_name + self.topology = JujuTopology.from_charm(charm) + self._promtail_resource_name = promtail_resource_name or "promtail-bin" + self.insecure_skip_verify = insecure_skip_verify + self._promtails_ports = self._generate_promtails_ports(logs_scheme) + + # architecture used for promtail binary + arch = platform.machine() + if arch in ["x86_64", "amd64"]: + self._arch = "amd64" + elif arch in ["aarch64", "arm64", "armv8b", "armv8l"]: + self._arch = "arm64" + else: + self._arch = arch + + events = self._charm.on[relation_name] + self.framework.observe(events.relation_created, self._on_relation_created) + self.framework.observe(events.relation_changed, self._on_relation_changed) + self.framework.observe(events.relation_departed, self._on_relation_departed) + self._observe_pebble_ready() + + def _observe_pebble_ready(self): + for container in self._containers.keys(): + snake_case_container_name = container.replace("-", "_") + self.framework.observe( + getattr(self._charm.on, f"{snake_case_container_name}_pebble_ready"), + self._on_pebble_ready, + ) + + def _on_pebble_ready(self, event: WorkloadEvent): + """Event handler for `pebble_ready`.""" + if self.model.relations[self._relation_name]: + self._setup_promtail(event.workload) + + def _on_relation_created(self, _: RelationCreatedEvent) -> None: + """Event handler for `relation_created`.""" + for container in self._containers.values(): + if container.can_connect(): + self._setup_promtail(container) + + def _on_relation_changed(self, event: RelationEvent) -> None: + """Event handler for `relation_changed`. + + Args: + event: The event object `RelationChangedEvent`. + """ + self._handle_alert_rules(event.relation) + + if self._charm.unit.is_leader(): + self._handle_alert_rule_status_changed(event) + + for container in self._containers.values(): + if not container.can_connect(): + continue + if self.model.relations[self._relation_name]: + if not self._is_promtail_set_up(container): + self._setup_promtail(container) + continue + + new_config = self._promtail_config(container.name) + if new_config != self._current_config(container): + container.push( + WORKLOAD_CONFIG_PATH, yaml.safe_dump(new_config), make_dirs=True + ) + + # Loki may send endpoints late. Don't necessarily start, there may be + # no clients + if new_config["clients"]: + if self._restart_promtail(container): + self.on.log_proxy_endpoint_joined.emit() + else: + self.on.promtail_digest_error.emit("No promtail client endpoints available!") + + def _handle_alert_rule_status_changed(self, event: RelationEvent) -> None: + """Relay the alert rule validation status reported by the Loki provider.""" + ev = json.loads(event.relation.data[event.app].get("event", "{}")) + + if ev: + valid = bool(ev.get("valid", True)) + errors = ev.get("errors", "") + + if valid and not errors: + self.on.alert_rule_status_changed.emit(valid=valid) + else: + self.on.alert_rule_status_changed.emit(valid=valid, errors=errors) + + def _on_relation_departed(self, _: RelationEvent) -> None: + """Event handler for `relation_departed`. + + Args: + event: The event object `RelationDepartedEvent`. + """ + for container in self._containers.values(): + if not container.can_connect(): + continue + if not self._charm.model.relations[self._relation_name]: + container.stop(WORKLOAD_SERVICE_NAME) + continue + + new_config = self._promtail_config(container.name) + if new_config != self._current_config(container): + container.push(WORKLOAD_CONFIG_PATH, yaml.safe_dump(new_config), make_dirs=True) + + if new_config["clients"]: + self._restart_promtail(container) + else: + container.stop(WORKLOAD_SERVICE_NAME) + self.on.log_proxy_endpoint_departed.emit() + + def _restart_promtail(self, container: Container) -> bool: + """Restart promtail, surfacing a Pebble failure as a digest error. + + Args: + container: the workload container running the promtail service. + + Returns: + True on success, False if the restart failed. + """ + try: + container.restart(WORKLOAD_SERVICE_NAME) + except ChangeError as e: + logger.warning("Failed to restart promtail: %s", e) + self.on.promtail_digest_error.emit(str(e)) + return False + return True + + def _add_pebble_layer(self, workload_binary_path: str, container: Container) -> None: + """Adds Pebble layer that manages Promtail service in Workload container. + + Args: + workload_binary_path: string providing path to promtail binary in workload container. + container: container into which the layer is to be added. + """ + pebble_layer = Layer( + { + "summary": "promtail layer", + "description": "pebble config layer for promtail", + "services": { + WORKLOAD_SERVICE_NAME: { + "override": "replace", + "summary": WORKLOAD_SERVICE_NAME, + "command": f"{workload_binary_path} {self._cli_args}", + "startup": "disabled", + } + }, + } + ) + container.add_layer(container.name, pebble_layer, combine=True) + + def _create_directories(self, container: Container) -> None: + """Creates the directories for Promtail binary and config file.""" + container.make_dir(path=WORKLOAD_BINARY_DIR, make_parents=True) + container.make_dir(path=WORKLOAD_CONFIG_DIR, make_parents=True) + + def _obtain_promtail(self, promtail_info: dict, container: Container) -> None: + """Obtain promtail binary from an attached resource or download it. + + Args: + promtail_info: dictionary containing information about promtail binary + that must be used. The dictionary must have three keys + - "filename": filename of promtail binary + - "zipsha": sha256 sum of zip file of promtail binary + - "binsha": sha256 sum of unpacked promtail binary + container: container into which promtail is to be obtained. + """ + workload_binary_path = os.path.join(WORKLOAD_BINARY_DIR, promtail_info["filename"]) + if self._promtail_attached_as_resource: + self._push_promtail_if_attached(container, workload_binary_path) + return + + if self._promtail_must_be_downloaded(promtail_info): + self._download_and_push_promtail_to_workload(container, promtail_info) + else: + binary_path = os.path.join(BINARY_DIR, promtail_info["filename"]) + self._push_binary_to_workload(container, binary_path, workload_binary_path) + + def _push_binary_to_workload( + self, container: Container, binary_path: str, workload_binary_path: str + ) -> None: + """Push promtail binary into workload container. + + Args: + binary_path: path in charm container from which promtail binary is read. + workload_binary_path: path in workload container to which promtail binary is pushed. + container: container into which promtail is to be uploaded. + """ + with open(binary_path, "rb") as f: + container.push(workload_binary_path, f, permissions=0o755, make_dirs=True) + logger.debug("The promtail binary file has been pushed to the workload container.") + + @property + def _promtail_attached_as_resource(self) -> bool: + """Checks whether Promtail binary is attached to the charm or not. + + Returns: + a boolean representing whether Promtail binary is attached as a resource or not. + """ + try: + self._charm.model.resources.fetch(self._promtail_resource_name) + return True + except ModelError: + return False + except NameError as e: + if "invalid resource name" in str(e): + return False + raise + + def _push_promtail_if_attached(self, container: Container, workload_binary_path: str) -> bool: + """Checks whether Promtail binary is attached to the charm or not. + + Args: + workload_binary_path: string specifying expected path of promtail + in workload container + container: container into which promtail is to be pushed. + + Returns: + a boolean representing whether Promtail binary is attached or not. + """ + logger.info("Promtail binary file has been obtained from an attached resource.") + resource_path = self._charm.model.resources.fetch(self._promtail_resource_name) + self._push_binary_to_workload(container, resource_path, workload_binary_path) + return True + + def _promtail_must_be_downloaded(self, promtail_info: dict) -> bool: + """Checks whether promtail binary must be downloaded or not. + + Args: + promtail_info: dictionary containing information about promtail binary + that must be used. The dictionary must have three keys + - "filename": filename of promtail binary + - "zipsha": sha256 sum of zip file of promtail binary + - "binsha": sha256 sum of unpacked promtail binary + + Returns: + a boolean representing whether Promtail binary must be downloaded or not. + """ + binary_path = os.path.join(BINARY_DIR, promtail_info["filename"]) + if not self._is_promtail_binary_in_charm(binary_path): + return True + + if not self._sha256sums_matches(binary_path, promtail_info["binsha"]): + return True + + logger.debug("Promtail binary file is already in the the charm container.") + return False + + def _sha256sums_matches(self, file_path: str, sha256sum: str) -> bool: + """Checks whether a file's sha256sum matches or not with a specific sha256sum. + + Args: + file_path: A string representing the files' patch. + sha256sum: The sha256sum against which we want to verify. + + Returns: + a boolean representing whether a file's sha256sum matches or not with + a specific sha256sum. + """ + try: + with open(file_path, "rb") as f: + file_bytes = f.read() + result = sha256(file_bytes).hexdigest() + + if result != sha256sum: + msg = "File sha256sum mismatch, expected:'{}' but got '{}'".format( + sha256sum, result + ) + logger.debug(msg) + return False + + return True + except (APIError, FileNotFoundError): + msg = "File: '{}' could not be opened".format(file_path) + logger.error(msg) + return False + + def _is_promtail_binary_in_charm(self, binary_path: str) -> bool: + """Check if Promtail binary is already stored in charm container. + + Args: + binary_path: string path of promtail binary to check + + Returns: + a boolean representing whether Promtail is present or not. + """ + return True if Path(binary_path).is_file() else False + + def _download_and_push_promtail_to_workload( + self, container: Container, promtail_info: dict + ) -> None: + """Downloads a Promtail zip file and pushes the binary to the workload. + + Args: + promtail_info: dictionary containing information about promtail binary + that must be used. The dictionary must have three keys + - "filename": filename of promtail binary + - "zipsha": sha256 sum of zip file of promtail binary + - "binsha": sha256 sum of unpacked promtail binary + container: container into which promtail is to be uploaded. + """ + # Check for Juju proxy variables and fall back to standard ones if not set + # If no Juju proxy variable was set, we set proxies to None to let the ProxyHandler get + # the proxy env variables from the environment + proxies = { + # The ProxyHandler uses only the protocol names as keys + # https://docs.python.org/3/library/urllib.request.html#urllib.request.ProxyHandler + "https": os.environ.get("JUJU_CHARM_HTTPS_PROXY", ""), + "http": os.environ.get("JUJU_CHARM_HTTP_PROXY", ""), + # The ProxyHandler uses `no` for the no_proxy key + # https://github.com/python/cpython/blob/3.12/Lib/urllib/request.py#L2553 + "no": os.environ.get("JUJU_CHARM_NO_PROXY", ""), + } + proxies = {k: v for k, v in proxies.items() if v != ""} or None + + proxy_handler = request.ProxyHandler(proxies) + opener = request.build_opener(proxy_handler) + + with opener.open(promtail_info["url"]) as r: + file_bytes = r.read() + file_path = os.path.join(BINARY_DIR, promtail_info["filename"] + ".gz") + with open(file_path, "wb") as f: + f.write(file_bytes) + logger.info( + "Promtail binary zip file has been downloaded and stored in: %s", + file_path, + ) + + decompressed_file = GzipFile(fileobj=BytesIO(file_bytes)) + binary_path = os.path.join(BINARY_DIR, promtail_info["filename"]) + with open(binary_path, "wb") as outfile: + outfile.write(decompressed_file.read()) + logger.debug("Promtail binary file has been downloaded.") + + workload_binary_path = os.path.join(WORKLOAD_BINARY_DIR, promtail_info["filename"]) + self._push_binary_to_workload(container, binary_path, workload_binary_path) + + @property + def _cli_args(self) -> str: + """Return the cli arguments to pass to promtail. + + Returns: + The arguments as a string + """ + return "-config.file={}".format(WORKLOAD_CONFIG_PATH) + + def _current_config(self, container) -> dict: + """Property that returns the current Promtail configuration. + + Returns: + A dict containing Promtail configuration. + """ + if not container.can_connect(): + logger.debug("Could not connect to promtail container!") + return {} + try: + raw_current = container.pull(WORKLOAD_CONFIG_PATH).read() + return yaml.safe_load(raw_current) + except (ProtocolError, PathError) as e: + logger.warning( + "Could not check the current promtail configuration due to " + "a failure in retrieving the file: %s", + e, + ) + return {} + + def _promtail_config(self, container_name: str) -> dict: + """Generates the config file for Promtail. + + Reference: https://grafana.com/docs/loki/latest/send-data/promtail/configuration + """ + config = {"clients": self._clients_list()} + if self.insecure_skip_verify: + for client in config["clients"]: + client["tls_config"] = {"insecure_skip_verify": True} + + config.update(self._server_config(container_name)) + config.update(self._positions) + config.update(self._scrape_configs(container_name)) + return config + + def _clients_list(self) -> list: + """Generates a list of clients for use in the promtail config. + + Returns: + A list of endpoints + """ + return self.loki_endpoints + + def _server_config(self, container_name: str) -> dict: + """Generates the server section of the Promtail config file. + + Returns: + A dict representing the `server` section. + """ + return { + "server": { + "http_listen_port": self._promtails_ports[container_name]["http_listen_port"], + "grpc_listen_port": self._promtails_ports[container_name]["grpc_listen_port"], + } + } + + @property + def _positions(self) -> dict: + """Generates the positions section of the Promtail config file. + + Returns: + A dict representing the `positions` section. + """ + return {"positions": {"filename": WORKLOAD_POSITIONS_PATH}} + + def _scrape_configs(self, container_name: str) -> dict: + """Generates the scrape_configs section of the Promtail config file. + + Returns: + A dict representing the `scrape_configs` section. + """ + job_name = f"juju_{self.topology.identifier}" + + # The new JujuTopology doesn't include unit, but LogProxyConsumer should have it + common_labels = { + f"juju_{k}": v + for k, v in self.topology.as_dict(remapped_keys={"charm_name": "charm"}).items() + } + common_labels["container"] = container_name + scrape_configs = [] + + # Files config + labels = common_labels.copy() + labels.update( + { + "job": job_name, + "__path__": "", + } + ) + config = {"targets": ["localhost"], "labels": labels} + scrape_config = { + "job_name": "system", + "static_configs": self._generate_static_configs(config, container_name), + } + scrape_configs.append(scrape_config) + + # Syslog config + syslog_port = self._logs_scheme.get(container_name, {}).get("syslog-port") + if syslog_port: + relabel_mappings = [ + "severity", + "facility", + "hostname", + "app_name", + "proc_id", + "msg_id", + ] + syslog_labels = common_labels.copy() + syslog_labels.update({"job": f"{job_name}_syslog"}) + syslog_config = { + "job_name": "syslog", + "syslog": { + "listen_address": f"127.0.0.1:{syslog_port}", + "label_structured_data": True, + "labels": syslog_labels, + }, + "relabel_configs": [ + {"source_labels": [f"__syslog_message_{val}"], "target_label": val} + for val in relabel_mappings + ] + + [{"action": "labelmap", "regex": "__syslog_message_sd_(.+)"}], + } + scrape_configs.append(syslog_config) # type: ignore + + return {"scrape_configs": scrape_configs} + + def _generate_static_configs(self, config: dict, container_name: str) -> list: + """Generates static_configs section. + + Returns: + - a list of dictionaries representing static_configs section + """ + static_configs = [] + + for _file in self._logs_scheme.get(container_name, {}).get("log-files", []): + conf = deepcopy(config) + conf["labels"]["__path__"] = _file + static_configs.append(conf) + + return static_configs + + def _promtail_binary_spec(self) -> dict: + """The promtail binary metadata advertised on the log-proxy relation.""" + relations = self._charm.model.relations[self._relation_name] + if not relations: + return {} + relation = relations[0] + return json.loads(relation.data[relation.app].get("promtail_binary_zip_url", "{}")) + + def _is_promtail_set_up(self, container: Container) -> bool: + """Whether promtail is fully usable in this container. + + Unlike ``_is_promtail_installed`` (binary only), this also requires the + pebble service to be registered. + + The pebble plan alone is not proof: if the workload container lost its + ephemeral filesystem (e.g. after pod churn) the layer may still be in + the plan while the runtime-pushed binary is gone. Trusting the plan and + restarting unconditionally wedges the unit + (https://github.com/canonical/loki-k8s-operator/issues/659). + """ + if "promtail" not in container.get_plan().services: + return False + promtail_info = self._promtail_binary_spec().get(self._arch) + if not promtail_info: + # No promtail binary advertised for this architecture (or nothing + # published on the relation yet), so promtail cannot be running here. + return False + return self._is_promtail_installed(promtail_info, container) + + def _setup_promtail(self, container: Container) -> None: + # Use the first + relations = self._charm.model.relations[self._relation_name] + if len(relations) > 1: + logger.debug( + "Multiple log_proxy relations. Getting Promtail from application {}".format( + relations[0].app.name + ) + ) + relation = relations[0] + promtail_binaries = json.loads( + relation.data[relation.app].get("promtail_binary_zip_url", "{}") + ) + if not promtail_binaries: + # The Loki charm hasn't published the binary metadata yet; a later + # relation-changed will bring us back here. + return + + if self._arch not in promtail_binaries: + msg = f"No promtail binary available for architecture {self._arch}" + logger.warning(msg) + self.on.promtail_digest_error.emit(msg) + return + + self._create_directories(container) + if not self._ensure_promtail_binary(promtail_binaries, container): + # Do not add the pebble layer: a service whose command points to a + # missing binary would wedge the unit on every restart attempt + # (https://github.com/canonical/loki-k8s-operator/issues/659). + return + + container.push( + WORKLOAD_CONFIG_PATH, + yaml.safe_dump(self._promtail_config(container.name)), + make_dirs=True, + ) + + workload_binary_path = os.path.join( + WORKLOAD_BINARY_DIR, promtail_binaries[self._arch]["filename"] + ) + self._add_pebble_layer(workload_binary_path, container) + + if self._current_config(container).get("clients"): + try: + container.restart(WORKLOAD_SERVICE_NAME) + except ChangeError as e: + self.on.promtail_digest_error.emit(str(e)) + else: + self.on.log_proxy_endpoint_joined.emit() + else: + self.on.promtail_digest_error.emit("No promtail client endpoints available!") + + def _ensure_promtail_binary(self, promtail_binaries: dict, container: Container) -> bool: + """Ensure the promtail binary is present in the workload container. + + Args: + promtail_binaries: dictionary of promtail binaries per architecture. + container: container in which promtail must be installed. + + Returns: + True if the binary is available, False if it could not be obtained. + """ + if self._is_promtail_installed(promtail_binaries[self._arch], container): + return True + + try: + self._obtain_promtail(promtail_binaries[self._arch], container) + except URLError as e: + msg = f"Promtail binary couldn't be downloaded - {str(e)}" + logger.warning(msg) + self.on.promtail_digest_error.emit(msg) + return False + return True + + def _is_promtail_installed(self, promtail_info: dict, container: Container) -> bool: + """Determine if promtail has already been installed to the container. + + Args: + promtail_info: dictionary containing information about promtail binary + that must be used. The dictionary must at least contain a key + "filename" giving the name of promtail binary + container: container in which to check whether promtail is installed. + """ + workload_binary_path = f"{WORKLOAD_BINARY_DIR}/{promtail_info['filename']}" + try: + container.list_files(workload_binary_path) + except (APIError, FileNotFoundError): + return False + return True + + def _generate_promtails_ports(self, logs_scheme) -> dict: + return { + container: { + "http_listen_port": HTTP_LISTEN_PORT_START + 2 * i, + "grpc_listen_port": GRPC_LISTEN_PORT_START + 2 * i, + } + for i, container in enumerate(logs_scheme.keys()) + } + + def syslog_port(self, container_name: str) -> str: + """Gets the port on which promtail is listening for syslog in this container. + + Returns: + A str representing the port + """ + return str(self._logs_scheme.get(container_name, {}).get("syslog-port")) + + def rsyslog_config(self, container_name: str) -> str: + """Generates a config line for use with rsyslog. + + Returns: + The rsyslog config line as a string + """ + return 'action(type="omfwd" protocol="tcp" target="127.0.0.1" port="{}" Template="RSYSLOG_SyslogProtocol23Format" TCP_Framing="octet-counted")'.format( + self._logs_scheme.get(container_name, {}).get("syslog-port") + ) + + @property + def _containers(self) -> Dict[str, Container]: + return {cont: self._charm.unit.get_container(cont) for cont in self._logs_scheme.keys()} + + +class _PebbleLogClient: + @staticmethod + def check_juju_version() -> bool: + """Make sure the Juju version supports Log Forwarding.""" + juju_version = JujuVersion.from_environ() + if not juju_version > JujuVersion(version=str("3.3")): + msg = f"Juju version {juju_version} does not support Pebble log forwarding. Juju >= 3.4 is needed." + logger.warning(msg) + return False + return True + + @staticmethod + def _build_log_target( + unit_name: str, loki_endpoint: str, topology: JujuTopology, enable: bool + ) -> Dict: + """Build a log target for the log forwarding Pebble layer. + + Log target's syntax for enabling/disabling forwarding is explained here: + https://github.com/canonical/pebble?tab=readme-ov-file#log-forwarding + """ + services_value = ["all"] if enable else ["-all"] + + log_target = { + "override": "replace", + "services": services_value, + "type": "loki", + "location": loki_endpoint, + } + if enable: + log_target.update( + { + "labels": { + "product": "Juju", + "charm": topology._charm_name, + "juju_model": topology._model, + "juju_model_uuid": topology._model_uuid, + "juju_application": topology._application, + "juju_unit": topology._unit, + "job": f"juju_{topology.identifier}", + }, + } + ) + + return {unit_name: log_target} + + @staticmethod + def _build_log_targets( + loki_endpoints: Optional[Dict[str, str]], topology: JujuTopology, enable: bool + ): + """Build all the targets for the log forwarding Pebble layer.""" + targets = {} + if not loki_endpoints: + return targets + + for unit_name, endpoint in loki_endpoints.items(): + targets.update( + _PebbleLogClient._build_log_target( + unit_name=unit_name, + loki_endpoint=endpoint, + topology=topology, + enable=enable, + ) + ) + return targets + + @staticmethod + def disable_inactive_endpoints( + container: Container, active_endpoints: Dict[str, str], topology: JujuTopology + ): + """Disable forwarding for inactive endpoints by checking against the Pebble plan.""" + pebble_layer = container.get_plan().to_dict().get("log-targets", None) + if not pebble_layer: + return + + for unit_name, target in pebble_layer.items(): + # If the layer is a disabled log forwarding endpoint, skip it + if "-all" in target["services"]: # pyright: ignore + continue + + if unit_name not in active_endpoints: + layer = Layer( + { # pyright: ignore + "log-targets": _PebbleLogClient._build_log_targets( + loki_endpoints={unit_name: "(removed)"}, + topology=topology, + enable=False, + ) + } + ) + container.add_layer(f"{container.name}-log-forwarding", layer=layer, combine=True) + + @staticmethod + def enable_endpoints( + container: Container, active_endpoints: Dict[str, str], topology: JujuTopology + ): + """Enable forwarding for the specified Loki endpoints.""" + layer = Layer( + { # pyright: ignore + "log-targets": _PebbleLogClient._build_log_targets( + loki_endpoints=active_endpoints, + topology=topology, + enable=True, + ) + } + ) + container.add_layer(f"{container.name}-log-forwarding", layer, combine=True) + + +class LogForwarder(ConsumerBase): + """Forward the standard outputs of all workloads operated by a charm to one or multiple Loki endpoints. + + This class implements Pebble log forwarding. Juju >= 3.4 is needed. + """ + + def __init__( + self, + charm: CharmBase, + *, + relation_name: str = DEFAULT_RELATION_NAME, + alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, + recursive: bool = True, + skip_alert_topology_labeling: bool = False, + refresh_event: Optional[Union[BoundEvent, List[BoundEvent]]] = None, + forward_alert_rules: bool = True, + ): + _PebbleLogClient.check_juju_version() + super().__init__( + charm, + relation_name, + alert_rules_path, + recursive, + skip_alert_topology_labeling, + forward_alert_rules=forward_alert_rules, + ) + self._charm = charm + self._relation_name = relation_name + + on = self._charm.on[self._relation_name] + self.framework.observe(on.relation_joined, self._update_logging) + self.framework.observe(on.relation_changed, self._update_logging) + self.framework.observe(on.relation_departed, self._update_logging) + self.framework.observe(on.relation_broken, self._update_logging) + + if refresh_event: + if not isinstance(refresh_event, list): + refresh_event = [refresh_event] + for ev in refresh_event: + self.framework.observe(ev, self._update_logging) + + for container_name in self._charm.meta.containers.keys(): + snake_case_container_name = container_name.replace("-", "_") + self.framework.observe( + getattr(self._charm.on, f"{snake_case_container_name}_pebble_ready"), + self._on_pebble_ready, + ) + + def _on_pebble_ready(self, event: PebbleReadyEvent): + if not (loki_endpoints := self._retrieve_endpoints_from_relation()): + logger.warning("No Loki endpoints available") + return + + self._update_endpoints(event.workload, loki_endpoints) + + def _update_logging(self, event: RelationEvent): + """Update the log forwarding to match the active Loki endpoints.""" + if not (loki_endpoints := self._retrieve_endpoints_from_relation()): + logger.warning("No Loki endpoints available") + return + + for container in self._charm.unit.containers.values(): + if container.can_connect(): + self._update_endpoints(container, loki_endpoints) + # else: `_update_endpoints` will be called on pebble-ready anyway. + + self._handle_alert_rules(event.relation) + + def _retrieve_endpoints_from_relation(self) -> dict: + loki_endpoints = {} + + # Get the endpoints from relation data + for relation in self._charm.model.relations[self._relation_name]: + loki_endpoints.update(self._fetch_endpoints(relation)) + + return loki_endpoints + + def _update_endpoints(self, container: Container, loki_endpoints: dict): + _PebbleLogClient.disable_inactive_endpoints( + container=container, + active_endpoints=loki_endpoints, + topology=self.topology, + ) + _PebbleLogClient.enable_endpoints( + container=container, active_endpoints=loki_endpoints, topology=self.topology + ) + + def is_ready(self, relation: Optional[Relation] = None): + """Check if the relation is active and healthy.""" + if not relation: + relations = self._charm.model.relations[self._relation_name] + if not relations: + return False + return all(self.is_ready(relation) for relation in relations) + + try: + if self._extract_urls(relation): + return True + return False + except (KeyError, json.JSONDecodeError): + return False + + def _extract_urls(self, relation: Relation) -> Dict[str, str]: + """Default getter function to extract Loki endpoints from a relation. + + Returns: + A dictionary of remote units and the respective Loki endpoint. + { + "loki/0": "http://loki:3100/loki/api/v1/push", + "another-loki/0": "http://another-loki:3100/loki/api/v1/push", + } + """ + endpoints: Dict = {} + + for unit in relation.units: + endpoint = relation.data[unit]["endpoint"] + deserialized_endpoint = json.loads(endpoint) + url = deserialized_endpoint["url"] + endpoints[unit.name] = url + + return endpoints + + def _fetch_endpoints(self, relation: Relation) -> Dict[str, str]: + """Fetch Loki Push API endpoints from relation data using the endpoints getter.""" + endpoints: Dict = {} + + if not self.is_ready(relation): + logger.warning(f"The relation '{relation.name}' is not ready yet.") + return endpoints + + # if the code gets here, the function won't raise anymore because it's + # also called in is_ready() + endpoints = self._extract_urls(relation) + + return endpoints + + +def charm_logging_config( + endpoint_requirer: LokiPushApiConsumer, cert_path: Optional[Union[Path, str]] +) -> Tuple[Optional[List[str]], Optional[str]]: + """Utility function to determine the charm_logging config you will likely want. + + If no endpoint is provided: + disable charm logging. + If https endpoint is provided but cert_path is not found on disk: + disable charm logging. + If https endpoint is provided and cert_path is None: + ERROR + Else: + proceed with charm logging (with or without tls, as appropriate) + + Args: + endpoint_requirer: an instance of LokiPushApiConsumer. + cert_path: a path where a cert is stored. + + Returns: + A tuple with (optionally) the values of the endpoints and the certificate path. + + Raises: + LokiPushApiError: if some endpoint are http and others https. + """ + endpoints = [ep["url"] for ep in endpoint_requirer.loki_endpoints] + if not endpoints: + return None, None + + https = tuple(endpoint.startswith("https://") for endpoint in endpoints) + + if all(https): # all endpoints are https + if cert_path is None: + raise LokiPushApiError("Cannot send logs to https endpoints without a certificate.") + if not Path(cert_path).exists(): + # if endpoints is https BUT we don't have a server_cert yet: + # disable charm logging until we do to prevent tls errors + return None, None + return endpoints, str(cert_path) + + if all(not x for x in https): # all endpoints are http + return endpoints, None + + # if there's a disagreement, that's very weird: + raise LokiPushApiError("Some endpoints are http, some others are https. That's not good.")