From c7c7125501e7929f418b3b8c278364cb87ccbe66 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Mon, 11 Aug 2025 12:41:05 +0200 Subject: [PATCH 01/34] Initial commit to the branch. --- mreg/api/permissions.py | 27 ++++++- mreg/api/treetop.py | 110 ++++++++++++++++++++++++++ pyproject.toml | 3 +- treetop/data/host_labels.json | 18 +++++ treetop/data/mreg.cedar | 141 ++++++++++++++++++++++++++++++++++ treetop/docker-compose.yml | 19 +++++ uv.lock | 114 +++++++++++++++++++++++---- 7 files changed, 414 insertions(+), 18 deletions(-) create mode 100644 mreg/api/treetop.py create mode 100644 treetop/data/host_labels.json create mode 100644 treetop/data/mreg.cedar create mode 100644 treetop/docker-compose.yml diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index f16aa32c..95e49b1b 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -11,6 +11,7 @@ from mreg.models.network import NetGroupRegexPermission, Network from mreg.models.auth import User +from mreg.api.treetop import policy_parity # NOTE: We _must_ import `rest_framework.generics` in an `if TYPE_CHECKING:` # block because DRF does some dynamic import shenanigans on runtime using @@ -64,7 +65,18 @@ class IsSuperGroupMember(IsAuthenticated): def has_permission(self, request, view): if not super().has_permission(request, view): return False - return User.from_request(request).is_mreg_superuser + + user = + + return policy_parity( + User.from_request(request).is_mreg_superuser, + request=request, + view=view, + permission_class=self.__class__.__name__, + action="is_superuser", + resource_kind="Generic", + resource_attrs={"kind": "Any", "id": "any"}, + ) class IsSuperOrAdminOrReadOnly(IsAuthenticated): @@ -77,8 +89,17 @@ def has_permission(self, request, view): return False if request.method in SAFE_METHODS: return True - return User.from_request(request).is_mreg_superuser_or_admin - + return policy_parity( + User.from_request(request).is_mreg_superuser_or_admin, + request=request, + view=view, + permission_class=self.__class__.__name__, + action="is_admin", # Superadmins don't care what the action is + resource_kind="Generic", + resource_attrs={"kind": "Any", "id": "any"}, + ) + + class IsSuperOrNetworkAdminMember(IsAuthenticated): """ diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py new file mode 100644 index 00000000..645d35f0 --- /dev/null +++ b/mreg/api/treetop.py @@ -0,0 +1,110 @@ +from __future__ import annotations +import logging +from typing import Any, Optional + +from django.conf import settings +from rest_framework.request import Request +from django.views import View + +from mreg.models.auth import User as MregUser # your request->user wrapper + +from treetop_client.client import TreeTopClient +from treetop_client.models import Request as TreeTopRequest, User as TreeTopUser, Action as TreeTopAction, Resource as TreeTopResource + +logger = logging.getLogger("mreg.policy.parity") + +# Configure these in settings.py +POLICY_PARITY_ENABLED = getattr(settings, "POLICY_PARITY_ENABLED", True) +POLICY_BASE_URL = getattr(settings, "POLICY_BASE_URL", "http://localhost:9999") +POLICY_NAMESPACE = getattr(settings, "POLICY_NAMESPACE", ["MREG"]) +POLICY_EXTRA_LOG_FILE_NAME = getattr(settings, "POLICY_EXTRA_LOG_FILE_NAME", "policy_parity.log") +POLICY_TRUNCATE_LOG_FILE = getattr(settings, "POLICY_TRUNCATE_LOG_FILE", True) + +if POLICY_TRUNCATE_LOG_FILE: + with open(POLICY_EXTRA_LOG_FILE_NAME, "w"): + pass + +treetopclient = TreeTopClient(base_url=POLICY_BASE_URL) + +def _corr_id(request: Request) -> Optional[str]: + return request.headers.get("X-Correlation-ID") or request.META.get("HTTP_X_CORRELATION_ID") + +def _model_name_from_view(view) -> str: # type: ignore + # Best effort: try serializer model, else view class name + try: + sc = view.get_serializer_class() + return sc.Meta.model.__name__ + except Exception: + return view.__class__.__name__ + +def policy_parity( + decision: bool, + *, + request: Request, + view: Optional[View] = None, + permission_class: Optional[str] = None, + action: str, + resource_kind: str, + resource_attrs: dict[str, Any], +) -> bool: + """ + Log legacy-vs-policy parity and return `decision` unchanged. + Use this anywhere you currently 'return True/False'. + """ + if not POLICY_PARITY_ENABLED: + return decision + + # Build policy request + muser = MregUser.from_request(request) + principal = TreeTopUser.new(muser.username, POLICY_NAMESPACE, groups=list(muser.group_list)) + pol_action = TreeTopAction.new(action, POLICY_NAMESPACE) + res = TreeTopResource.new(resource_kind, resource_attrs) + + context = { + "path": request.path, + "method": request.method, + "permission": permission_class or (view and view.__class__.__name__), + "view": view and view.__class__.__name__, + "resource_kind": resource_kind, + "action": getattr(pol_action, "name", str(pol_action)), + "principal": muser.username, + "groups": list(muser.group_list), + "model": _model_name_from_view(view), + "correlation_id": _corr_id(request), + } + + pol_allowed, error = None, None + try: + resp = treetopclient.check(TreeTopRequest(principal=principal, action=pol_action, resource=res)) + pol_allowed = bool(resp.is_allowed()) + except Exception as exc: + error = repr(exc) + + parity = False + if bool(decision) and pol_allowed: + parity = True + elif not bool(decision) and not pol_allowed: + parity = True + + payload: dict[str, object] = { + **context, + "legacy_decision": bool(decision), + "policy_decision": pol_allowed, + "parity": parity, + "resource_attrs": resource_attrs, + "error": error, + } + + if parity: + logger.warning("policy_parity_mismatch", extra=payload) + log_policy_parity("OK", payload) + else: + logger.info("policy_parity_ok", extra=payload) + log_policy_parity("MISMATCH", payload) + + return decision + +# Log data to a file in addition to normal logging +def log_policy_parity(result: str, payload: dict[str, Any]): + with open(POLICY_EXTRA_LOG_FILE_NAME, "a") as log_file: + log_file.write(f"{result}: {payload}\n") diff --git a/pyproject.toml b/pyproject.toml index bdfbe703..87a156c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mreg" -requires-python = ">=3.10" +requires-python = ">=3.12" dependencies = [ "Django>=5.2", "djangorestframework>=3.16.0,<3.17", @@ -26,6 +26,7 @@ dependencies = [ "pyyaml", # For testing inside Docker image "unittest-parametrize", + "treetop-client>=0.0.1", "prometheus-client>=0.20", ] dynamic = ["version"] diff --git a/treetop/data/host_labels.json b/treetop/data/host_labels.json new file mode 100644 index 00000000..619c36dd --- /dev/null +++ b/treetop/data/host_labels.json @@ -0,0 +1,18 @@ +[ + { + "name": "in_domain", + "regex": "example\\.com$" + }, + { + "name": "valid_webserver_name", + "regex": "^web-\\d+" + }, + { + "name": "admin_subdomain", + "regex": "^admin\\." + }, + { + "name": "staging_environment", + "regex": "^staging\\." + } +] \ No newline at end of file diff --git a/treetop/data/mreg.cedar b/treetop/data/mreg.cedar new file mode 100644 index 00000000..20d19ebe --- /dev/null +++ b/treetop/data/mreg.cedar @@ -0,0 +1,141 @@ +// MREG permissions example. + +@id("MREG.admins_policy") +permit ( + principal in MREG::Group::"admins", + action in + [MREG::Action::"create_host", + MREG::Action::"delete_host", + MREG::Action::"view_host", + MREG::Action::"edit_host"], + resource is Host +); + +// Webadmins can edit, delete, or create hosts with a name label containing "webserver", and the IP +// address must be in the range 192.168.1.0/24 +@id("MREG.webadmins_policy") +permit ( + principal in MREG::Group::"webadmins", + action in + [MREG::Action::"edit_host", + MREG::Action::"delete_host", + MREG::Action::"create_host"], + resource is Host +) +when +{ + resource.nameLabels.contains("webserver") && + resource.ip.isInRange("192.168.1.0/24") +}; + +// Users can only view hosts +@id("MREG.users_policy") +permit ( + principal in MREG::Group::"users", + action == MREG::Action::"view_host", + resource is Host +); + +// Charlie does not get to delete hosts, no matter what. +@id("MREG.charlie_forbid_delete_host_policy") +forbid ( + principal == MREG::User::"charlie", + action == MREG::Action::"delete_host", + resource is Host +); + +// Admins can manipulate any IP address, even if it is a gw, a broadcast address, +// the network address, reserved. These three groups are unified as "restricted" IPs. +@id("MREG.admins_ip_policy") +permit ( + principal in MREG::Group::"admins", + action in + [MREG::Action::"ip_gw_management", + MREG::Action::"ip_broadcast_management", + MREG::Action::"ip_network_management", + MREG::Action::"ip_reserved_management", + MREG::Action::"ip_restricted_management"], + resource is IPAddress +); + +/// Network Admins can manage any IP in any network +@id("MREG.network_admins_ip_network_policy") +permit ( + principal in MREG::Group::"default-networkadmin-group", + action == MREG::Action::"ip_network_management", + resource is IPAddress +); + +/// Users can only manage IPs in specific networks +@id("MREG.users_ip_network_policy") +permit ( + principal in MREG::Group::"users", + action == MREG::Action::"ip_network_management", + resource is IPAddress +) +when +{ + resource.ip.isInRange("192.168.1.0/24") || + resource.ip.isInRange("10.0.0.0/8") +}; + +/// Admins can do whatever with labels. +@id("MREG.labels_admin_policy") +permit ( + principal in MREG::Group::"default-super-group", + action in + [MREG::Action::"create_label", + MREG::Action::"delete_label", + MREG::Action::"view_label", + MREG::Action::"edit_label"], + resource is Label +); + +/// Superadmins +@id("MREG.is_superuser") +permit ( + principal in MREG::Group::"default-super-group", + action, // Superadmins don't care what the action is + resource +); + +/// Normal (?) admins +@id("MREG.is_admin") +permit ( + principal in MREG::Group::"default-admin-group", + action == MREG::Action::"is_admin", + resource +); + +/// Host Policy Admins +@id("MREG.is_hostpolicy_admin") +permit ( + principal in MREG::Group::"default-hostpolicyadmin-group", + action == MREG::Action::"is_hostpolicy_admin", + resource +); + +/// DNS Wildcard Admins +@id("MREG.is_dns_wildcard_admin") +permit ( + principal in MREG::Group::"default-dns-wildcard-group", + action == MREG::Action::"is_dns_wildcard_admin", + resource +); + +/// DNS Underscore Admins +@id("MREG.is_dns_underscore_admin") +permit ( + principal in MREG::Group::"default-dns-underscore-group", + action == MREG::Action::"is_dns_underscore_admin", + resource +); + +/// We also have a global super admin policy that allows the root super user +/// to do anything to any resource. +@id("global.super_admin_allow_all_policy") +permit ( + principal == User::"super", + action, + resource +); \ No newline at end of file diff --git a/treetop/docker-compose.yml b/treetop/docker-compose.yml new file mode 100644 index 00000000..fc7f817e --- /dev/null +++ b/treetop/docker-compose.yml @@ -0,0 +1,19 @@ +# This docker file allows you to set up a TreeTop policy server for tests. +services: + cedar-server: + image: svenstaro/miniserve + ports: + - "8080:8080" + volumes: + - ./data:/data:ro + command: ["/data", "--port", "8080"] + + treetop-server: + image: ghcr.io/terjekv/treetop-rest:develop + ports: + - "9999:9999" + environment: + - APP_POLICY_URL=http://cedar-server:8080/mreg.cedar + - APP_HOST_LABEL_URL=http://cedar-server:8080/host_labels.json + - RUST_LOG=info,treetop=debug + command: ["server", "--host", "0.0.0.0"] diff --git a/uv.lock b/uv.lock index af430883..505c36d3 100644 --- a/uv.lock +++ b/uv.lock @@ -7,16 +7,27 @@ resolution-markers = [ ] [[package]] -name = "asgiref" -version = "3.8.1" +name = "anyio" +version = "4.10.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "sniffio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/b4/636b3b65173d3ce9a38ef5f0522789614e590dab6a8d505340a4efe4c567/anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6", size = 213252, upload_time = "2025-08-04T08:54:26.451Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload_time = "2025-08-04T08:54:24.882Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/29/38/b3395cc9ad1b56d2ddac9970bc8f4141312dbaec28bc7c218b0dfafd0f42/asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590", size = 35186, upload-time = "2024-03-22T14:39:36.863Z" } + +[[package]] +name = "asgiref" +version = "3.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/38/b3395cc9ad1b56d2ddac9970bc8f4141312dbaec28bc7c218b0dfafd0f42/asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590", size = 35186, upload_time = "2024-03-22T14:39:36.863Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/39/e3/893e8757be2612e6c266d9bb58ad2e3651524b5b40cf56761e985a28b13e/asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47", size = 23828, upload-time = "2024-03-22T14:39:34.521Z" }, ] @@ -207,7 +218,7 @@ name = "coveralls" version = "4.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coverage", extra = ["toml"] }, + { name = "coverage" }, { name = "docopt" }, { name = "requests" }, ] @@ -399,7 +410,44 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload_time = "2024-08-10T20:25:24.996Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload_time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload_time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload_time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload_time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload_time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload_time = "2024-12-06T15:37:21.509Z" }, ] [[package]] @@ -462,6 +510,7 @@ dependencies = [ { name = "rich" }, { name = "sentry-sdk" }, { name = "structlog" }, + { name = "treetop-client" }, { name = "tzdata" }, { name = "unittest-parametrize" }, { name = "uritemplate" }, @@ -469,7 +518,7 @@ dependencies = [ [package.dev-dependencies] ci = [ - { name = "coverage", extra = ["toml"] }, + { name = "coverage" }, { name = "coveralls" }, { name = "pytest" }, { name = "pytest-django" }, @@ -478,7 +527,7 @@ ci = [ { name = "uv" }, ] dev = [ - { name = "coverage", extra = ["toml"] }, + { name = "coverage" }, { name = "pytest" }, { name = "pytest-django" }, { name = "tox-uv" }, @@ -711,7 +760,6 @@ version = "1.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/45/7b/c0e1333b61d41c69e59e5366e727b18c4992688caf0de1be10b3e5265f6b/pyproject_api-1.10.0.tar.gz", hash = "sha256:40c6f2d82eebdc4afee61c773ed208c04c19db4c4a60d97f8d7be3ebc0bbb330", size = 22785, upload-time = "2025-10-09T19:12:27.21Z" } wheels = [ @@ -724,11 +772,9 @@ version = "8.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8b/6c/62bbd536103af674e227c41a8f3dcd022d591f6eed5facb5a0f31ee33bbc/pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181", size = 1442487, upload-time = "2024-09-10T10:52:15.003Z" } wheels = [ @@ -851,6 +897,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254", size = 11053, upload-time = "2021-05-05T14:18:17.237Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload_time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload_time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload_time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload_time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "sqlparse" version = "0.5.1" @@ -934,8 +998,6 @@ dependencies = [ { name = "platformdirs" }, { name = "pluggy" }, { name = "pyproject-api" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, { name = "virtualenv" }, ] sdist = { url = "https://files.pythonhosted.org/packages/59/bf/0e4dbd42724cbae25959f0e34c95d0c730df03ab03f54d52accd9abfc614/tox-4.32.0.tar.gz", hash = "sha256:1ad476b5f4d3679455b89a992849ffc3367560bbc7e9495ee8a3963542e7c8ff", size = 203330, upload-time = "2025-10-24T18:03:38.132Z" } @@ -967,7 +1029,31 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/4f/90/06752775b8cfadba8856190f5beae9f552547e0f287e0246677972107375/tox_uv-1.29.0.tar.gz", hash = "sha256:30fa9e6ad507df49d3c6a2f88894256bcf90f18e240a00764da6ecab1db24895", size = 23427, upload-time = "2025-10-09T20:40:27.384Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/17/221d62937c4130b044bb437caac4181e7e13d5536bbede65264db1f0ac9f/tox_uv-1.29.0-py3-none-any.whl", hash = "sha256:b1d251286edeeb4bc4af1e24c8acfdd9404700143c2199ccdbb4ea195f7de6cc", size = 17254, upload-time = "2025-10-09T20:40:25.885Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8e/94afb25547f5e4987801e8f6aa11e357190f72f31eb363267a3cb2fa6a88/tox_uv-1.13.1-py3-none-any.whl", hash = "sha256:b163dd28ca37a9f4c6d8cbac11153be27c2e929b58bcae62e323ffa8f71c327d", size = 13383, upload-time = "2024-10-11T16:14:55.885Z" }, +] + +[[package]] +name = "treetop-client" +version = "0.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/a1/ae06eb362cdc44ccf2bea1c2af6856e038cebac63ceb4908434482e1323a/treetop_client-0.0.2.tar.gz", hash = "sha256:c89794d09fc9c31d39cb0e13ee03f1559bbe2ec893a0647698f48747a1759f8f", size = 5103, upload_time = "2025-07-28T19:34:36.677Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/e4/9587921902ece105f616310b66cae9e4f40e38c8673d5e74ff2a66b61d57/treetop_client-0.0.2-py3-none-any.whl", hash = "sha256:af08d71d9d74a3913c7655fe3a5e1dc579e335ddecf562710ca2443b5227d16a", size = 6108, upload_time = "2025-07-28T19:34:35.405Z" }, +] + +[[package]] +name = "treetop-client" +version = "0.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/a1/ae06eb362cdc44ccf2bea1c2af6856e038cebac63ceb4908434482e1323a/treetop_client-0.0.2.tar.gz", hash = "sha256:c89794d09fc9c31d39cb0e13ee03f1559bbe2ec893a0647698f48747a1759f8f", size = 5103, upload_time = "2025-07-28T19:34:36.677Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/e4/9587921902ece105f616310b66cae9e4f40e38c8673d5e74ff2a66b61d57/treetop_client-0.0.2-py3-none-any.whl", hash = "sha256:af08d71d9d74a3913c7655fe3a5e1dc579e335ddecf562710ca2443b5227d16a", size = 6108, upload_time = "2025-07-28T19:34:35.405Z" }, ] [[package]] From 19fa8a4b653d6757286248845a4341bec5e1a9a9 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Mon, 11 Aug 2025 12:41:39 +0200 Subject: [PATCH 02/34] Sigh. --- mreg/api/permissions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index 95e49b1b..9f3e91ab 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -66,8 +66,6 @@ def has_permission(self, request, view): if not super().has_permission(request, view): return False - user = - return policy_parity( User.from_request(request).is_mreg_superuser, request=request, From 97947510f84178737aeca67ff89a7cba67e4865e Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 12 Nov 2025 11:40:26 +0100 Subject: [PATCH 03/34] Refactoring, flattning permissions for now. --- mreg/api/permissions.py | 474 +++++++++++++++------ mreg/api/treetop.py | 54 ++- mreg/api/v1/tests/test_host_permissions.py | 1 + mreg/api/v1/tests/test_permissions.py | 5 +- mreg/models/network.py | 6 + pyproject.toml | 4 +- treetop/data/mreg.cedar | 62 ++- treetop/docker-compose.yml | 1 + uv.lock | 6 +- 9 files changed, 435 insertions(+), 178 deletions(-) diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index 9f3e91ab..20a46f82 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -1,16 +1,19 @@ from __future__ import annotations import ipaddress -from typing import TYPE_CHECKING +from django.db import models +from typing import TYPE_CHECKING, Iterable, Mapping, Optional, Tuple, Any from rest_framework import exceptions from rest_framework.permissions import IsAuthenticated as DRFIsAuthenticated, SAFE_METHODS from rest_framework.request import Request +from structlog import get_logger + from mreg.api.v1.serializers import HostSerializer from mreg.models.host import HostGroup from mreg.models.network import NetGroupRegexPermission, Network -from mreg.models.auth import User +from mreg.models.auth import User, MregAdminGroup from mreg.api.treetop import policy_parity # NOTE: We _must_ import `rest_framework.generics` in an `if TYPE_CHECKING:` @@ -22,7 +25,226 @@ from rest_framework.serializers import Serializer from mreg.models.base import BaseModel +logger = get_logger() + +DEFAULT_RESOURCE_ATTRS = {"kind": "Any", "id": "any"} + + +class ParityMixin: + """Small helpers to reduce repetition around policy_parity.""" + + def pp( + self, + *, + decision: bool, + action: str, + request: Request, + view: "GenericAPIView", + resource_kind: str = "Generic", + resource_id: str = "any", + resource_attrs: Optional[Mapping[str, str]] = None, + ) -> bool: + return policy_parity( + decision, + request=request, + view=view, + permission_class=self.__class__.__name__, + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + ) + + def pp_host( + self, + *, + decision: bool, + request: Request, + view: "GenericAPIView", + resource_id: str = "", + action: str = "host_access", + resource_attrs: Optional[Mapping[str, str]] = None, + ) -> bool: + """Helper for host-related actions. + + Assumes `resource_kind="Host"` and `action="host_access"`, and tries to extract the resource ID from + `resource_attrs["hostname"]` if not explicitly given. + """ + + if not resource_id and resource_attrs and hasattr(resource_attrs, "hostname"): + resource_id = resource_attrs["hostname"] + + return self.pp( + decision=decision, + action=action, + request=request, + view=view, + resource_kind="Host", + resource_id=resource_id or "any", + resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + ) + + def pp_any( + self, + *, + checks: Iterable[Tuple[bool, str]], # (decision, action) + request: Request, + view: "GenericAPIView", + resource_kind: str = "Generic", + resource_attrs: Optional[Mapping[str, str]] = None, + ) -> bool: + for decision, action in checks: + if self.pp( + decision=decision, + action=action, + request=request, + view=view, + resource_kind=resource_kind, + resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + ): + return True + return False + + def pp_all( + self, + *, + checks: Iterable[Tuple[bool, str]], + request: Request, + view: "GenericAPIView", + resource_kind: str = "Generic", + resource_attrs: Optional[Mapping[str, str]] = None, + ) -> bool: + for decision, action in checks: + if not self.pp( + decision=decision, + action=action, + request=request, + view=view, + resource_kind=resource_kind, + resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + ): + return False + return True + def pp_generic_action(self, attrs: Mapping[str, str], decision: bool, action: str, request: Request, view: GenericAPIView, kind: str = "Generic", id: str = "Any") -> bool: + return self.pp( + decision=decision, + action=action, + request=request, + view=view, + resource_kind=kind, + resource_id=id, + resource_attrs={ kind: kind, **attrs } + ) + + def user_has_permission(self, membership: MregAdminGroup, request: Request, view: GenericAPIView, exclude_superuser: bool = False) -> bool: + """ + Check if the user has a given generic permission level. + """ + user = User.from_request(request) + memberlist = membership.settings_groups_or_raise() + + if not exclude_superuser and membership != MregAdminGroup.SUPERUSER: + memberlist.extend(MregAdminGroup.SUPERUSER.settings_groups_or_raise()) + + is_member = user.is_member_of_any(memberlist) + + match membership: + case MregAdminGroup.SUPERUSER: + action = "superuser_access" + case MregAdminGroup.ADMINUSER: + action = "admin_access" + case MregAdminGroup.GROUP_ADMIN: + action = "hostgroup_admin_access" + case MregAdminGroup.NETWORK_ADMIN: + action = "network_admin_access" + case MregAdminGroup.DNS_WILDCARD: + action = "dns_wildcard_admin_access" + case MregAdminGroup.DNS_UNDERSCORE: + action = "dns_underscore_admin_access" + case MregAdminGroup.HOSTPOLICY_ADMIN: + action = "hostpolicy_admin_access" + + return self.pp( + decision=is_member, + action=action, + request=request, + view=view, + ) + + def user_is_superuser(self, request: Request, view: GenericAPIView) -> bool: + """ + Check if the user is a superuser. + """ + return self.user_has_permission( + membership=MregAdminGroup.SUPERUSER, + request=request, + view=view, + ) + + def user_is_admin(self, request: Request, view: GenericAPIView) -> bool: + """ + Check if the user is an admin. + """ + return self.user_has_permission( + membership=MregAdminGroup.ADMINUSER, + request=request, + view=view, + ) + + def user_is_network_admin(self, request: Request, view: GenericAPIView) -> bool: + """ + Check if the user is a network admin. + """ + return self.user_has_permission( + membership=MregAdminGroup.NETWORK_ADMIN, + request=request, + view=view, + ) + + def user_is_dns_wildcard_admin(self, request: Request, view: GenericAPIView) -> bool: + """ + Check if the user is a DNS wildcard admin. + """ + return self.user_has_permission( + membership=MregAdminGroup.DNS_WILDCARD, + request=request, + view=view, + ) + + def user_is_dns_underscore_admin(self, request: Request, view: GenericAPIView) -> bool: + """ + Check if the user is a DNS underscore admin. + """ + return self.user_has_permission( + membership=MregAdminGroup.DNS_UNDERSCORE, + request=request, + view=view, + ) + + def user_is_hostgroup_admin(self, request: Request, view: GenericAPIView) -> bool: + """ + Check if the user is a hostgroup admin. + """ + return self.user_has_permission( + membership=MregAdminGroup.GROUP_ADMIN, + request=request, + view=view, + ) + + def user_is_any( + self, + *memberships: MregAdminGroup, + request: Request, + view: GenericAPIView + ) -> bool: + """ + Check if the user is a member of any of the given groups. + """ + for membership in memberships: + if self.user_has_permission(membership, request, view): + return True + return False class CRUDPermissionsMixin: """ @@ -43,10 +265,55 @@ def has_destroy_permission(self, request: Request, view: GenericAPIView, validat return False -class IsAuthenticated(DRFIsAuthenticated, CRUDPermissionsMixin): +class IsAuthenticated(DRFIsAuthenticated, CRUDPermissionsMixin, ParityMixin): """ Allows access only to authenticated users. """ + + def deny_superuser_only_names(self, data=None, name=None, view=None, request=None): + """Check for superuser only names. If match, return True.""" + import mreg.api.v1.views as v1_views + + if data is not None: + name = data.get('name', '') + if not name: + if 'host' in data: + name = data['host'].name + + name = (name or '').strip() # Guarantee coercion to string + + if not request: # pragma: no cover + return False + + if not view: # pragma: no cover + return False + + # Underscore is allowed for non-superuser in SRV records, + # and for members of in all records. + if '_' in name and not isinstance(view, (v1_views.SrvDetail, v1_views.SrvList)) \ + and not self.user_is_dns_underscore_admin(request, view): + return True + + # Except for super-users, only members of the DNS wildcard group can create wildcard records. + # And then only below subdomains, like *.sub.example.com + if '*' in name and (not self.user_is_dns_wildcard_admin(request, view) or name.count('.') < 3): + return True + + return False + + def deny_reserved_ipaddress(self, ip: str, request: Request, view: GenericAPIView) -> bool: + """Check if an ip address is reserved, and if so, only permit + NETWORK_ADMIN_GROUP members.""" + + if self.user_is_network_admin(request, view): + return False + + network = Network.objects.filter(network__net_contains=ip).first() + if not network: + return False + + return network.is_reserved_ipaddress(ip) + pass @@ -66,15 +333,7 @@ def has_permission(self, request, view): if not super().has_permission(request, view): return False - return policy_parity( - User.from_request(request).is_mreg_superuser, - request=request, - view=view, - permission_class=self.__class__.__name__, - action="is_superuser", - resource_kind="Generic", - resource_attrs={"kind": "Any", "id": "any"}, - ) + return self.user_is_superuser(request=request, view=view) class IsSuperOrAdminOrReadOnly(IsAuthenticated): @@ -87,15 +346,7 @@ def has_permission(self, request, view): return False if request.method in SAFE_METHODS: return True - return policy_parity( - User.from_request(request).is_mreg_superuser_or_admin, - request=request, - view=view, - permission_class=self.__class__.__name__, - action="is_admin", # Superadmins don't care what the action is - resource_kind="Generic", - resource_attrs={"kind": "Any", "id": "any"}, - ) + return self.user_is_admin(request=request, view=view) @@ -108,12 +359,7 @@ def has_permission(self, request, view): if not super().has_permission(request, view): return False - user = User.from_request(request) - if user.is_mreg_superuser: - return True - if user.is_mreg_network_admin: - return True - return False + return self.user_is_any(MregAdminGroup.SUPERUSER, MregAdminGroup.NETWORK_ADMIN, request=request, view=view) class IsSuperOrGroupAdminOrReadOnly(IsAuthenticated): @@ -124,57 +370,11 @@ class IsSuperOrGroupAdminOrReadOnly(IsAuthenticated): def has_permission(self, request, view): if not super().has_permission(request, view): return False - user = User.from_request(request) if request.method in SAFE_METHODS: return True - return user.is_mreg_superuser or user.is_mreg_hostgroup_admin - - -def _deny_superuser_only_names(data=None, name=None, view=None, request=None): - """Check for superuser only names. If match, return True.""" - import mreg.api.v1.views - - if data is not None: - name = data.get('name', '') - if not name: - if 'host' in data: - name = data['host'].name - - if not request: # pragma: no cover - return False - - user = User.from_request(request) - - # Underscore is allowed for non-superuser in SRV records, - # and for members of in all records. - if '_' in name and not isinstance(view, (mreg.api.v1.views.SrvDetail, - mreg.api.v1.views.SrvList)) \ - and not user.is_mreg_dns_underscore_admin: - return True - - # Except for super-users, only members of the DNS wildcard group can create wildcard records. - # And then only below subdomains, like *.sub.example.com - if '*' in name and (not user.is_mreg_dns_wildcard_admin or name.count('.') < 3): - return True - return False + return self.user_is_any(MregAdminGroup.SUPERUSER, MregAdminGroup.GROUP_ADMIN, request=request, view=view) - -def is_reserved_ip(ip): - network = Network.objects.filter(network__net_contains=ip).first() - if network: - return any(ip == str(i) for i in network.get_reserved_ipaddresses()) - return False - - -def _deny_reserved_ipaddress(ip, request): - """Check if an ip address is reserved, and if so, only permit - NETWORK_ADMIN_GROUP members.""" - if is_reserved_ip(ip): - if User.from_request(request).is_mreg_network_admin: - return False - return True - return False class IsGrantedNetGroupRegexPermission(IsAuthenticated): """ Permit user if the user has been granted access through a @@ -208,55 +408,87 @@ def has_permission(self, request, view): return True return False - @staticmethod - def has_perm(user, hostname, ips, require_ip=True): - return bool(NetGroupRegexPermission.find_perm(user.group_list, - hostname, ips, require_ip)) + def has_perm(self, user, hostname, ips, request: Request, view: GenericAPIView, require_ip=True): + legacy = bool(NetGroupRegexPermission.find_perm(user.group_list, hostname, ips, require_ip)) + policy: list[bool] = [] + if ips: + # This will perform one policy lookup per IP for the host. This should probably be optimized server side. + for ip in ips: + policy.append(self.pp_host(decision=legacy, request=request, view=view, resource_attrs={"hostname": hostname, "ip": ip})) + else: + policy.append(self.pp_host(decision=legacy, request=request, view=view, resource_attrs={"hostname": hostname})) + + return any(policy) - def has_obj_perm(self, user, obj): - return self.has_perm(user, *self._get_hostname_and_ips(obj)) + def has_obj_perm(self, user: User, obj: str, request: Request, view: GenericAPIView) -> bool: + return self.has_perm(user, *self._get_hostname_and_ips(obj), request=request, view=view) def has_create_permission(self, request, view, validated_serializer): - import mreg.api.v1.views + import mreg.api.v1.views as v1_views user = User.from_request(request) - if user.is_mreg_superuser: + + logger.debug("create_permission_check", user=user.username, view=view.__class__.__name__, data=validated_serializer.validated_data) + + if self.user_is_superuser(request=request, view=view): return True hostname = None ips = [] - data = validated_serializer.validated_data - if _deny_superuser_only_names(data=data, view=view, request=request): + + attrs: dict[str, Any] = {} + data: dict[str, Any] = validated_serializer.validated_data # type: ignore + + # Convert all data from the serializer to strings to feed as attributes to the policy engine. + # We also introspect BaseModel instances to flatten them out (one level deep). + # For example: + # key: Host value: hostobj -> attrs["host.id"] = "1", attrs["host.name"] = "hostname.example.com" + if data: + for key, value in data.items(): + if isinstance(value, (str, int, float, bool)): + attrs[key] = value + elif isinstance(value, models.Model): + for field in value._meta.fields: + attrs[f"{key}_{field.name}"] = str(getattr(value, field.name, '')) + else: + attrs[key] = str(value) + + + ipaddress = data.get('ipaddress', None) + host = data.get('host', None) + + object_type = validated_serializer.instance.__class__.__name__.lower() + # First check if we are asking for a restricted name. + if self.deny_superuser_only_names(data=data, view=view, request=request): return False - if 'ipaddress' in data: - if _deny_reserved_ipaddress(data['ipaddress'], request): - return False - if user.is_mreg_admin: + # Then check if we are asking for an IP address *and* it is reserved. + if ipaddress and self.deny_reserved_ipaddress(ip=ipaddress, view=view, request=request): + return False + # If the user is an admin, they are now free to create (minus the above checks). + if self.pp_generic_action(decision=user.is_mreg_admin, action="create", kind=object_type, attrs=attrs, request=request, view=view): return True - if isinstance(view, (mreg.api.v1.views.IpaddressList, - mreg.api.v1.views.PtrOverrideList)): - if 'host' in data: - if not self.has_obj_perm(user, data['host']): - return False - if isinstance(view, mreg.api.v1.views.CnameList): - # only check the cname, don't care about ip addresses - return self.has_perm(user, data['name'], (), require_ip=False) - if isinstance(view, (mreg.api.v1.views.HostList, - mreg.api.v1.views.IpaddressList, - mreg.api.v1.views.PtrOverrideList)): - # HostList does not require ipaddress, but if none, the permissions - # will not match, so just refuse it. - ip = data.get('ipaddress', None) - if ip is None: + # Now check if the user has permission to the host object (if any). + if isinstance(view, (v1_views.IpaddressList, v1_views.PtrOverrideList)): + if host and not self.has_obj_perm(user, host, request=request, view=view): + return False + # CNAMEs are special, we check only the cname, not the ip addresses. + if isinstance(view, v1_views.CnameList): + return self.has_perm(user, data['name'], (), require_ip=False, request=request, view=view) + # For hosts and other objects, we need to check the host and its IPs. + if isinstance(view, (v1_views.HostList, v1_views.IpaddressList, v1_views.PtrOverrideList)): + # HostList does not require ipaddress, but if none, the permissions will not match, so just refuse it. + # If the Host object is missing or invalid, refuse it (this should be caught by the serializer anyway). + if not (ipaddress and host): return False - ips.append(ip) - hostname = data['host'].name + + ips.append(ipaddress) + hostname = host.name elif 'host' in data: hostname, ips = self._get_hostname_and_ips(data['host']) else: raise exceptions.PermissionDenied(f"Unhandled view: {view}") if ips and hostname: - return self.has_perm(user, hostname, ips) + return self.has_perm(user, hostname, ips, request=request, view=view) return False def has_destroy_permission(self, request, view, validated_serializer): @@ -272,45 +504,47 @@ def has_destroy_permission(self, request, view, validated_serializer): obj = obj.host else: raise exceptions.PermissionDenied(f"Unhandled view: {view}") - if _deny_superuser_only_names(name=obj.name, view=view, request=request): + if self.deny_superuser_only_names(name=obj.name, view=view, request=request): return False if hasattr(obj, 'ipaddress'): - if _deny_reserved_ipaddress(obj.ipaddress, request): + if self.deny_reserved_ipaddress(ip=obj.ipaddress, view=view, request=request): return False - if user.is_mreg_admin: + + object_type = obj.__class__.__name__.lower() + if self.pp_generic_action(decision=user.is_mreg_admin, action="destroy", kind=object_type, attrs={"id": str(obj)}, request=request, view=view): return True - return self.has_obj_perm(user, obj) + return self.has_obj_perm(user, obj, request=request, view=view) def has_update_permission(self, request, view, validated_serializer): - import mreg.api.v1.views + import mreg.api.v1.views as v1_views user = User.from_request(request) if user.is_mreg_superuser: return True - data = validated_serializer.validated_data - if _deny_superuser_only_names(data=data, view=view, request=request): + data: dict[str, Any] = validated_serializer.validated_data # type: ignore + if self.deny_superuser_only_names(data=data, view=view, request=request): return False if 'ipaddress' in data: - if _deny_reserved_ipaddress(data['ipaddress'], request): + if self.deny_reserved_ipaddress(ip=data['ipaddress'], view=view, request=request): return False - if user.is_mreg_admin: + if self.user_is_admin(request=request, view=view): return True obj = view.get_object() - if isinstance(view, mreg.api.v1.views.HostDetail): + if isinstance(view, v1_views.HostDetail): hostname, ips = self._get_hostname_and_ips(obj) # If renaming a host, make sure the user has permission to both the # new and and old hostname. if 'name' in data: - if not self.has_perm(user, data['name'], ips): + if not self.has_perm(user, data['name'], ips, request=request, view=view): return False - return self.has_perm(user, hostname, ips) + return self.has_perm(user, hostname, ips, request=request, view=view) elif hasattr(obj, 'host'): # If changing host object, make sure the user has permission the # new one. if 'host' in data and data['host'] != obj.host: - if not self.has_obj_perm(user, data['host']): + if not self.has_obj_perm(user, data['host'], request=request, view=view): return False - return self.has_obj_perm(user, obj.host) + return self.has_obj_perm(user, obj.host, request=request, view=view) # Testing these kinds of should-never-happen codepaths is hard. # We have to basically mock a complete API call and then break it. raise exceptions.PermissionDenied(f"Unhandled view: {view}") # pragma: no cover diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index 645d35f0..5ee7f2fa 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -1,6 +1,8 @@ from __future__ import annotations import logging -from typing import Any, Optional +from typing import Any, Optional, Mapping +import ipaddress +import json from django.conf import settings from rest_framework.request import Request @@ -9,7 +11,7 @@ from mreg.models.auth import User as MregUser # your request->user wrapper from treetop_client.client import TreeTopClient -from treetop_client.models import Request as TreeTopRequest, User as TreeTopUser, Action as TreeTopAction, Resource as TreeTopResource +from treetop_client.models import Request as TreeTopRequest, User as TreeTopUser, Action, Resource, ResourceAttribute, ResourceAttributeType logger = logging.getLogger("mreg.policy.parity") @@ -45,7 +47,8 @@ def policy_parity( permission_class: Optional[str] = None, action: str, resource_kind: str, - resource_attrs: dict[str, Any], + resource_id: str, + resource_attrs: Mapping[str, str], ) -> bool: """ Log legacy-vs-policy parity and return `decision` unchanged. @@ -57,19 +60,41 @@ def policy_parity( # Build policy request muser = MregUser.from_request(request) principal = TreeTopUser.new(muser.username, POLICY_NAMESPACE, groups=list(muser.group_list)) - pol_action = TreeTopAction.new(action, POLICY_NAMESPACE) - res = TreeTopResource.new(resource_kind, resource_attrs) + pol_action = Action.new(action, POLICY_NAMESPACE) + + attrs = {} + + for k, v in resource_attrs.items(): + try: + ip = ipaddress.ip_address(v) + attrs[k] = ResourceAttribute.new(str(ip), ResourceAttributeType.IP) + except ValueError: + attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) +# if v.isdigit(): +# attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.NUMBER) +# elif v.lower() in ("true", "false"): +# attrs[k] = ResourceAttribute.new(v.lower(), ResourceAttributeType.BOOLEAN) +# else: +# attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) + + res = Resource.new(resource_kind, resource_id, attrs=attrs) + + if len(pol_action.id.namespace) > 0: + fully_qualified_action = "::".join(pol_action.id.namespace) + f"::{pol_action.id.id}" + else: + fully_qualified_action = f"{pol_action.id.id}" context = { "path": request.path, "method": request.method, "permission": permission_class or (view and view.__class__.__name__), "view": view and view.__class__.__name__, - "resource_kind": resource_kind, - "action": getattr(pol_action, "name", str(pol_action)), + "model": _model_name_from_view(view), "principal": muser.username, "groups": list(muser.group_list), - "model": _model_name_from_view(view), + "action": fully_qualified_action, + "resource_kind": resource_kind, + "resource_attrs": resource_attrs, "correlation_id": _corr_id(request), } @@ -87,24 +112,23 @@ def policy_parity( parity = True payload: dict[str, object] = { - **context, + "parity": parity, "legacy_decision": bool(decision), "policy_decision": pol_allowed, - "parity": parity, - "resource_attrs": resource_attrs, "error": error, + "context": context, } if parity: logger.warning("policy_parity_mismatch", extra=payload) - log_policy_parity("OK", payload) + log_policy_parity(payload) else: logger.info("policy_parity_ok", extra=payload) - log_policy_parity("MISMATCH", payload) + log_policy_parity(payload) return decision # Log data to a file in addition to normal logging -def log_policy_parity(result: str, payload: dict[str, Any]): +def log_policy_parity(payload: dict[str, Any]): with open(POLICY_EXTRA_LOG_FILE_NAME, "a") as log_file: - log_file.write(f"{result}: {payload}\n") + log_file.write(f"{json.dumps(payload)}\n") diff --git a/mreg/api/v1/tests/test_host_permissions.py b/mreg/api/v1/tests/test_host_permissions.py index 4d975915..9280390f 100644 --- a/mreg/api/v1/tests/test_host_permissions.py +++ b/mreg/api/v1/tests/test_host_permissions.py @@ -126,6 +126,7 @@ def test_can_not_change_host_out_of_permissions(self): def _post_and_get(name, ipaddress, client=self.client): data = {'name': name, 'ipaddress': ipaddress} ret = client.post('/api/v1/hosts/', data) + assert ret.status_code == 201 return self.assert_get(ret['Location']) Network.objects.create(network='10.2.0.0/25') diff --git a/mreg/api/v1/tests/test_permissions.py b/mreg/api/v1/tests/test_permissions.py index abb20466..f7b1ff72 100644 --- a/mreg/api/v1/tests/test_permissions.py +++ b/mreg/api/v1/tests/test_permissions.py @@ -50,6 +50,7 @@ def set_attr(name: str): user.configure_mock(**group_attrs) user.group_list = [] + user.username = "mockuser" return user @@ -87,9 +88,10 @@ def test_unhandled_view( self, mock_get_hostname_and_ips, mock_has_obj_perm, - mock_user_from_request + mock_user_from_request, ): user = get_mock_user() # Regular user + user.is_member_of_any.return_value = False request = get_mock_request(user, mock_user_from_request) # Mock view that is not an instance of any of the checked classes @@ -97,6 +99,7 @@ def test_unhandled_view( # Mock object that doesn't have 'host' attribute view.get_object = mock.Mock(return_value=None) + view.__class__.__name__ = "MockView" # Mock serializer with data that doesn't have 'host' or 'ipaddress' serializer = mock.Mock() diff --git a/mreg/models/network.py b/mreg/models/network.py index a8b46cd8..131b2414 100644 --- a/mreg/models/network.py +++ b/mreg/models/network.py @@ -74,6 +74,12 @@ def get_reserved_ipaddresses(self): ret.add(network.broadcast_address) return ret + def is_reserved_ipaddress(self, ip: str) -> bool: + """ + Check if the given IP address is reserved for this network. + """ + return any(ip == str(i) for i in self.get_reserved_ipaddresses()) + def get_excluded_ranges_start_end(self): excluded = [] for start_ip, end_ip in self.excluded_ranges.values_list("start_ip", "end_ip"): diff --git a/pyproject.toml b/pyproject.toml index 87a156c5..62a5809e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,8 @@ dependencies = [ "pyyaml", # For testing inside Docker image "unittest-parametrize", - "treetop-client>=0.0.1", - "prometheus-client>=0.20", + "treetop-client>=0.0.7", + "prometheus-client>=0.24", ] dynamic = ["version"] diff --git a/treetop/data/mreg.cedar b/treetop/data/mreg.cedar index 20d19ebe..33bc5fe8 100644 --- a/treetop/data/mreg.cedar +++ b/treetop/data/mreg.cedar @@ -3,11 +3,7 @@ @id("MREG.admins_policy") permit ( principal in MREG::Group::"admins", - action in - [MREG::Action::"create_host", - MREG::Action::"delete_host", - MREG::Action::"view_host", - MREG::Action::"edit_host"], + action in MREG::Action::"host_access", resource is Host ); @@ -16,34 +12,15 @@ permit ( @id("MREG.webadmins_policy") permit ( principal in MREG::Group::"webadmins", - action in - [MREG::Action::"edit_host", - MREG::Action::"delete_host", - MREG::Action::"create_host"], + action in MREG::Action::"host_access", resource is Host ) when { resource.nameLabels.contains("webserver") && - resource.ip.isInRange("192.168.1.0/24") + resource.ip.isInRange(ip("192.168.1.0/24")) }; -// Users can only view hosts -@id("MREG.users_policy") -permit ( - principal in MREG::Group::"users", - action == MREG::Action::"view_host", - resource is Host -); - -// Charlie does not get to delete hosts, no matter what. -@id("MREG.charlie_forbid_delete_host_policy") -forbid ( - principal == MREG::User::"charlie", - action == MREG::Action::"delete_host", - resource is Host -); - // Admins can manipulate any IP address, even if it is a gw, a broadcast address, // the network address, reserved. These three groups are unified as "restricted" IPs. @id("MREG.admins_ip_policy") @@ -58,6 +35,17 @@ permit ( resource is IPAddress ); + +/// Test group access, used during testing. +@id("MREG.test_group_policy") +permit ( + principal in MREG::Group::"testgroup", + action in [MREG::Action::"host_access"], + resource is Host +) when { + resource.ip.isInRange(ip("10.0.0.0/24")) +}; + /// Network Admins can manage any IP in any network @id("MREG.network_admins_ip_network_policy") permit ( @@ -75,8 +63,8 @@ permit ( ) when { - resource.ip.isInRange("192.168.1.0/24") || - resource.ip.isInRange("10.0.0.0/8") + resource.ip.isInRange(ip("192.168.1.0/24")) || + resource.ip.isInRange(ip("10.0.0.0/8")) }; /// Admins can do whatever with labels. @@ -92,7 +80,7 @@ permit ( ); /// Superadmins -@id("MREG.is_superuser") +@id("MREG.superadmin") permit ( principal in MREG::Group::"default-super-group", action, // Superadmins don't care what the action is @@ -100,34 +88,34 @@ permit ( ); /// Normal (?) admins -@id("MREG.is_admin") +@id("MREG.admin") permit ( principal in MREG::Group::"default-admin-group", - action == MREG::Action::"is_admin", + action == MREG::Action::"admin_access", resource ); /// Host Policy Admins -@id("MREG.is_hostpolicy_admin") +@id("MREG.hostpolicy_admin") permit ( principal in MREG::Group::"default-hostpolicyadmin-group", - action == MREG::Action::"is_hostpolicy_admin", + action == MREG::Action::"hostpolicy_admin_access", resource ); /// DNS Wildcard Admins -@id("MREG.is_dns_wildcard_admin") +@id("MREG.dns_wildcard_admin") permit ( principal in MREG::Group::"default-dns-wildcard-group", - action == MREG::Action::"is_dns_wildcard_admin", + action == MREG::Action::"dns_wildcard_admin_access", resource ); /// DNS Underscore Admins -@id("MREG.is_dns_underscore_admin") +@id("MREG.dns_underscore_admin") permit ( principal in MREG::Group::"default-dns-underscore-group", - action == MREG::Action::"is_dns_underscore_admin", + action == MREG::Action::"dns_underscore_admin_access", resource ); diff --git a/treetop/docker-compose.yml b/treetop/docker-compose.yml index fc7f817e..381e1fa6 100644 --- a/treetop/docker-compose.yml +++ b/treetop/docker-compose.yml @@ -10,6 +10,7 @@ services: treetop-server: image: ghcr.io/terjekv/treetop-rest:develop + pull_policy: "always" ports: - "9999:9999" environment: diff --git a/uv.lock b/uv.lock index 505c36d3..0614dfcd 100644 --- a/uv.lock +++ b/uv.lock @@ -1046,14 +1046,14 @@ wheels = [ [[package]] name = "treetop-client" -version = "0.0.2" +version = "0.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fa/a1/ae06eb362cdc44ccf2bea1c2af6856e038cebac63ceb4908434482e1323a/treetop_client-0.0.2.tar.gz", hash = "sha256:c89794d09fc9c31d39cb0e13ee03f1559bbe2ec893a0647698f48747a1759f8f", size = 5103, upload_time = "2025-07-28T19:34:36.677Z" } +sdist = { url = "https://files.pythonhosted.org/packages/87/2e/1920bff4d2f621cbab9d97a54eff9bc71a3a4c9eaabf16a4164a56ac7565/treetop_client-0.0.3.tar.gz", hash = "sha256:863b3a7f01e794e03674b4c7e011655c753432621c71e2f909ab966f4a7b5dda", size = 5409, upload_time = "2025-09-01T10:35:26.513Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/e4/9587921902ece105f616310b66cae9e4f40e38c8673d5e74ff2a66b61d57/treetop_client-0.0.2-py3-none-any.whl", hash = "sha256:af08d71d9d74a3913c7655fe3a5e1dc579e335ddecf562710ca2443b5227d16a", size = 6108, upload_time = "2025-07-28T19:34:35.405Z" }, + { url = "https://files.pythonhosted.org/packages/00/c6/77e4dd3519bab99832d7b45869d57230f1028352c5a6942b7c33484b83e3/treetop_client-0.0.3-py3-none-any.whl", hash = "sha256:cf294f607401755ef02813a26cb8228fdba06835d1081ee43412e53fe4a8a21c", size = 6443, upload_time = "2025-09-01T10:35:25.446Z" }, ] [[package]] From 6586313200de4a4f841f0dd7887843b93ee593fd Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Thu, 8 Jan 2026 09:15:44 +0100 Subject: [PATCH 04/34] Update towards master. --- docs/parity_testing.md | 99 +++ mreg/api/permissions.py | 72 +- mreg/api/test_utils.py | 56 ++ mreg/api/treetop.py | 57 +- .../v1/tests/test_parity_disable_example.py | 171 ++++ pyproject.toml | 2 +- tox.ini | 4 +- uv.lock | 797 ++++++++---------- 8 files changed, 780 insertions(+), 478 deletions(-) create mode 100644 docs/parity_testing.md create mode 100644 mreg/api/test_utils.py create mode 100644 mreg/api/v1/tests/test_parity_disable_example.py diff --git a/docs/parity_testing.md b/docs/parity_testing.md new file mode 100644 index 00000000..65589381 --- /dev/null +++ b/docs/parity_testing.md @@ -0,0 +1,99 @@ +# Disabling Parity Checking in Tests + +## Problem + +Tests that modify permissions or group memberships mid-test cause the legacy permission system and the TreeTop policy engine to be out of sync. Since TreeTop's policy content is immutable, these tests cannot maintain parity between the two systems. + +## Solutions + +### Option 1: Context Manager (Recommended for individual test sections) + +Use the `disable_policy_parity()` context manager to temporarily disable parity checking: + +```python +from mreg.api.treetop import disable_policy_parity + +class TestPermissions(MregAPITestCase): + def test_permission_change(self): + # Normal parity checking is active here + self.client.get('/api/v1/hosts/') + + # Disable parity checking for permission modifications + with disable_policy_parity(): + # Add user to a group + user.groups.add(some_group) + + # Make API calls - parity checking is skipped + response = self.client.post('/api/v1/hosts/', data) + self.assertEqual(response.status_code, 201) + + # Parity checking resumes after the context exits +``` + +### Option 2: Test Class Mixin (Recommended for entire test classes) + +Use the `PermissionModifyingTestCase` mixin for test classes that modify permissions throughout: + +```python +from mreg.api.test_utils import PermissionModifyingTestCase + +class TestGroupPermissions(PermissionModifyingTestCase, MregAPITestCase): + """All tests in this class have parity checking disabled.""" + + def test_add_group(self): + # Parity checking is disabled for all tests in this class + user.groups.add(admin_group) + response = self.client.post('/api/v1/hosts/', data) + self.assertEqual(response.status_code, 201) + + def test_remove_group(self): + # Still disabled here + user.groups.remove(admin_group) + response = self.client.post('/api/v1/hosts/', data) + self.assertEqual(response.status_code, 403) +``` + +### Option 3: Pytest Fixture (For pytest-style tests) + +Use the `no_parity_check` fixture: + +```python +def test_permission_modifications(no_parity_check): + """This test has parity checking disabled.""" + user.groups.add(some_group) + # Make API calls without parity checking +``` + +### Option 4: Pytest Marker (Documentation only) + +Mark tests that modify permissions for documentation purposes: + +```python +@pytest.mark.modifies_permissions +def test_permission_changes(): + """This marker documents that this test modifies permissions.""" + with disable_policy_parity(): + user.groups.add(some_group) + # Test code +``` + +## When to Use + +Disable parity checking when your test: + +- Adds or removes users from groups +- Changes NetGroupRegexPermission entries +- Modifies any permission-related database state +- Tests permission escalation/de-escalation scenarios + +## When NOT to Use + +Do NOT disable parity checking for: + +- Tests that only read data +- Tests that modify non-permission data (hosts, networks, etc.) +- Tests where both legacy and policy systems should agree + +## Implementation Details + +The `disable_policy_parity()` context manager uses thread-local storage to safely disable parity checking for the current thread only, ensuring test isolation in parallel test execution. diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index 20a46f82..a2be74cd 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -31,9 +31,14 @@ class ParityMixin: - """Small helpers to reduce repetition around policy_parity.""" + """Small helpers to reduce repetition around policy_parity. - def pp( + The public pp() method logs every call. For cases where multiple checks + feed into a single decision (pp_any, pp_all), use _pp() internally to avoid + nested logging and only log the final result. + """ + + def _pp( self, *, decision: bool, @@ -43,7 +48,12 @@ def pp( resource_kind: str = "Generic", resource_id: str = "any", resource_attrs: Optional[Mapping[str, str]] = None, + log: bool = True, ) -> bool: + """Internal policy parity check. Set log=False to skip logging.""" + if not log: + # For internal use: return decision without calling policy_parity + return decision return policy_parity( decision, request=request, @@ -55,6 +65,28 @@ def pp( resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, ) + def pp( + self, + *, + decision: bool, + action: str, + request: Request, + view: "GenericAPIView", + resource_kind: str = "Generic", + resource_id: str = "any", + resource_attrs: Optional[Mapping[str, str]] = None, + ) -> bool: + return self._pp( + decision=decision, + action=action, + request=request, + view=view, + resource_kind=resource_kind, + resource_id=resource_id, + resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + log=True, + ) + def pp_host( self, *, @@ -93,14 +125,16 @@ def pp_any( resource_kind: str = "Generic", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: + # Use internal _pp with log=False to avoid nested logging for each check for decision, action in checks: - if self.pp( + if self._pp( decision=decision, action=action, request=request, view=view, resource_kind=resource_kind, resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + log=False, ): return True return False @@ -114,19 +148,30 @@ def pp_all( resource_kind: str = "Generic", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: + # Use internal _pp with log=False to avoid nested logging for each check for decision, action in checks: - if not self.pp( + if not self._pp( decision=decision, action=action, request=request, view=view, resource_kind=resource_kind, resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + log=False, ): return False return True - def pp_generic_action(self, attrs: Mapping[str, str], decision: bool, action: str, request: Request, view: GenericAPIView, kind: str = "Generic", id: str = "Any") -> bool: + def pp_generic_action( + self, + attrs: Mapping[str, str], + decision: bool, + action: str, + request: Request, + view: GenericAPIView, + kind: str = "Generic", + id: str = "Any" + ) -> bool: return self.pp( decision=decision, action=action, @@ -137,7 +182,13 @@ def pp_generic_action(self, attrs: Mapping[str, str], decision: bool, action: st resource_attrs={ kind: kind, **attrs } ) - def user_has_permission(self, membership: MregAdminGroup, request: Request, view: GenericAPIView, exclude_superuser: bool = False) -> bool: + def user_has_permission( + self, + membership: MregAdminGroup, + request: Request, + view: GenericAPIView, + exclude_superuser: bool = False + ) -> bool: """ Check if the user has a given generic permission level. """ @@ -511,7 +562,14 @@ def has_destroy_permission(self, request, view, validated_serializer): return False object_type = obj.__class__.__name__.lower() - if self.pp_generic_action(decision=user.is_mreg_admin, action="destroy", kind=object_type, attrs={"id": str(obj)}, request=request, view=view): + if self.pp_generic_action( + decision=user.is_mreg_admin, + action="destroy", + kind=object_type, + attrs={"id": str(obj)}, + request=request, + view=view + ): return True return self.has_obj_perm(user, obj, request=request, view=view) diff --git a/mreg/api/test_utils.py b/mreg/api/test_utils.py new file mode 100644 index 00000000..7c16ba31 --- /dev/null +++ b/mreg/api/test_utils.py @@ -0,0 +1,56 @@ +"""Test utilities for parity checking and permission management.""" + +import pytest +from mreg.api.treetop import disable_policy_parity + + +# Pytest marker for tests that modify permissions +def pytest_configure(config): + """Register custom pytest markers.""" + config.addinivalue_line( + "markers", + "modifies_permissions: mark test as modifying permissions (will skip parity checking)" + ) + + +class PermissionModifyingTestCase: + """Mixin for test classes that modify permissions during tests. + + This mixin automatically disables parity checking for all tests in the class + since modifying permissions mid-test would cause the legacy and policy + systems to be out of sync. + + Usage: + class TestSomePermissions(PermissionModifyingTestCase, TestCase): + def test_something(self): + # This test can safely modify permissions + user.groups.add(some_group) + # Parity checking will be skipped + """ + + def setUp(self) -> None: + """Set up test with parity checking disabled.""" + self._parity_context = disable_policy_parity() + self._parity_context.__enter__() + if hasattr(super(), "setUp"): + super().setUp() # type: ignore[misc] + + def tearDown(self) -> None: + """Clean up parity checking context.""" + self._parity_context.__exit__(None, None, None) + if hasattr(super(), "tearDown"): + super().tearDown() # type: ignore[misc] + + +@pytest.fixture +def no_parity_check(): + """Pytest fixture to disable parity checking for a test. + + Usage: + def test_modify_permissions(no_parity_check): + # Parity checking is disabled in this test + user.groups.add(some_group) + # Make API calls + """ + with disable_policy_parity(): + yield diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index 5ee7f2fa..77f87088 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -3,6 +3,8 @@ from typing import Any, Optional, Mapping import ipaddress import json +import threading +from contextlib import contextmanager from django.conf import settings from rest_framework.request import Request @@ -15,6 +17,9 @@ logger = logging.getLogger("mreg.policy.parity") +# Thread-local storage for parity checking bypass flag +_thread_local = threading.local() + # Configure these in settings.py POLICY_PARITY_ENABLED = getattr(settings, "POLICY_PARITY_ENABLED", True) POLICY_BASE_URL = getattr(settings, "POLICY_BASE_URL", "http://localhost:9999") @@ -28,6 +33,34 @@ treetopclient = TreeTopClient(base_url=POLICY_BASE_URL) +@contextmanager +def disable_policy_parity(): + """Context manager to temporarily disable policy parity checking. + + Useful for tests that modify permissions/state mid-test, which would + cause the legacy and policy systems to be out of sync. + + Example: + def test_permission_changes(self): + with disable_policy_parity(): + # Modify permissions here + user.groups.add(some_group) + # Make API calls - parity checking will be skipped + """ + old_value = getattr(_thread_local, "skip_parity", False) + _thread_local.skip_parity = True + try: + yield + finally: + _thread_local.skip_parity = old_value + +def _is_parity_enabled() -> bool: + """Check if parity checking should be performed in current context.""" + if not POLICY_PARITY_ENABLED: + return False + # Skip parity checking if we're in a disabled context + return not getattr(_thread_local, "skip_parity", False) + def _corr_id(request: Request) -> Optional[str]: return request.headers.get("X-Correlation-ID") or request.META.get("HTTP_X_CORRELATION_ID") @@ -54,12 +87,12 @@ def policy_parity( Log legacy-vs-policy parity and return `decision` unchanged. Use this anywhere you currently 'return True/False'. """ - if not POLICY_PARITY_ENABLED: + if not _is_parity_enabled(): return decision # Build policy request muser = MregUser.from_request(request) - principal = TreeTopUser.new(muser.username, POLICY_NAMESPACE, groups=list(muser.group_list)) + principal = TreeTopUser.new(str(muser.username), POLICY_NAMESPACE, groups=list(muser.group_list)) pol_action = Action.new(action, POLICY_NAMESPACE) attrs = {} @@ -77,7 +110,7 @@ def policy_parity( # else: # attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) - res = Resource.new(resource_kind, resource_id, attrs=attrs) + res = Resource.new(str(resource_kind), resource_id, attrs=attrs) if len(pol_action.id.namespace) > 0: fully_qualified_action = "::".join(pol_action.id.namespace) + f"::{pol_action.id.id}" @@ -104,6 +137,18 @@ def policy_parity( pol_allowed = bool(resp.is_allowed()) except Exception as exc: error = repr(exc) + # Log policy server errors prominently + logger.error( + f"Policy server error: {type(exc).__name__}: {exc}", + extra={ + "error_type": type(exc).__name__, + "error_msg": str(exc), + "path": request.path, + "correlation_id": _corr_id(request), + }, + ) + # If policy server fails, we cannot determine parity. Return legacy decision + # but flag this in the payload for monitoring. parity = False if bool(decision) and pol_allowed: @@ -111,7 +156,7 @@ def policy_parity( elif not bool(decision) and not pol_allowed: parity = True - payload: dict[str, object] = { + payload: dict[str, Any] = { "parity": parity, "legacy_decision": bool(decision), "policy_decision": pol_allowed, @@ -120,10 +165,10 @@ def policy_parity( } if parity: - logger.warning("policy_parity_mismatch", extra=payload) + logger.info("policy_parity_ok", extra=payload) log_policy_parity(payload) else: - logger.info("policy_parity_ok", extra=payload) + logger.warning("policy_parity_mismatch", extra=payload) log_policy_parity(payload) return decision diff --git a/mreg/api/v1/tests/test_parity_disable_example.py b/mreg/api/v1/tests/test_parity_disable_example.py new file mode 100644 index 00000000..084ec608 --- /dev/null +++ b/mreg/api/v1/tests/test_parity_disable_example.py @@ -0,0 +1,171 @@ +"""Example tests demonstrating how to disable parity checking for permission-modifying tests. + +This file serves as documentation and can be used as a template. + +Note: This file contains example code and is not meant to be run as actual tests. +Type checking is disabled for simplicity. +""" +# type: ignore + +from django.contrib.auth.models import Group + +from mreg.api.v1.tests.tests import MregAPITestCase +from mreg.api.treetop import disable_policy_parity +from mreg.api.test_utils import PermissionModifyingTestCase +from mreg.models.network import NetGroupRegexPermission + + +class ExamplePermissionTestWithContextManager(MregAPITestCase): + """Example: Using context manager to disable parity checking for specific test sections.""" + + def test_user_gains_permission_mid_test(self): + """Test that a user gains access when added to a group. + + This test modifies permissions mid-test, so we disable parity checking + during the modification and subsequent API calls. + """ + # Create a group and permission + group = Group.objects.create(name='example_group') + NetGroupRegexPermission.objects.create( + group='example_group', + range='10.0.0.0/24', + regex=r'.*\.example\.org$' + ) + + # Get a non-privileged user client + client = self.get_token_client(superuser=False, adminuser=False) + + # First, verify user cannot create host (should fail) + # This is still subject to parity checking (no modifications yet) + response = client.post('/api/v1/hosts/', { + 'name': 'test.example.org', + 'ipaddress': '10.0.0.1' + }) + self.assertEqual(response.status_code, 403) + + # Now we're going to modify permissions, so disable parity checking + with disable_policy_parity(): + # Add user to the permission group + self.user.groups.add(group) + + # Now the user should have permission + # (parity checking is disabled because legacy and policy are out of sync) + response = client.post('/api/v1/hosts/', { + 'name': 'test2.example.org', + 'ipaddress': '10.0.0.2' + }) + self.assertEqual(response.status_code, 201) + + # Clean up + host_url = response['Location'] + client.delete(host_url) + + # Parity checking resumes after the context exits + # (though we typically don't make more permission-sensitive calls after this) + + +class ExamplePermissionTestWithMixin(PermissionModifyingTestCase, MregAPITestCase): + """Example: Using mixin to disable parity checking for entire test class. + + Use this approach when ALL tests in a class modify permissions. + """ + + def test_add_user_to_group(self): + """All tests in this class have parity checking disabled automatically.""" + group = Group.objects.create(name='test_group') + + # Parity checking is already disabled by the mixin + self.user.groups.add(group) + + # Make API calls without worrying about parity + response = self.client.get('/api/v1/hosts/') + self.assertEqual(response.status_code, 200) + + def test_remove_user_from_group(self): + """Another test - still no parity checking.""" + group = Group.objects.create(name='test_group') + self.user.groups.add(group) + + # Remove from group + self.user.groups.remove(group) + + # Make API calls + response = self.client.get('/api/v1/hosts/') + self.assertEqual(response.status_code, 200) + + +class ExampleComplexPermissionTest(MregAPITestCase): + """Example: Complex test with multiple permission changes.""" + + def test_permission_escalation_and_deescalation(self): + """Test user gaining and losing permissions multiple times. + + This shows how to use multiple context managers in sequence. + """ + # Create multiple groups with different permissions + group1 = Group.objects.create(name='group1') + group2 = Group.objects.create(name='group2') + + NetGroupRegexPermission.objects.create( + group='group1', + range='10.0.0.0/24', + regex=r'.*\.example\.org$' + ) + NetGroupRegexPermission.objects.create( + group='group2', + range='10.0.1.0/24', + regex=r'.*\.example\.com$' + ) + + client = self.get_token_client(superuser=False, adminuser=False) + + # User starts with no permissions + response = client.post('/api/v1/hosts/', { + 'name': 'test.example.org', + 'ipaddress': '10.0.0.1' + }) + self.assertEqual(response.status_code, 403) + + # Gain permission to .org domain + with disable_policy_parity(): + self.user.groups.add(group1) + + response = client.post('/api/v1/hosts/', { + 'name': 'test.example.org', + 'ipaddress': '10.0.0.2' + }) + self.assertEqual(response.status_code, 201) + client.delete(response['Location']) + + # Switch to different permission group + with disable_policy_parity(): + self.user.groups.remove(group1) + self.user.groups.add(group2) + + # Should now have access to .com but not .org + response = client.post('/api/v1/hosts/', { + 'name': 'test.example.com', + 'ipaddress': '10.0.1.1' + }) + self.assertEqual(response.status_code, 201) + client.delete(response['Location']) + + response = client.post('/api/v1/hosts/', { + 'name': 'test.example.org', + 'ipaddress': '10.0.0.3' + }) + self.assertEqual(response.status_code, 403) + + +# Note: You can also use pytest markers for documentation purposes: +# +# import pytest +# +# @pytest.mark.modifies_permissions +# class TestWithMarker(MregAPITestCase): +# """Tests marked for documentation that they modify permissions.""" +# +# def test_something(self): +# with disable_policy_parity(): +# # modify permissions +# pass diff --git a/pyproject.toml b/pyproject.toml index 62a5809e..9de8dd5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires-python = ">=3.12" dependencies = [ "Django>=5.2", "djangorestframework>=3.16.0,<3.17", - "django-auth-ldap>=5.2.0", + "django-auth-ldap>=5.3.0", "django-logging-json>=1.15", "django-netfields>=1.3.2", "django-filter>=25", diff --git a/tox.ini b/tox.ini index 49191123..88678c22 100644 --- a/tox.ini +++ b/tox.ini @@ -4,14 +4,13 @@ skip_missing_interpreters = true envlist = lint coverage - python{311,312,313}-django52 + python{312,313}-django52 python{312,313,314}-django60 toxworkdir = {env:TOX_WORKDIR:.tox} [gh-actions] python = - 3.11: python311 3.12: python312 3.13: python313 3.14: python314 @@ -31,7 +30,6 @@ deps = django52: Django>=5.2,<5.3 django60: Django>=6.0,<6.1 basepython = - python311: python3.11 python312: python3.12 python313: python3.13 python314: python3.14 diff --git a/uv.lock b/uv.lock index 0614dfcd..445c38a4 100644 --- a/uv.lock +++ b/uv.lock @@ -1,65 +1,45 @@ version = 1 revision = 3 -requires-python = ">=3.10" -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version < '3.12'", -] +requires-python = ">=3.12" [[package]] name = "anyio" -version = "4.10.0" +version = "4.12.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.12'", -] dependencies = [ { name = "idna" }, - { name = "sniffio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/b4/636b3b65173d3ce9a38ef5f0522789614e590dab6a8d505340a4efe4c567/anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6", size = 213252, upload_time = "2025-08-04T08:54:26.451Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload_time = "2025-08-04T08:54:24.882Z" }, -] - -[[package]] -name = "asgiref" -version = "3.8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/29/38/b3395cc9ad1b56d2ddac9970bc8f4141312dbaec28bc7c218b0dfafd0f42/asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590", size = 35186, upload_time = "2024-03-22T14:39:36.863Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/e3/893e8757be2612e6c266d9bb58ad2e3651524b5b40cf56761e985a28b13e/asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47", size = 23828, upload-time = "2024-03-22T14:39:34.521Z" }, + { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, ] [[package]] name = "asgiref" -version = "3.11.0" +version = "3.11.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", -] -sdist = { url = "https://files.pythonhosted.org/packages/76/b9/4db2509eabd14b4a8c71d1b24c8d5734c52b8560a7b1e1a8b56c8d25568b/asgiref-3.11.0.tar.gz", hash = "sha256:13acff32519542a1736223fb79a715acdebe24286d98e8b164a73085f40da2c4", size = 37969, upload-time = "2025-11-19T15:32:20.106Z" } +sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.33Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/91/be/317c2c55b8bbec407257d45f5c8d1b6867abc76d12043f2d3d58c538a4ea/asgiref-3.11.0-py3-none-any.whl", hash = "sha256:1db9021efadb0d9512ce8ffaf72fcef601c7b73a8807a1bb2ef143dc6b14846d", size = 24096, upload-time = "2025-11-19T15:32:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039Z" }, ] [[package]] name = "cachetools" -version = "6.2.4" +version = "7.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bc/1d/ede8680603f6016887c062a2cf4fc8fdba905866a3ab8831aa8aa651320c/cachetools-6.2.4.tar.gz", hash = "sha256:82c5c05585e70b6ba2d3ae09ea60b79548872185d2f24ae1f2709d37299fd607", size = 31731, upload-time = "2025-12-15T18:24:53.744Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/07/56595285564e90777d758ebd383d6b0b971b87729bbe2184a849932a3736/cachetools-7.0.1.tar.gz", hash = "sha256:e31e579d2c5b6e2944177a0397150d312888ddf4e16e12f1016068f0c03b8341", size = 36126, upload-time = "2026-02-10T22:24:05.03Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/fc/1d7b80d0eb7b714984ce40efc78859c022cd930e402f599d8ca9e39c78a4/cachetools-6.2.4-py3-none-any.whl", hash = "sha256:69a7a52634fed8b8bf6e24a050fb60bff1c9bd8f6d24572b99c32d4e71e62a51", size = 11551, upload-time = "2025-12-15T18:24:52.332Z" }, + { url = "https://files.pythonhosted.org/packages/ed/9e/5faefbf9db1db466d633735faceda1f94aa99ce506ac450d232536266b32/cachetools-7.0.1-py3-none-any.whl", hash = "sha256:8f086515c254d5664ae2146d14fc7f65c9a4bce75152eb247e5a9c5e6d7b2ecf", size = 13484, upload-time = "2026-02-10T22:24:03.741Z" }, ] [[package]] name = "certifi" -version = "2024.8.30" +version = "2026.1.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/ee/9b19140fe824b367c04c5e1b369942dd754c4c5462d5674002f75c4dedc1/certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9", size = 168507, upload-time = "2024-08-30T01:55:04.365Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", size = 167321, upload-time = "2024-08-30T01:55:02.591Z" }, + { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, ] [[package]] @@ -73,71 +53,59 @@ wheels = [ [[package]] name = "charset-normalizer" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f2/4f/e1808dc01273379acc506d18f1504eb2d299bd4131743b9fc54d7be4df1e/charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", size = 106620, upload-time = "2024-10-09T07:40:20.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/8b/825cc84cf13a28bfbcba7c416ec22bf85a9584971be15b21dd8300c65b7f/charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6", size = 196363, upload-time = "2024-10-09T07:38:02.622Z" }, - { url = "https://files.pythonhosted.org/packages/23/81/d7eef6a99e42c77f444fdd7bc894b0ceca6c3a95c51239e74a722039521c/charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b", size = 125639, upload-time = "2024-10-09T07:38:04.044Z" }, - { url = "https://files.pythonhosted.org/packages/21/67/b4564d81f48042f520c948abac7079356e94b30cb8ffb22e747532cf469d/charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99", size = 120451, upload-time = "2024-10-09T07:38:04.997Z" }, - { url = "https://files.pythonhosted.org/packages/c2/72/12a7f0943dd71fb5b4e7b55c41327ac0a1663046a868ee4d0d8e9c369b85/charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca", size = 140041, upload-time = "2024-10-09T07:38:06.676Z" }, - { url = "https://files.pythonhosted.org/packages/67/56/fa28c2c3e31217c4c52158537a2cf5d98a6c1e89d31faf476c89391cd16b/charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d", size = 150333, upload-time = "2024-10-09T07:38:08.626Z" }, - { url = "https://files.pythonhosted.org/packages/f9/d2/466a9be1f32d89eb1554cf84073a5ed9262047acee1ab39cbaefc19635d2/charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7", size = 142921, upload-time = "2024-10-09T07:38:10.301Z" }, - { url = "https://files.pythonhosted.org/packages/f8/01/344ec40cf5d85c1da3c1f57566c59e0c9b56bcc5566c08804a95a6cc8257/charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3", size = 144785, upload-time = "2024-10-09T07:38:12.019Z" }, - { url = "https://files.pythonhosted.org/packages/73/8b/2102692cb6d7e9f03b9a33a710e0164cadfce312872e3efc7cfe22ed26b4/charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907", size = 146631, upload-time = "2024-10-09T07:38:13.701Z" }, - { url = "https://files.pythonhosted.org/packages/d8/96/cc2c1b5d994119ce9f088a9a0c3ebd489d360a2eb058e2c8049f27092847/charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b", size = 140867, upload-time = "2024-10-09T07:38:15.403Z" }, - { url = "https://files.pythonhosted.org/packages/c9/27/cde291783715b8ec30a61c810d0120411844bc4c23b50189b81188b273db/charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912", size = 149273, upload-time = "2024-10-09T07:38:16.433Z" }, - { url = "https://files.pythonhosted.org/packages/3a/a4/8633b0fc1a2d1834d5393dafecce4a1cc56727bfd82b4dc18fc92f0d3cc3/charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95", size = 152437, upload-time = "2024-10-09T07:38:18.013Z" }, - { url = "https://files.pythonhosted.org/packages/64/ea/69af161062166b5975ccbb0961fd2384853190c70786f288684490913bf5/charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e", size = 150087, upload-time = "2024-10-09T07:38:19.089Z" }, - { url = "https://files.pythonhosted.org/packages/3b/fd/e60a9d9fd967f4ad5a92810138192f825d77b4fa2a557990fd575a47695b/charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe", size = 145142, upload-time = "2024-10-09T07:38:20.78Z" }, - { url = "https://files.pythonhosted.org/packages/6d/02/8cb0988a1e49ac9ce2eed1e07b77ff118f2923e9ebd0ede41ba85f2dcb04/charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc", size = 94701, upload-time = "2024-10-09T07:38:21.851Z" }, - { url = "https://files.pythonhosted.org/packages/d6/20/f1d4670a8a723c46be695dff449d86d6092916f9e99c53051954ee33a1bc/charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749", size = 102191, upload-time = "2024-10-09T07:38:23.467Z" }, - { url = "https://files.pythonhosted.org/packages/9c/61/73589dcc7a719582bf56aae309b6103d2762b526bffe189d635a7fcfd998/charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", size = 193339, upload-time = "2024-10-09T07:38:24.527Z" }, - { url = "https://files.pythonhosted.org/packages/77/d5/8c982d58144de49f59571f940e329ad6e8615e1e82ef84584c5eeb5e1d72/charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", size = 124366, upload-time = "2024-10-09T07:38:26.488Z" }, - { url = "https://files.pythonhosted.org/packages/bf/19/411a64f01ee971bed3231111b69eb56f9331a769072de479eae7de52296d/charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", size = 118874, upload-time = "2024-10-09T07:38:28.115Z" }, - { url = "https://files.pythonhosted.org/packages/4c/92/97509850f0d00e9f14a46bc751daabd0ad7765cff29cdfb66c68b6dad57f/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", size = 138243, upload-time = "2024-10-09T07:38:29.822Z" }, - { url = "https://files.pythonhosted.org/packages/e2/29/d227805bff72ed6d6cb1ce08eec707f7cfbd9868044893617eb331f16295/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6", size = 148676, upload-time = "2024-10-09T07:38:30.869Z" }, - { url = "https://files.pythonhosted.org/packages/13/bc/87c2c9f2c144bedfa62f894c3007cd4530ba4b5351acb10dc786428a50f0/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", size = 141289, upload-time = "2024-10-09T07:38:32.557Z" }, - { url = "https://files.pythonhosted.org/packages/eb/5b/6f10bad0f6461fa272bfbbdf5d0023b5fb9bc6217c92bf068fa5a99820f5/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", size = 142585, upload-time = "2024-10-09T07:38:33.649Z" }, - { url = "https://files.pythonhosted.org/packages/3b/a0/a68980ab8a1f45a36d9745d35049c1af57d27255eff8c907e3add84cf68f/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5", size = 144408, upload-time = "2024-10-09T07:38:34.687Z" }, - { url = "https://files.pythonhosted.org/packages/d7/a1/493919799446464ed0299c8eef3c3fad0daf1c3cd48bff9263c731b0d9e2/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", size = 139076, upload-time = "2024-10-09T07:38:36.417Z" }, - { url = "https://files.pythonhosted.org/packages/fb/9d/9c13753a5a6e0db4a0a6edb1cef7aee39859177b64e1a1e748a6e3ba62c2/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c", size = 146874, upload-time = "2024-10-09T07:38:37.59Z" }, - { url = "https://files.pythonhosted.org/packages/75/d2/0ab54463d3410709c09266dfb416d032a08f97fd7d60e94b8c6ef54ae14b/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365", size = 150871, upload-time = "2024-10-09T07:38:38.666Z" }, - { url = "https://files.pythonhosted.org/packages/8d/c9/27e41d481557be53d51e60750b85aa40eaf52b841946b3cdeff363105737/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", size = 148546, upload-time = "2024-10-09T07:38:40.459Z" }, - { url = "https://files.pythonhosted.org/packages/ee/44/4f62042ca8cdc0cabf87c0fc00ae27cd8b53ab68be3605ba6d071f742ad3/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", size = 143048, upload-time = "2024-10-09T07:38:42.178Z" }, - { url = "https://files.pythonhosted.org/packages/01/f8/38842422988b795220eb8038745d27a675ce066e2ada79516c118f291f07/charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99", size = 94389, upload-time = "2024-10-09T07:38:43.339Z" }, - { url = "https://files.pythonhosted.org/packages/0b/6e/b13bd47fa9023b3699e94abf565b5a2f0b0be6e9ddac9812182596ee62e4/charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27", size = 101752, upload-time = "2024-10-09T07:38:44.276Z" }, - { url = "https://files.pythonhosted.org/packages/d3/0b/4b7a70987abf9b8196845806198975b6aab4ce016632f817ad758a5aa056/charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6", size = 194445, upload-time = "2024-10-09T07:38:45.275Z" }, - { url = "https://files.pythonhosted.org/packages/50/89/354cc56cf4dd2449715bc9a0f54f3aef3dc700d2d62d1fa5bbea53b13426/charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf", size = 125275, upload-time = "2024-10-09T07:38:46.449Z" }, - { url = "https://files.pythonhosted.org/packages/fa/44/b730e2a2580110ced837ac083d8ad222343c96bb6b66e9e4e706e4d0b6df/charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db", size = 119020, upload-time = "2024-10-09T07:38:48.88Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e4/9263b8240ed9472a2ae7ddc3e516e71ef46617fe40eaa51221ccd4ad9a27/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1", size = 139128, upload-time = "2024-10-09T07:38:49.86Z" }, - { url = "https://files.pythonhosted.org/packages/6b/e3/9f73e779315a54334240353eaea75854a9a690f3f580e4bd85d977cb2204/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03", size = 149277, upload-time = "2024-10-09T07:38:52.306Z" }, - { url = "https://files.pythonhosted.org/packages/1a/cf/f1f50c2f295312edb8a548d3fa56a5c923b146cd3f24114d5adb7e7be558/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284", size = 142174, upload-time = "2024-10-09T07:38:53.458Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/92a76dc2ff3a12e69ba94e7e05168d37d0345fa08c87e1fe24d0c2a42223/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15", size = 143838, upload-time = "2024-10-09T07:38:54.691Z" }, - { url = "https://files.pythonhosted.org/packages/a4/01/2117ff2b1dfc61695daf2babe4a874bca328489afa85952440b59819e9d7/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8", size = 146149, upload-time = "2024-10-09T07:38:55.737Z" }, - { url = "https://files.pythonhosted.org/packages/f6/9b/93a332b8d25b347f6839ca0a61b7f0287b0930216994e8bf67a75d050255/charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2", size = 140043, upload-time = "2024-10-09T07:38:57.44Z" }, - { url = "https://files.pythonhosted.org/packages/ab/f6/7ac4a01adcdecbc7a7587767c776d53d369b8b971382b91211489535acf0/charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719", size = 148229, upload-time = "2024-10-09T07:38:58.782Z" }, - { url = "https://files.pythonhosted.org/packages/9d/be/5708ad18161dee7dc6a0f7e6cf3a88ea6279c3e8484844c0590e50e803ef/charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631", size = 151556, upload-time = "2024-10-09T07:39:00.467Z" }, - { url = "https://files.pythonhosted.org/packages/5a/bb/3d8bc22bacb9eb89785e83e6723f9888265f3a0de3b9ce724d66bd49884e/charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b", size = 149772, upload-time = "2024-10-09T07:39:01.5Z" }, - { url = "https://files.pythonhosted.org/packages/f7/fa/d3fc622de05a86f30beea5fc4e9ac46aead4731e73fd9055496732bcc0a4/charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565", size = 144800, upload-time = "2024-10-09T07:39:02.491Z" }, - { url = "https://files.pythonhosted.org/packages/9a/65/bdb9bc496d7d190d725e96816e20e2ae3a6fa42a5cac99c3c3d6ff884118/charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7", size = 94836, upload-time = "2024-10-09T07:39:04.607Z" }, - { url = "https://files.pythonhosted.org/packages/3e/67/7b72b69d25b89c0b3cea583ee372c43aa24df15f0e0f8d3982c57804984b/charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9", size = 102187, upload-time = "2024-10-09T07:39:06.247Z" }, - { url = "https://files.pythonhosted.org/packages/f3/89/68a4c86f1a0002810a27f12e9a7b22feb198c59b2f05231349fbce5c06f4/charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114", size = 194617, upload-time = "2024-10-09T07:39:07.317Z" }, - { url = "https://files.pythonhosted.org/packages/4f/cd/8947fe425e2ab0aa57aceb7807af13a0e4162cd21eee42ef5b053447edf5/charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed", size = 125310, upload-time = "2024-10-09T07:39:08.353Z" }, - { url = "https://files.pythonhosted.org/packages/5b/f0/b5263e8668a4ee9becc2b451ed909e9c27058337fda5b8c49588183c267a/charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250", size = 119126, upload-time = "2024-10-09T07:39:09.327Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6e/e445afe4f7fda27a533f3234b627b3e515a1b9429bc981c9a5e2aa5d97b6/charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920", size = 139342, upload-time = "2024-10-09T07:39:10.322Z" }, - { url = "https://files.pythonhosted.org/packages/a1/b2/4af9993b532d93270538ad4926c8e37dc29f2111c36f9c629840c57cd9b3/charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64", size = 149383, upload-time = "2024-10-09T07:39:12.042Z" }, - { url = "https://files.pythonhosted.org/packages/fb/6f/4e78c3b97686b871db9be6f31d64e9264e889f8c9d7ab33c771f847f79b7/charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23", size = 142214, upload-time = "2024-10-09T07:39:13.059Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c9/1c8fe3ce05d30c87eff498592c89015b19fade13df42850aafae09e94f35/charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc", size = 144104, upload-time = "2024-10-09T07:39:14.815Z" }, - { url = "https://files.pythonhosted.org/packages/ee/68/efad5dcb306bf37db7db338338e7bb8ebd8cf38ee5bbd5ceaaaa46f257e6/charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d", size = 146255, upload-time = "2024-10-09T07:39:15.868Z" }, - { url = "https://files.pythonhosted.org/packages/0c/75/1ed813c3ffd200b1f3e71121c95da3f79e6d2a96120163443b3ad1057505/charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88", size = 140251, upload-time = "2024-10-09T07:39:16.995Z" }, - { url = "https://files.pythonhosted.org/packages/7d/0d/6f32255c1979653b448d3c709583557a4d24ff97ac4f3a5be156b2e6a210/charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90", size = 148474, upload-time = "2024-10-09T07:39:18.021Z" }, - { url = "https://files.pythonhosted.org/packages/ac/a0/c1b5298de4670d997101fef95b97ac440e8c8d8b4efa5a4d1ef44af82f0d/charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b", size = 151849, upload-time = "2024-10-09T07:39:19.243Z" }, - { url = "https://files.pythonhosted.org/packages/04/4f/b3961ba0c664989ba63e30595a3ed0875d6790ff26671e2aae2fdc28a399/charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d", size = 149781, upload-time = "2024-10-09T07:39:20.397Z" }, - { url = "https://files.pythonhosted.org/packages/d8/90/6af4cd042066a4adad58ae25648a12c09c879efa4849c705719ba1b23d8c/charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482", size = 144970, upload-time = "2024-10-09T07:39:21.452Z" }, - { url = "https://files.pythonhosted.org/packages/cc/67/e5e7e0cbfefc4ca79025238b43cdf8a2037854195b37d6417f3d0895c4c2/charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67", size = 94973, upload-time = "2024-10-09T07:39:22.509Z" }, - { url = "https://files.pythonhosted.org/packages/65/97/fc9bbc54ee13d33dc54a7fcf17b26368b18505500fc01e228c27b5222d80/charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b", size = 102308, upload-time = "2024-10-09T07:39:23.524Z" }, - { url = "https://files.pythonhosted.org/packages/bf/9b/08c0432272d77b04803958a4598a51e2a4b51c06640af8b8f0f908c18bf2/charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", size = 49446, upload-time = "2024-10-09T07:40:19.383Z" }, +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, ] [[package]] @@ -151,137 +119,136 @@ wheels = [ [[package]] name = "coverage" -version = "7.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/08/7e37f82e4d1aead42a7443ff06a1e406aabf7302c4f00a546e4b320b994c/coverage-7.6.1.tar.gz", hash = "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d", size = 798791, upload-time = "2024-08-04T19:45:30.9Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/61/eb7ce5ed62bacf21beca4937a90fe32545c91a3c8a42a30c6616d48fc70d/coverage-7.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16", size = 206690, upload-time = "2024-08-04T19:43:07.695Z" }, - { url = "https://files.pythonhosted.org/packages/7d/73/041928e434442bd3afde5584bdc3f932fb4562b1597629f537387cec6f3d/coverage-7.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36", size = 207127, upload-time = "2024-08-04T19:43:10.15Z" }, - { url = "https://files.pythonhosted.org/packages/c7/c8/6ca52b5147828e45ad0242388477fdb90df2c6cbb9a441701a12b3c71bc8/coverage-7.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02", size = 235654, upload-time = "2024-08-04T19:43:12.405Z" }, - { url = "https://files.pythonhosted.org/packages/d5/da/9ac2b62557f4340270942011d6efeab9833648380109e897d48ab7c1035d/coverage-7.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc", size = 233598, upload-time = "2024-08-04T19:43:14.078Z" }, - { url = "https://files.pythonhosted.org/packages/53/23/9e2c114d0178abc42b6d8d5281f651a8e6519abfa0ef460a00a91f80879d/coverage-7.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23", size = 234732, upload-time = "2024-08-04T19:43:16.632Z" }, - { url = "https://files.pythonhosted.org/packages/0f/7e/a0230756fb133343a52716e8b855045f13342b70e48e8ad41d8a0d60ab98/coverage-7.6.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34", size = 233816, upload-time = "2024-08-04T19:43:19.049Z" }, - { url = "https://files.pythonhosted.org/packages/28/7c/3753c8b40d232b1e5eeaed798c875537cf3cb183fb5041017c1fdb7ec14e/coverage-7.6.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c", size = 232325, upload-time = "2024-08-04T19:43:21.246Z" }, - { url = "https://files.pythonhosted.org/packages/57/e3/818a2b2af5b7573b4b82cf3e9f137ab158c90ea750a8f053716a32f20f06/coverage-7.6.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959", size = 233418, upload-time = "2024-08-04T19:43:22.945Z" }, - { url = "https://files.pythonhosted.org/packages/c8/fb/4532b0b0cefb3f06d201648715e03b0feb822907edab3935112b61b885e2/coverage-7.6.1-cp310-cp310-win32.whl", hash = "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232", size = 209343, upload-time = "2024-08-04T19:43:25.121Z" }, - { url = "https://files.pythonhosted.org/packages/5a/25/af337cc7421eca1c187cc9c315f0a755d48e755d2853715bfe8c418a45fa/coverage-7.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0", size = 210136, upload-time = "2024-08-04T19:43:26.851Z" }, - { url = "https://files.pythonhosted.org/packages/ad/5f/67af7d60d7e8ce61a4e2ddcd1bd5fb787180c8d0ae0fbd073f903b3dd95d/coverage-7.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93", size = 206796, upload-time = "2024-08-04T19:43:29.115Z" }, - { url = "https://files.pythonhosted.org/packages/e1/0e/e52332389e057daa2e03be1fbfef25bb4d626b37d12ed42ae6281d0a274c/coverage-7.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3", size = 207244, upload-time = "2024-08-04T19:43:31.285Z" }, - { url = "https://files.pythonhosted.org/packages/aa/cd/766b45fb6e090f20f8927d9c7cb34237d41c73a939358bc881883fd3a40d/coverage-7.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff", size = 239279, upload-time = "2024-08-04T19:43:33.581Z" }, - { url = "https://files.pythonhosted.org/packages/70/6c/a9ccd6fe50ddaf13442a1e2dd519ca805cbe0f1fcd377fba6d8339b98ccb/coverage-7.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d", size = 236859, upload-time = "2024-08-04T19:43:35.301Z" }, - { url = "https://files.pythonhosted.org/packages/14/6f/8351b465febb4dbc1ca9929505202db909c5a635c6fdf33e089bbc3d7d85/coverage-7.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6", size = 238549, upload-time = "2024-08-04T19:43:37.578Z" }, - { url = "https://files.pythonhosted.org/packages/68/3c/289b81fa18ad72138e6d78c4c11a82b5378a312c0e467e2f6b495c260907/coverage-7.6.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56", size = 237477, upload-time = "2024-08-04T19:43:39.92Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1c/aa1efa6459d822bd72c4abc0b9418cf268de3f60eeccd65dc4988553bd8d/coverage-7.6.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234", size = 236134, upload-time = "2024-08-04T19:43:41.453Z" }, - { url = "https://files.pythonhosted.org/packages/fb/c8/521c698f2d2796565fe9c789c2ee1ccdae610b3aa20b9b2ef980cc253640/coverage-7.6.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133", size = 236910, upload-time = "2024-08-04T19:43:43.037Z" }, - { url = "https://files.pythonhosted.org/packages/7d/30/033e663399ff17dca90d793ee8a2ea2890e7fdf085da58d82468b4220bf7/coverage-7.6.1-cp311-cp311-win32.whl", hash = "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c", size = 209348, upload-time = "2024-08-04T19:43:44.787Z" }, - { url = "https://files.pythonhosted.org/packages/20/05/0d1ccbb52727ccdadaa3ff37e4d2dc1cd4d47f0c3df9eb58d9ec8508ca88/coverage-7.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6", size = 210230, upload-time = "2024-08-04T19:43:46.707Z" }, - { url = "https://files.pythonhosted.org/packages/7e/d4/300fc921dff243cd518c7db3a4c614b7e4b2431b0d1145c1e274fd99bd70/coverage-7.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778", size = 206983, upload-time = "2024-08-04T19:43:49.082Z" }, - { url = "https://files.pythonhosted.org/packages/e1/ab/6bf00de5327ecb8db205f9ae596885417a31535eeda6e7b99463108782e1/coverage-7.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391", size = 207221, upload-time = "2024-08-04T19:43:52.15Z" }, - { url = "https://files.pythonhosted.org/packages/92/8f/2ead05e735022d1a7f3a0a683ac7f737de14850395a826192f0288703472/coverage-7.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8", size = 240342, upload-time = "2024-08-04T19:43:53.746Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ef/94043e478201ffa85b8ae2d2c79b4081e5a1b73438aafafccf3e9bafb6b5/coverage-7.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d", size = 237371, upload-time = "2024-08-04T19:43:55.993Z" }, - { url = "https://files.pythonhosted.org/packages/1f/0f/c890339dd605f3ebc269543247bdd43b703cce6825b5ed42ff5f2d6122c7/coverage-7.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca", size = 239455, upload-time = "2024-08-04T19:43:57.618Z" }, - { url = "https://files.pythonhosted.org/packages/d1/04/7fd7b39ec7372a04efb0f70c70e35857a99b6a9188b5205efb4c77d6a57a/coverage-7.6.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163", size = 238924, upload-time = "2024-08-04T19:44:00.012Z" }, - { url = "https://files.pythonhosted.org/packages/ed/bf/73ce346a9d32a09cf369f14d2a06651329c984e106f5992c89579d25b27e/coverage-7.6.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a", size = 237252, upload-time = "2024-08-04T19:44:01.713Z" }, - { url = "https://files.pythonhosted.org/packages/86/74/1dc7a20969725e917b1e07fe71a955eb34bc606b938316bcc799f228374b/coverage-7.6.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d", size = 238897, upload-time = "2024-08-04T19:44:03.898Z" }, - { url = "https://files.pythonhosted.org/packages/b6/e9/d9cc3deceb361c491b81005c668578b0dfa51eed02cd081620e9a62f24ec/coverage-7.6.1-cp312-cp312-win32.whl", hash = "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5", size = 209606, upload-time = "2024-08-04T19:44:05.532Z" }, - { url = "https://files.pythonhosted.org/packages/47/c8/5a2e41922ea6740f77d555c4d47544acd7dc3f251fe14199c09c0f5958d3/coverage-7.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb", size = 210373, upload-time = "2024-08-04T19:44:07.079Z" }, - { url = "https://files.pythonhosted.org/packages/8c/f9/9aa4dfb751cb01c949c990d136a0f92027fbcc5781c6e921df1cb1563f20/coverage-7.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106", size = 207007, upload-time = "2024-08-04T19:44:09.453Z" }, - { url = "https://files.pythonhosted.org/packages/b9/67/e1413d5a8591622a46dd04ff80873b04c849268831ed5c304c16433e7e30/coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9", size = 207269, upload-time = "2024-08-04T19:44:11.045Z" }, - { url = "https://files.pythonhosted.org/packages/14/5b/9dec847b305e44a5634d0fb8498d135ab1d88330482b74065fcec0622224/coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c", size = 239886, upload-time = "2024-08-04T19:44:12.83Z" }, - { url = "https://files.pythonhosted.org/packages/7b/b7/35760a67c168e29f454928f51f970342d23cf75a2bb0323e0f07334c85f3/coverage-7.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a", size = 237037, upload-time = "2024-08-04T19:44:15.393Z" }, - { url = "https://files.pythonhosted.org/packages/f7/95/d2fd31f1d638df806cae59d7daea5abf2b15b5234016a5ebb502c2f3f7ee/coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060", size = 239038, upload-time = "2024-08-04T19:44:17.466Z" }, - { url = "https://files.pythonhosted.org/packages/6e/bd/110689ff5752b67924efd5e2aedf5190cbbe245fc81b8dec1abaffba619d/coverage-7.6.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862", size = 238690, upload-time = "2024-08-04T19:44:19.336Z" }, - { url = "https://files.pythonhosted.org/packages/d3/a8/08d7b38e6ff8df52331c83130d0ab92d9c9a8b5462f9e99c9f051a4ae206/coverage-7.6.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388", size = 236765, upload-time = "2024-08-04T19:44:20.994Z" }, - { url = "https://files.pythonhosted.org/packages/d6/6a/9cf96839d3147d55ae713eb2d877f4d777e7dc5ba2bce227167d0118dfe8/coverage-7.6.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155", size = 238611, upload-time = "2024-08-04T19:44:22.616Z" }, - { url = "https://files.pythonhosted.org/packages/74/e4/7ff20d6a0b59eeaab40b3140a71e38cf52547ba21dbcf1d79c5a32bba61b/coverage-7.6.1-cp313-cp313-win32.whl", hash = "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a", size = 209671, upload-time = "2024-08-04T19:44:24.418Z" }, - { url = "https://files.pythonhosted.org/packages/35/59/1812f08a85b57c9fdb6d0b383d779e47b6f643bc278ed682859512517e83/coverage-7.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129", size = 210368, upload-time = "2024-08-04T19:44:26.276Z" }, - { url = "https://files.pythonhosted.org/packages/9c/15/08913be1c59d7562a3e39fce20661a98c0a3f59d5754312899acc6cb8a2d/coverage-7.6.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e", size = 207758, upload-time = "2024-08-04T19:44:29.028Z" }, - { url = "https://files.pythonhosted.org/packages/c4/ae/b5d58dff26cade02ada6ca612a76447acd69dccdbb3a478e9e088eb3d4b9/coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962", size = 208035, upload-time = "2024-08-04T19:44:30.673Z" }, - { url = "https://files.pythonhosted.org/packages/b8/d7/62095e355ec0613b08dfb19206ce3033a0eedb6f4a67af5ed267a8800642/coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb", size = 250839, upload-time = "2024-08-04T19:44:32.412Z" }, - { url = "https://files.pythonhosted.org/packages/7c/1e/c2967cb7991b112ba3766df0d9c21de46b476d103e32bb401b1b2adf3380/coverage-7.6.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704", size = 246569, upload-time = "2024-08-04T19:44:34.547Z" }, - { url = "https://files.pythonhosted.org/packages/8b/61/a7a6a55dd266007ed3b1df7a3386a0d760d014542d72f7c2c6938483b7bd/coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b", size = 248927, upload-time = "2024-08-04T19:44:36.313Z" }, - { url = "https://files.pythonhosted.org/packages/c8/fa/13a6f56d72b429f56ef612eb3bc5ce1b75b7ee12864b3bd12526ab794847/coverage-7.6.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f", size = 248401, upload-time = "2024-08-04T19:44:38.155Z" }, - { url = "https://files.pythonhosted.org/packages/75/06/0429c652aa0fb761fc60e8c6b291338c9173c6aa0f4e40e1902345b42830/coverage-7.6.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223", size = 246301, upload-time = "2024-08-04T19:44:39.883Z" }, - { url = "https://files.pythonhosted.org/packages/52/76/1766bb8b803a88f93c3a2d07e30ffa359467810e5cbc68e375ebe6906efb/coverage-7.6.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3", size = 247598, upload-time = "2024-08-04T19:44:41.59Z" }, - { url = "https://files.pythonhosted.org/packages/66/8b/f54f8db2ae17188be9566e8166ac6df105c1c611e25da755738025708d54/coverage-7.6.1-cp313-cp313t-win32.whl", hash = "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f", size = 210307, upload-time = "2024-08-04T19:44:43.301Z" }, - { url = "https://files.pythonhosted.org/packages/9f/b0/e0dca6da9170aefc07515cce067b97178cefafb512d00a87a1c717d2efd5/coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657", size = 211453, upload-time = "2024-08-04T19:44:45.677Z" }, - { url = "https://files.pythonhosted.org/packages/a5/2b/0354ed096bca64dc8e32a7cbcae28b34cb5ad0b1fe2125d6d99583313ac0/coverage-7.6.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df", size = 198926, upload-time = "2024-08-04T19:45:28.875Z" }, -] - -[package.optional-dependencies] -toml = [ - { name = "tomli", marker = "python_full_version <= '3.11'" }, +version = "7.13.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/56/95b7e30fa389756cb56630faa728da46a27b8c6eb46f9d557c68fff12b65/coverage-7.13.4.tar.gz", hash = "sha256:e5c8f6ed1e61a8b2dcdf31eb0b9bbf0130750ca79c1c49eb898e2ad86f5ccc91", size = 827239, upload-time = "2026-02-09T12:59:03.86Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/81/4ce2fdd909c5a0ed1f6dedb88aa57ab79b6d1fbd9b588c1ac7ef45659566/coverage-7.13.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:02231499b08dabbe2b96612993e5fc34217cdae907a51b906ac7fca8027a4459", size = 219449, upload-time = "2026-02-09T12:56:54.889Z" }, + { url = "https://files.pythonhosted.org/packages/5d/96/5238b1efc5922ddbdc9b0db9243152c09777804fb7c02ad1741eb18a11c0/coverage-7.13.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40aa8808140e55dc022b15d8aa7f651b6b3d68b365ea0398f1441e0b04d859c3", size = 219810, upload-time = "2026-02-09T12:56:56.33Z" }, + { url = "https://files.pythonhosted.org/packages/78/72/2f372b726d433c9c35e56377cf1d513b4c16fe51841060d826b95caacec1/coverage-7.13.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5b856a8ccf749480024ff3bd7310adaef57bf31fd17e1bfc404b7940b6986634", size = 251308, upload-time = "2026-02-09T12:56:57.858Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a0/2ea570925524ef4e00bb6c82649f5682a77fac5ab910a65c9284de422600/coverage-7.13.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c048ea43875fbf8b45d476ad79f179809c590ec7b79e2035c662e7afa3192e3", size = 254052, upload-time = "2026-02-09T12:56:59.754Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ac/45dc2e19a1939098d783c846e130b8f862fbb50d09e0af663988f2f21973/coverage-7.13.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7b38448866e83176e28086674fe7368ab8590e4610fb662b44e345b86d63ffa", size = 255165, upload-time = "2026-02-09T12:57:01.287Z" }, + { url = "https://files.pythonhosted.org/packages/2d/4d/26d236ff35abc3b5e63540d3386e4c3b192168c1d96da5cb2f43c640970f/coverage-7.13.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:de6defc1c9badbf8b9e67ae90fd00519186d6ab64e5cc5f3d21359c2a9b2c1d3", size = 257432, upload-time = "2026-02-09T12:57:02.637Z" }, + { url = "https://files.pythonhosted.org/packages/ec/55/14a966c757d1348b2e19caf699415a2a4c4f7feaa4bbc6326a51f5c7dd1b/coverage-7.13.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7eda778067ad7ffccd23ecffce537dface96212576a07924cbf0d8799d2ded5a", size = 251716, upload-time = "2026-02-09T12:57:04.056Z" }, + { url = "https://files.pythonhosted.org/packages/77/33/50116647905837c66d28b2af1321b845d5f5d19be9655cb84d4a0ea806b4/coverage-7.13.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e87f6c587c3f34356c3759f0420693e35e7eb0e2e41e4c011cb6ec6ecbbf1db7", size = 253089, upload-time = "2026-02-09T12:57:05.503Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b4/8efb11a46e3665d92635a56e4f2d4529de6d33f2cb38afd47d779d15fc99/coverage-7.13.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8248977c2e33aecb2ced42fef99f2d319e9904a36e55a8a68b69207fb7e43edc", size = 251232, upload-time = "2026-02-09T12:57:06.879Z" }, + { url = "https://files.pythonhosted.org/packages/51/24/8cd73dd399b812cc76bb0ac260e671c4163093441847ffe058ac9fda1e32/coverage-7.13.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:25381386e80ae727608e662474db537d4df1ecd42379b5ba33c84633a2b36d47", size = 255299, upload-time = "2026-02-09T12:57:08.245Z" }, + { url = "https://files.pythonhosted.org/packages/03/94/0a4b12f1d0e029ce1ccc1c800944a9984cbe7d678e470bb6d3c6bc38a0da/coverage-7.13.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ee756f00726693e5ba94d6df2bdfd64d4852d23b09bb0bc700e3b30e6f333985", size = 250796, upload-time = "2026-02-09T12:57:10.142Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/6002fbf88f6698ca034360ce474c406be6d5a985b3fdb3401128031eef6b/coverage-7.13.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fdfc1e28e7c7cdce44985b3043bc13bbd9c747520f94a4d7164af8260b3d91f0", size = 252673, upload-time = "2026-02-09T12:57:12.197Z" }, + { url = "https://files.pythonhosted.org/packages/de/c6/a0279f7c00e786be75a749a5674e6fa267bcbd8209cd10c9a450c655dfa7/coverage-7.13.4-cp312-cp312-win32.whl", hash = "sha256:01d4cbc3c283a17fc1e42d614a119f7f438eabb593391283adca8dc86eff1246", size = 221990, upload-time = "2026-02-09T12:57:14.085Z" }, + { url = "https://files.pythonhosted.org/packages/77/4e/c0a25a425fcf5557d9abd18419c95b63922e897bc86c1f327f155ef234a9/coverage-7.13.4-cp312-cp312-win_amd64.whl", hash = "sha256:9401ebc7ef522f01d01d45532c68c5ac40fb27113019b6b7d8b208f6e9baa126", size = 222800, upload-time = "2026-02-09T12:57:15.944Z" }, + { url = "https://files.pythonhosted.org/packages/47/ac/92da44ad9a6f4e3a7debd178949d6f3769bedca33830ce9b1dcdab589a37/coverage-7.13.4-cp312-cp312-win_arm64.whl", hash = "sha256:b1ec7b6b6e93255f952e27ab58fbc68dcc468844b16ecbee881aeb29b6ab4d8d", size = 221415, upload-time = "2026-02-09T12:57:17.497Z" }, + { url = "https://files.pythonhosted.org/packages/db/23/aad45061a31677d68e47499197a131eea55da4875d16c1f42021ab963503/coverage-7.13.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b66a2da594b6068b48b2692f043f35d4d3693fb639d5ea8b39533c2ad9ac3ab9", size = 219474, upload-time = "2026-02-09T12:57:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/a5/70/9b8b67a0945f3dfec1fd896c5cefb7c19d5a3a6d74630b99a895170999ae/coverage-7.13.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3599eb3992d814d23b35c536c28df1a882caa950f8f507cef23d1cbf334995ac", size = 219844, upload-time = "2026-02-09T12:57:20.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/fd/7e859f8fab324cef6c4ad7cff156ca7c489fef9179d5749b0c8d321281c2/coverage-7.13.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:93550784d9281e374fb5a12bf1324cc8a963fd63b2d2f223503ef0fd4aa339ea", size = 250832, upload-time = "2026-02-09T12:57:22.007Z" }, + { url = "https://files.pythonhosted.org/packages/e4/dc/b2442d10020c2f52617828862d8b6ee337859cd8f3a1f13d607dddda9cf7/coverage-7.13.4-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b720ce6a88a2755f7c697c23268ddc47a571b88052e6b155224347389fdf6a3b", size = 253434, upload-time = "2026-02-09T12:57:23.339Z" }, + { url = "https://files.pythonhosted.org/packages/5a/88/6728a7ad17428b18d836540630487231f5470fb82454871149502f5e5aa2/coverage-7.13.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7b322db1284a2ed3aa28ffd8ebe3db91c929b7a333c0820abec3d838ef5b3525", size = 254676, upload-time = "2026-02-09T12:57:24.774Z" }, + { url = "https://files.pythonhosted.org/packages/7c/bc/21244b1b8cedf0dff0a2b53b208015fe798d5f2a8d5348dbfece04224fff/coverage-7.13.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f4594c67d8a7c89cf922d9df0438c7c7bb022ad506eddb0fdb2863359ff78242", size = 256807, upload-time = "2026-02-09T12:57:26.125Z" }, + { url = "https://files.pythonhosted.org/packages/97/a0/ddba7ed3251cff51006737a727d84e05b61517d1784a9988a846ba508877/coverage-7.13.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:53d133df809c743eb8bce33b24bcababb371f4441340578cd406e084d94a6148", size = 251058, upload-time = "2026-02-09T12:57:27.614Z" }, + { url = "https://files.pythonhosted.org/packages/9b/55/e289addf7ff54d3a540526f33751951bf0878f3809b47f6dfb3def69c6f7/coverage-7.13.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76451d1978b95ba6507a039090ba076105c87cc76fc3efd5d35d72093964d49a", size = 252805, upload-time = "2026-02-09T12:57:29.066Z" }, + { url = "https://files.pythonhosted.org/packages/13/4e/cc276b1fa4a59be56d96f1dabddbdc30f4ba22e3b1cd42504c37b3313255/coverage-7.13.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7f57b33491e281e962021de110b451ab8a24182589be17e12a22c79047935e23", size = 250766, upload-time = "2026-02-09T12:57:30.522Z" }, + { url = "https://files.pythonhosted.org/packages/94/44/1093b8f93018f8b41a8cf29636c9292502f05e4a113d4d107d14a3acd044/coverage-7.13.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1731dc33dc276dafc410a885cbf5992f1ff171393e48a21453b78727d090de80", size = 254923, upload-time = "2026-02-09T12:57:31.946Z" }, + { url = "https://files.pythonhosted.org/packages/8b/55/ea2796da2d42257f37dbea1aab239ba9263b31bd91d5527cdd6db5efe174/coverage-7.13.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:bd60d4fe2f6fa7dff9223ca1bbc9f05d2b6697bc5961072e5d3b952d46e1b1ea", size = 250591, upload-time = "2026-02-09T12:57:33.842Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/7c4bb72aacf8af5020675aa633e59c1fbe296d22aed191b6a5b711eb2bc7/coverage-7.13.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9181a3ccead280b828fae232df12b16652702b49d41e99d657f46cc7b1f6ec7a", size = 252364, upload-time = "2026-02-09T12:57:35.743Z" }, + { url = "https://files.pythonhosted.org/packages/5c/38/a8d2ec0146479c20bbaa7181b5b455a0c41101eed57f10dd19a78ab44c80/coverage-7.13.4-cp313-cp313-win32.whl", hash = "sha256:f53d492307962561ac7de4cd1de3e363589b000ab69617c6156a16ba7237998d", size = 222010, upload-time = "2026-02-09T12:57:37.25Z" }, + { url = "https://files.pythonhosted.org/packages/e2/0c/dbfafbe90a185943dcfbc766fe0e1909f658811492d79b741523a414a6cc/coverage-7.13.4-cp313-cp313-win_amd64.whl", hash = "sha256:e6f70dec1cc557e52df5306d051ef56003f74d56e9c4dd7ddb07e07ef32a84dd", size = 222818, upload-time = "2026-02-09T12:57:38.734Z" }, + { url = "https://files.pythonhosted.org/packages/04/d1/934918a138c932c90d78301f45f677fb05c39a3112b96fd2c8e60503cdc7/coverage-7.13.4-cp313-cp313-win_arm64.whl", hash = "sha256:fb07dc5da7e849e2ad31a5d74e9bece81f30ecf5a42909d0a695f8bd1874d6af", size = 221438, upload-time = "2026-02-09T12:57:40.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/57/ee93ced533bcb3e6df961c0c6e42da2fc6addae53fb95b94a89b1e33ebd7/coverage-7.13.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:40d74da8e6c4b9ac18b15331c4b5ebc35a17069410cad462ad4f40dcd2d50c0d", size = 220165, upload-time = "2026-02-09T12:57:41.639Z" }, + { url = "https://files.pythonhosted.org/packages/c5/e0/969fc285a6fbdda49d91af278488d904dcd7651b2693872f0ff94e40e84a/coverage-7.13.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4223b4230a376138939a9173f1bdd6521994f2aff8047fae100d6d94d50c5a12", size = 220516, upload-time = "2026-02-09T12:57:44.215Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b8/9531944e16267e2735a30a9641ff49671f07e8138ecf1ca13db9fd2560c7/coverage-7.13.4-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1d4be36a5114c499f9f1f9195e95ebf979460dbe2d88e6816ea202010ba1c34b", size = 261804, upload-time = "2026-02-09T12:57:45.989Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f3/e63df6d500314a2a60390d1989240d5f27318a7a68fa30ad3806e2a9323e/coverage-7.13.4-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:200dea7d1e8095cc6e98cdabe3fd1d21ab17d3cee6dab00cadbb2fe35d9c15b9", size = 263885, upload-time = "2026-02-09T12:57:47.42Z" }, + { url = "https://files.pythonhosted.org/packages/f3/67/7654810de580e14b37670b60a09c599fa348e48312db5b216d730857ffe6/coverage-7.13.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8eb931ee8e6d8243e253e5ed7336deea6904369d2fd8ae6e43f68abbf167092", size = 266308, upload-time = "2026-02-09T12:57:49.345Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/39d41eca0eab3cc82115953ad41c4e77935286c930e8fad15eaed1389d83/coverage-7.13.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75eab1ebe4f2f64d9509b984f9314d4aa788540368218b858dad56dc8f3e5eb9", size = 267452, upload-time = "2026-02-09T12:57:50.811Z" }, + { url = "https://files.pythonhosted.org/packages/50/6d/39c0fbb8fc5cd4d2090811e553c2108cf5112e882f82505ee7495349a6bf/coverage-7.13.4-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c35eb28c1d085eb7d8c9b3296567a1bebe03ce72962e932431b9a61f28facf26", size = 261057, upload-time = "2026-02-09T12:57:52.447Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a2/60010c669df5fa603bb5a97fb75407e191a846510da70ac657eb696b7fce/coverage-7.13.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:eb88b316ec33760714a4720feb2816a3a59180fd58c1985012054fa7aebee4c2", size = 263875, upload-time = "2026-02-09T12:57:53.938Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/63b22a6bdbd17f1f96e9ed58604c2a6b0e72a9133e37d663bef185877cf6/coverage-7.13.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7d41eead3cc673cbd38a4417deb7fd0b4ca26954ff7dc6078e33f6ff97bed940", size = 261500, upload-time = "2026-02-09T12:57:56.012Z" }, + { url = "https://files.pythonhosted.org/packages/70/bf/69f86ba1ad85bc3ad240e4c0e57a2e620fbc0e1645a47b5c62f0e941ad7f/coverage-7.13.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:fb26a934946a6afe0e326aebe0730cdff393a8bc0bbb65a2f41e30feddca399c", size = 265212, upload-time = "2026-02-09T12:57:57.5Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f2/5f65a278a8c2148731831574c73e42f57204243d33bedaaf18fa79c5958f/coverage-7.13.4-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:dae88bc0fc77edaa65c14be099bd57ee140cf507e6bfdeea7938457ab387efb0", size = 260398, upload-time = "2026-02-09T12:57:59.027Z" }, + { url = "https://files.pythonhosted.org/packages/ef/80/6e8280a350ee9fea92f14b8357448a242dcaa243cb2c72ab0ca591f66c8c/coverage-7.13.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:845f352911777a8e722bfce168958214951e07e47e5d5d9744109fa5fe77f79b", size = 262584, upload-time = "2026-02-09T12:58:01.129Z" }, + { url = "https://files.pythonhosted.org/packages/22/63/01ff182fc95f260b539590fb12c11ad3e21332c15f9799cb5e2386f71d9f/coverage-7.13.4-cp313-cp313t-win32.whl", hash = "sha256:2fa8d5f8de70688a28240de9e139fa16b153cc3cbb01c5f16d88d6505ebdadf9", size = 222688, upload-time = "2026-02-09T12:58:02.736Z" }, + { url = "https://files.pythonhosted.org/packages/a9/43/89de4ef5d3cd53b886afa114065f7e9d3707bdb3e5efae13535b46ae483d/coverage-7.13.4-cp313-cp313t-win_amd64.whl", hash = "sha256:9351229c8c8407645840edcc277f4a2d44814d1bc34a2128c11c2a031d45a5dd", size = 223746, upload-time = "2026-02-09T12:58:05.362Z" }, + { url = "https://files.pythonhosted.org/packages/35/39/7cf0aa9a10d470a5309b38b289b9bb07ddeac5d61af9b664fe9775a4cb3e/coverage-7.13.4-cp313-cp313t-win_arm64.whl", hash = "sha256:30b8d0512f2dc8c8747557e8fb459d6176a2c9e5731e2b74d311c03b78451997", size = 222003, upload-time = "2026-02-09T12:58:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/92/11/a9cf762bb83386467737d32187756a42094927150c3e107df4cb078e8590/coverage-7.13.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:300deaee342f90696ed186e3a00c71b5b3d27bffe9e827677954f4ee56969601", size = 219522, upload-time = "2026-02-09T12:58:08.623Z" }, + { url = "https://files.pythonhosted.org/packages/d3/28/56e6d892b7b052236d67c95f1936b6a7cf7c3e2634bf27610b8cbd7f9c60/coverage-7.13.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29e3220258d682b6226a9b0925bc563ed9a1ebcff3cad30f043eceea7eaf2689", size = 219855, upload-time = "2026-02-09T12:58:10.176Z" }, + { url = "https://files.pythonhosted.org/packages/e5/69/233459ee9eb0c0d10fcc2fe425a029b3fa5ce0f040c966ebce851d030c70/coverage-7.13.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:391ee8f19bef69210978363ca930f7328081c6a0152f1166c91f0b5fdd2a773c", size = 250887, upload-time = "2026-02-09T12:58:12.503Z" }, + { url = "https://files.pythonhosted.org/packages/06/90/2cdab0974b9b5bbc1623f7876b73603aecac11b8d95b85b5b86b32de5eab/coverage-7.13.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0dd7ab8278f0d58a0128ba2fca25824321f05d059c1441800e934ff2efa52129", size = 253396, upload-time = "2026-02-09T12:58:14.615Z" }, + { url = "https://files.pythonhosted.org/packages/ac/15/ea4da0f85bf7d7b27635039e649e99deb8173fe551096ea15017f7053537/coverage-7.13.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78cdf0d578b15148b009ccf18c686aa4f719d887e76e6b40c38ffb61d264a552", size = 254745, upload-time = "2026-02-09T12:58:16.162Z" }, + { url = "https://files.pythonhosted.org/packages/99/11/bb356e86920c655ca4d61daee4e2bbc7258f0a37de0be32d233b561134ff/coverage-7.13.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:48685fee12c2eb3b27c62f2658e7ea21e9c3239cba5a8a242801a0a3f6a8c62a", size = 257055, upload-time = "2026-02-09T12:58:17.892Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0f/9ae1f8cb17029e09da06ca4e28c9e1d5c1c0a511c7074592e37e0836c915/coverage-7.13.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4e83efc079eb39480e6346a15a1bcb3e9b04759c5202d157e1dd4303cd619356", size = 250911, upload-time = "2026-02-09T12:58:19.495Z" }, + { url = "https://files.pythonhosted.org/packages/89/3a/adfb68558fa815cbc29747b553bc833d2150228f251b127f1ce97e48547c/coverage-7.13.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecae9737b72408d6a950f7e525f30aca12d4bd8dd95e37342e5beb3a2a8c4f71", size = 252754, upload-time = "2026-02-09T12:58:21.064Z" }, + { url = "https://files.pythonhosted.org/packages/32/b1/540d0c27c4e748bd3cd0bd001076ee416eda993c2bae47a73b7cc9357931/coverage-7.13.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ae4578f8528569d3cf303fef2ea569c7f4c4059a38c8667ccef15c6e1f118aa5", size = 250720, upload-time = "2026-02-09T12:58:22.622Z" }, + { url = "https://files.pythonhosted.org/packages/c7/95/383609462b3ffb1fe133014a7c84fc0dd01ed55ac6140fa1093b5af7ebb1/coverage-7.13.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6fdef321fdfbb30a197efa02d48fcd9981f0d8ad2ae8903ac318adc653f5df98", size = 254994, upload-time = "2026-02-09T12:58:24.548Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ba/1761138e86c81680bfc3c49579d66312865457f9fe405b033184e5793cb3/coverage-7.13.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b0f6ccf3dbe577170bebfce1318707d0e8c3650003cb4b3a9dd744575daa8b5", size = 250531, upload-time = "2026-02-09T12:58:26.271Z" }, + { url = "https://files.pythonhosted.org/packages/f8/8e/05900df797a9c11837ab59c4d6fe94094e029582aab75c3309a93e6fb4e3/coverage-7.13.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75fcd519f2a5765db3f0e391eb3b7d150cce1a771bf4c9f861aeab86c767a3c0", size = 252189, upload-time = "2026-02-09T12:58:27.807Z" }, + { url = "https://files.pythonhosted.org/packages/00/bd/29c9f2db9ea4ed2738b8a9508c35626eb205d51af4ab7bf56a21a2e49926/coverage-7.13.4-cp314-cp314-win32.whl", hash = "sha256:8e798c266c378da2bd819b0677df41ab46d78065fb2a399558f3f6cae78b2fbb", size = 222258, upload-time = "2026-02-09T12:58:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4d/1f8e723f6829977410efeb88f73673d794075091c8c7c18848d273dc9d73/coverage-7.13.4-cp314-cp314-win_amd64.whl", hash = "sha256:245e37f664d89861cf2329c9afa2c1fe9e6d4e1a09d872c947e70718aeeac505", size = 223073, upload-time = "2026-02-09T12:58:31.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/5b/84100025be913b44e082ea32abcf1afbf4e872f5120b7a1cab1d331b1e13/coverage-7.13.4-cp314-cp314-win_arm64.whl", hash = "sha256:ad27098a189e5838900ce4c2a99f2fe42a0bf0c2093c17c69b45a71579e8d4a2", size = 221638, upload-time = "2026-02-09T12:58:32.599Z" }, + { url = "https://files.pythonhosted.org/packages/a7/e4/c884a405d6ead1370433dad1e3720216b4f9fd8ef5b64bfd984a2a60a11a/coverage-7.13.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:85480adfb35ffc32d40918aad81b89c69c9cc5661a9b8a81476d3e645321a056", size = 220246, upload-time = "2026-02-09T12:58:34.181Z" }, + { url = "https://files.pythonhosted.org/packages/81/5c/4d7ed8b23b233b0fffbc9dfec53c232be2e695468523242ea9fd30f97ad2/coverage-7.13.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:79be69cf7f3bf9b0deeeb062eab7ac7f36cd4cc4c4dd694bd28921ba4d8596cc", size = 220514, upload-time = "2026-02-09T12:58:35.704Z" }, + { url = "https://files.pythonhosted.org/packages/2f/6f/3284d4203fd2f28edd73034968398cd2d4cb04ab192abc8cff007ea35679/coverage-7.13.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:caa421e2684e382c5d8973ac55e4f36bed6821a9bad5c953494de960c74595c9", size = 261877, upload-time = "2026-02-09T12:58:37.864Z" }, + { url = "https://files.pythonhosted.org/packages/09/aa/b672a647bbe1556a85337dc95bfd40d146e9965ead9cc2fe81bde1e5cbce/coverage-7.13.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14375934243ee05f56c45393fe2ce81fe5cc503c07cee2bdf1725fb8bef3ffaf", size = 264004, upload-time = "2026-02-09T12:58:39.492Z" }, + { url = "https://files.pythonhosted.org/packages/79/a1/aa384dbe9181f98bba87dd23dda436f0c6cf2e148aecbb4e50fc51c1a656/coverage-7.13.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25a41c3104d08edb094d9db0d905ca54d0cd41c928bb6be3c4c799a54753af55", size = 266408, upload-time = "2026-02-09T12:58:41.852Z" }, + { url = "https://files.pythonhosted.org/packages/53/5e/5150bf17b4019bc600799f376bb9606941e55bd5a775dc1e096b6ffea952/coverage-7.13.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f01afcff62bf9a08fb32b2c1d6e924236c0383c02c790732b6537269e466a72", size = 267544, upload-time = "2026-02-09T12:58:44.093Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ed/f1de5c675987a4a7a672250d2c5c9d73d289dbf13410f00ed7181d8017dd/coverage-7.13.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eb9078108fbf0bcdde37c3f4779303673c2fa1fe8f7956e68d447d0dd426d38a", size = 260980, upload-time = "2026-02-09T12:58:45.721Z" }, + { url = "https://files.pythonhosted.org/packages/b3/e3/fe758d01850aa172419a6743fe76ba8b92c29d181d4f676ffe2dae2ba631/coverage-7.13.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0e086334e8537ddd17e5f16a344777c1ab8194986ec533711cbe6c41cde841b6", size = 263871, upload-time = "2026-02-09T12:58:47.334Z" }, + { url = "https://files.pythonhosted.org/packages/b6/76/b829869d464115e22499541def9796b25312b8cf235d3bb00b39f1675395/coverage-7.13.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:725d985c5ab621268b2edb8e50dfe57633dc69bda071abc470fed55a14935fd3", size = 261472, upload-time = "2026-02-09T12:58:48.995Z" }, + { url = "https://files.pythonhosted.org/packages/14/9e/caedb1679e73e2f6ad240173f55218488bfe043e38da577c4ec977489915/coverage-7.13.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:3c06f0f1337c667b971ca2f975523347e63ec5e500b9aa5882d91931cd3ef750", size = 265210, upload-time = "2026-02-09T12:58:51.178Z" }, + { url = "https://files.pythonhosted.org/packages/3a/10/0dd02cb009b16ede425b49ec344aba13a6ae1dc39600840ea6abcb085ac4/coverage-7.13.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:590c0ed4bf8e85f745e6b805b2e1c457b2e33d5255dd9729743165253bc9ad39", size = 260319, upload-time = "2026-02-09T12:58:53.081Z" }, + { url = "https://files.pythonhosted.org/packages/92/8e/234d2c927af27c6d7a5ffad5bd2cf31634c46a477b4c7adfbfa66baf7ebb/coverage-7.13.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:eb30bf180de3f632cd043322dad5751390e5385108b2807368997d1a92a509d0", size = 262638, upload-time = "2026-02-09T12:58:55.258Z" }, + { url = "https://files.pythonhosted.org/packages/2f/64/e5547c8ff6964e5965c35a480855911b61509cce544f4d442caa759a0702/coverage-7.13.4-cp314-cp314t-win32.whl", hash = "sha256:c4240e7eded42d131a2d2c4dec70374b781b043ddc79a9de4d55ca71f8e98aea", size = 223040, upload-time = "2026-02-09T12:58:56.936Z" }, + { url = "https://files.pythonhosted.org/packages/c7/96/38086d58a181aac86d503dfa9c47eb20715a79c3e3acbdf786e92e5c09a8/coverage-7.13.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4c7d3cc01e7350f2f0f6f7036caaf5673fb56b6998889ccfe9e1c1fe75a9c932", size = 224148, upload-time = "2026-02-09T12:58:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/ce/72/8d10abd3740a0beb98c305e0c3faf454366221c0f37a8bcf8f60020bb65a/coverage-7.13.4-cp314-cp314t-win_arm64.whl", hash = "sha256:23e3f687cf945070d1c90f85db66d11e3025665d8dafa831301a0e0038f3db9b", size = 222172, upload-time = "2026-02-09T12:59:00.396Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4a/331fe2caf6799d591109bb9c08083080f6de90a823695d412a935622abb2/coverage-7.13.4-py3-none-any.whl", hash = "sha256:1af1641e57cf7ba1bd67d677c9abdbcd6cc2ab7da3bca7fa1e2b7e50e65f2ad0", size = 211242, upload-time = "2026-02-09T12:59:02.032Z" }, ] [[package]] name = "coveralls" -version = "4.0.1" +version = "4.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "coverage" }, { name = "docopt" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/61/75/a454fb443eb6a053833f61603a432ffbd7dd6ae53a11159bacfadb9d6219/coveralls-4.0.1.tar.gz", hash = "sha256:7b2a0a2bcef94f295e3cf28dcc55ca40b71c77d1c2446b538e85f0f7bc21aa69", size = 12419, upload-time = "2024-05-15T12:56:14.297Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/b3/25862e4610461ca5ffe75cf8d05e830d0b920e023530ab486598ddbe8df8/coveralls-4.0.2.tar.gz", hash = "sha256:7c21ffa2808d3052fa0cfca3842a9f3d21cc8eada02538c192d932199e5f07d4", size = 12408, upload-time = "2025-11-07T19:18:50.54Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/63/e5/6708c75e2a4cfca929302d4d9b53b862c6dc65bd75e6933ea3d20016d41d/coveralls-4.0.1-py3-none-any.whl", hash = "sha256:7a6b1fa9848332c7b2221afb20f3df90272ac0167060f41b5fe90429b30b1809", size = 13599, upload-time = "2024-05-15T12:56:12.342Z" }, + { url = "https://files.pythonhosted.org/packages/06/96/3c046a70c27c9be06589aaab99e7e10067177a338a874311347e662aecb1/coveralls-4.0.2-py3-none-any.whl", hash = "sha256:3940f613eac6b3c14d1425741929e1d15f57666f5e7ae0572bbe92357bd6f7ee", size = 13549, upload-time = "2025-11-07T19:18:49.189Z" }, ] [[package]] name = "distlib" -version = "0.3.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0d/dd/1bec4c5ddb504ca60fc29472f3d27e8d4da1257a854e1d96742f15c1d02d/distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403", size = 613923, upload-time = "2024-10-09T18:35:47.551Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/a1/cf2472db20f7ce4a6be1253a81cfdf85ad9c7885ffbed7047fb72c24cf87/distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87", size = 468973, upload-time = "2024-10-09T18:35:44.272Z" }, -] - -[[package]] -name = "django" -version = "5.2.9" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.12'", -] -dependencies = [ - { name = "asgiref", version = "3.8.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "sqlparse", marker = "python_full_version < '3.12'" }, - { name = "tzdata", marker = "python_full_version < '3.12' and sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/1c/188ce85ee380f714b704283013434976df8d3a2df8e735221a02605b6794/django-5.2.9.tar.gz", hash = "sha256:16b5ccfc5e8c27e6c0561af551d2ea32852d7352c67d452ae3e76b4f6b2ca495", size = 10848762, upload-time = "2025-12-02T14:01:08.418Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/b0/7f42bfc38b8f19b78546d47147e083ed06e12fc29c42da95655e0962c6c2/django-5.2.9-py3-none-any.whl", hash = "sha256:3a4ea88a70370557ab1930b332fd2887a9f48654261cdffda663fef5976bb00a", size = 8290652, upload-time = "2025-12-02T14:01:03.485Z" }, + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] [[package]] name = "django" -version = "6.0" +version = "6.0.2" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", -] dependencies = [ - { name = "asgiref", version = "3.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "sqlparse", marker = "python_full_version >= '3.12'" }, - { name = "tzdata", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, + { name = "asgiref" }, + { name = "sqlparse" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/15/75/19762bfc4ea556c303d9af8e36f0cd910ab17dff6c8774644314427a2120/django-6.0.tar.gz", hash = "sha256:7b0c1f50c0759bbe6331c6a39c89ae022a84672674aeda908784617ef47d8e26", size = 10932418, upload-time = "2025-12-03T16:26:21.878Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/3e/a1c4207c5dea4697b7a3387e26584919ba987d8f9320f59dc0b5c557a4eb/django-6.0.2.tar.gz", hash = "sha256:3046a53b0e40d4b676c3b774c73411d7184ae2745fe8ce5e45c0f33d3ddb71a7", size = 10886874, upload-time = "2026-02-03T13:50:31.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/ae/f19e24789a5ad852670d6885f5480f5e5895576945fcc01817dfd9bc002a/django-6.0-py3-none-any.whl", hash = "sha256:1cc2c7344303bbfb7ba5070487c17f7fc0b7174bbb0a38cebf03c675f5f19b6d", size = 8339181, upload-time = "2025-12-03T16:26:16.231Z" }, + { url = "https://files.pythonhosted.org/packages/96/ba/a6e2992bc5b8c688249c00ea48cb1b7a9bc09839328c81dc603671460928/django-6.0.2-py3-none-any.whl", hash = "sha256:610dd3b13d15ec3f1e1d257caedd751db8033c5ad8ea0e2d1219a8acf446ecc6", size = 8339381, upload-time = "2026-02-03T13:50:15.501Z" }, ] [[package]] name = "django-auth-ldap" -version = "5.2.0" +version = "5.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "django", version = "5.2.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "django", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "django" }, { name = "python-ldap" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/88/70/6f6a89474667376080f8362f7c17c744d1c52720f0eb085cf74182149efe/django_auth_ldap-5.2.0.tar.gz", hash = "sha256:08ba6efc0340d9874725a962311b14991e29a33593eb150a8fb640709dbfa80f", size = 55287, upload-time = "2025-05-07T12:15:56.774Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/6d/d3ceb4b49e7153811a4b2d92bbe198a5ef2e2820469add3d6dc129ef2fab/django_auth_ldap-5.3.0.tar.gz", hash = "sha256:743d8107b146240b46f7e97207dc06cb11facc0cd70dce490b7ca09dd5643d19", size = 55272, upload-time = "2025-12-26T15:00:14.272Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/65/0d26a8b5c19039305d7ae0e8e702613a9a1fe1ef3ebbd6206b9e104b7c43/django_auth_ldap-5.2.0-py3-none-any.whl", hash = "sha256:7dc6eb576ba36051850b580e4bdf4464e04bbe7367c3827a3121b4d7c51fb175", size = 20913, upload-time = "2025-05-07T12:15:54.962Z" }, + { url = "https://files.pythonhosted.org/packages/a9/91/38ba24b9d76925ce166b2eebe1b4ea460063b8ba8cf91d39d97ee3bad517/django_auth_ldap-5.3.0-py3-none-any.whl", hash = "sha256:aa880415983149b072f876d976ef8ec755a438090e176817998263a6ed9e1038", size = 20975, upload-time = "2025-12-26T15:00:12.52Z" }, ] [[package]] @@ -289,8 +256,7 @@ name = "django-filter" version = "25.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "django", version = "5.2.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "django", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "django" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2c/e4/465d2699cd388c0005fb8d6ae6709f239917c6d8790ac35719676fffdcf3/django_filter-25.2.tar.gz", hash = "sha256:760e984a931f4468d096f5541787efb8998c61217b73006163bf2f9523fe8f23", size = 143818, upload-time = "2025-10-05T09:51:31.521Z" } wheels = [ @@ -303,8 +269,7 @@ version = "1.15" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, - { name = "django", version = "5.2.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "django", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "django" }, { name = "elasticsearch" }, { name = "six" }, ] @@ -318,8 +283,7 @@ name = "django-netfields" version = "1.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "django", version = "5.2.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "django", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "django" }, { name = "netaddr" }, { name = "six" }, ] @@ -330,8 +294,7 @@ name = "djangorestframework" version = "3.16.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "django", version = "5.2.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "django", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "django" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8a/95/5376fe618646fde6899b3cdc85fd959716bb67542e273a76a80d9f326f27/djangorestframework-3.16.1.tar.gz", hash = "sha256:166809528b1aced0a17dc66c24492af18049f2c9420dbd0be29422029cfc3ff7", size = 1089735, upload-time = "2025-08-06T17:50:53.251Z" } wheels = [ @@ -349,8 +312,7 @@ name = "drf-standardized-errors" version = "0.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "django", version = "5.2.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "django", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "django" }, { name = "djangorestframework" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/9301be05081261cebdc000f9fbad51dc2b2732f9decd8da693f7c2daae29/drf_standardized_errors-0.15.0.tar.gz", hash = "sha256:83112d072e751eb444c2f16ab4618273b912cffc07f12b81998060fdfa2eb655", size = 60729, upload-time = "2025-06-09T07:47:56.933Z" } @@ -360,66 +322,62 @@ wheels = [ [[package]] name = "elastic-transport" -version = "8.15.1" +version = "9.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, + { name = "sniffio" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6c/8a/54608571bf0a6686b9f49ccb3acf86e20bc50039ce87e368ebb11faebbc8/elastic_transport-8.15.1.tar.gz", hash = "sha256:9cac4ab5cf9402668cf305ae0b7d93ddc0c7b61461d6d1027850db6da9cc5742", size = 72726, upload-time = "2024-10-09T11:50:42.974Z" } +sdist = { url = "https://files.pythonhosted.org/packages/23/0a/a92140b666afdcb9862a16e4d80873b3c887c1b7e3f17e945fc3460edf1b/elastic_transport-9.2.1.tar.gz", hash = "sha256:97d9abd638ba8aa90faa4ca1bf1a18bde0fe2088fbc8757f2eb7b299f205773d", size = 77403, upload-time = "2025-12-23T11:54:12.849Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/20/864177d7ecbc98633ead4d8e12200ec1c7c7a0c8d45080ac636e043db1c7/elastic_transport-8.15.1-py3-none-any.whl", hash = "sha256:b5e82ff1679d8c7705a03fd85c7f6ef85d6689721762d41228dd312e34f331fc", size = 64426, upload-time = "2024-10-09T11:50:41.022Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e6/a42b600ae8b808371f740381f6c32050cad93f870d36cc697b8b7006bf7c/elastic_transport-9.2.1-py3-none-any.whl", hash = "sha256:39e1a25e486af34ce7aa1bc9005d1c736f1b6fb04c9b64ea0604ded5a61fc1d4", size = 65327, upload-time = "2025-12-23T11:54:11.681Z" }, ] [[package]] name = "elasticsearch" -version = "8.15.1" +version = "9.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "anyio" }, { name = "elastic-transport" }, + { name = "python-dateutil" }, + { name = "sniffio" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/87/eb/d69ca80a048601184a340df1670f667e5fd92862446cfbfc48d793fa5865/elasticsearch-8.15.1.tar.gz", hash = "sha256:40c0d312f8adf8bdc81795bc16a0b546ddf544cb1f90e829a244e4780c4dbfd8", size = 414690, upload-time = "2024-09-10T09:13:12.12Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/20/6f1d6977f68389116e40a0108a5bfd468f3a0cceabe90b522693834bb5ec/elasticsearch-8.15.1-py3-none-any.whl", hash = "sha256:02a0476e98768a30d7926335fc0d305c04fdb928eea1354c6e6040d8c2814569", size = 524643, upload-time = "2024-09-10T09:13:06.767Z" }, -] - -[[package]] -name = "exceptiongroup" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/09/35/2495c4ac46b980e4ca1f6ad6db102322ef3ad2410b79fdde159a4b0f3b92/exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc", size = 28883, upload-time = "2024-07-12T22:26:00.161Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/15/283459c9299d412ffa2aaab69b082857631c519233f5491d6c567e3320ca/elasticsearch-9.3.0.tar.gz", hash = "sha256:f76e149c0a22d5ccbba58bdc30c9f51cf894231b359ef4fd7e839b558b59f856", size = 893538, upload-time = "2026-02-03T20:26:38.914Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/cc/b7e31358aac6ed1ef2bb790a9746ac2c69bcb3c8588b41616914eb106eaf/exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b", size = 16453, upload-time = "2024-07-12T22:25:58.476Z" }, + { url = "https://files.pythonhosted.org/packages/05/37/3a196f8918743f2104cb66b1f56218079ecac6e128c061de7df7f4faef02/elasticsearch-9.3.0-py3-none-any.whl", hash = "sha256:67bd2bb4f0800f58c2847d29cd57d6e7bf5bc273483b4f17421f93e75ba09f39", size = 979405, upload-time = "2026-02-03T20:26:34.552Z" }, ] [[package]] name = "filelock" -version = "3.20.3" +version = "3.24.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/65/ce7f1b70157833bf3cb851b556a37d4547ceafc158aa9b34b36782f23696/filelock-3.20.3.tar.gz", hash = "sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1", size = 19485, upload-time = "2026-01-09T17:55:05.421Z" } +sdist = { url = "https://files.pythonhosted.org/packages/00/cd/fa3ab025a8f9772e8a9146d8fd8eef6d62649274d231ca84249f54a0de4a/filelock-3.24.0.tar.gz", hash = "sha256:aeeab479339ddf463a1cdd1f15a6e6894db976071e5883efc94d22ed5139044b", size = 37166, upload-time = "2026-02-14T16:05:28.723Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, + { url = "https://files.pythonhosted.org/packages/d9/dd/d7e7f4f49180e8591c9e1281d15ecf8e7f25eb2c829771d9682f1f9fe0c8/filelock-3.24.0-py3-none-any.whl", hash = "sha256:eebebb403d78363ef7be8e236b63cc6760b0004c7464dceaba3fd0afbd637ced", size = 23977, upload-time = "2026-02-14T16:05:27.578Z" }, ] [[package]] name = "gunicorn" -version = "23.0.0" +version = "25.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/13/ef67f59f6a7896fdc2c1d62b5665c5219d6b0a9a1784938eb9a28e55e128/gunicorn-25.1.0.tar.gz", hash = "sha256:1426611d959fa77e7de89f8c0f32eed6aa03ee735f98c01efba3e281b1c47616", size = 594377, upload-time = "2026-02-13T11:09:58.989Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload_time = "2024-08-10T20:25:24.996Z" }, + { url = "https://files.pythonhosted.org/packages/da/73/4ad5b1f6a2e21cf1e85afdaad2b7b1a933985e2f5d679147a1953aaa192c/gunicorn-25.1.0-py3-none-any.whl", hash = "sha256:d0b1236ccf27f72cfe14bce7caadf467186f19e865094ca84221424e839b8b8b", size = 197067, upload-time = "2026-02-13T11:09:57.146Z" }, ] [[package]] name = "h11" version = "0.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload_time = "2025-04-24T03:35:25.427Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload_time = "2025-04-24T03:35:24.344Z" }, + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] [[package]] @@ -430,9 +388,9 @@ dependencies = [ { name = "certifi" }, { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload_time = "2025-04-24T22:06:22.219Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload_time = "2025-04-24T22:06:20.566Z" }, + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] [[package]] @@ -445,9 +403,9 @@ dependencies = [ { name = "httpcore" }, { name = "idna" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload_time = "2024-12-06T15:37:23.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload_time = "2024-12-06T15:37:21.509Z" }, + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] [[package]] @@ -461,23 +419,23 @@ wheels = [ [[package]] name = "iniconfig" -version = "2.0.0" +version = "2.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646, upload-time = "2023-01-07T11:08:11.254Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892, upload-time = "2023-01-07T11:08:09.864Z" }, + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] [[package]] name = "markdown-it-py" -version = "3.0.0" +version = "4.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mdurl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] [[package]] @@ -493,8 +451,7 @@ wheels = [ name = "mreg" source = { editable = "." } dependencies = [ - { name = "django", version = "5.2.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "django", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "django" }, { name = "django-auth-ldap" }, { name = "django-filter" }, { name = "django-logging-json" }, @@ -537,7 +494,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "django", specifier = ">=5.2" }, - { name = "django-auth-ldap", specifier = ">=5.2.0" }, + { name = "django-auth-ldap", specifier = ">=5.3.0" }, { name = "django-filter", specifier = ">=25" }, { name = "django-logging-json", specifier = ">=1.15" }, { name = "django-netfields", specifier = ">=1.3.2" }, @@ -546,12 +503,13 @@ requires-dist = [ { name = "gunicorn", specifier = ">=23.0.0" }, { name = "idna", specifier = ">=3.11" }, { name = "pika", specifier = ">=1.3.2" }, - { name = "prometheus-client", specifier = ">=0.20" }, + { name = "prometheus-client", specifier = ">=0.24" }, { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3" }, { name = "pyyaml" }, { name = "rich", specifier = ">=14" }, { name = "sentry-sdk", specifier = ">=2.48.0" }, { name = "structlog", specifier = ">=25" }, + { name = "treetop-client", specifier = ">=0.0.7" }, { name = "tzdata", specifier = ">=2025.3" }, { name = "unittest-parametrize" }, { name = "uritemplate" }, @@ -586,11 +544,11 @@ wheels = [ [[package]] name = "packaging" -version = "25.0" +version = "26.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, ] [[package]] @@ -604,11 +562,11 @@ wheels = [ [[package]] name = "platformdirs" -version = "4.5.1" +version = "4.9.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cf/86/0248f086a84f01b37aaec0fa567b397df1a119f73c16f6c7a9aac73ea309/platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", size = 21715, upload-time = "2025-12-05T13:52:58.638Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/d5/763666321efaded11112de8b7a7f2273dd8d1e205168e73c334e54b0ab9a/platformdirs-4.9.1.tar.gz", hash = "sha256:f310f16e89c4e29117805d8328f7c10876eeff36c94eac879532812110f7d39f", size = 28392, upload-time = "2026-02-14T21:02:44.973Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" }, + { url = "https://files.pythonhosted.org/packages/70/77/e8c95e95f1d4cdd88c90a96e31980df7e709e51059fac150046ad67fac63/platformdirs-4.9.1-py3-none-any.whl", hash = "sha256:61d8b967d34791c162d30d60737369cbbd77debad5b981c4bfda1842e71e0d66", size = 21307, upload-time = "2026-02-14T21:02:43.492Z" }, ] [[package]] @@ -622,11 +580,11 @@ wheels = [ [[package]] name = "prometheus-client" -version = "0.23.1" +version = "0.24.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/23/53/3edb5d68ecf6b38fcbcc1ad28391117d2a322d9a1a3eff04bfdb184d8c3b/prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce", size = 80481, upload-time = "2025-09-18T20:47:25.043Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/58/a794d23feb6b00fc0c72787d7e87d872a6730dd9ed7c7b3e954637d8f280/prometheus_client-0.24.1.tar.gz", hash = "sha256:7e0ced7fbbd40f7b84962d5d2ab6f17ef88a72504dcf7c0b40737b43b2a461f9", size = 85616, upload-time = "2026-01-14T15:26:26.965Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" }, + { url = "https://files.pythonhosted.org/packages/74/c3/24a2f845e3917201628ecaba4f18bab4d18a337834c1df2a159ee9d22a42/prometheus_client-0.24.1-py3-none-any.whl", hash = "sha256:150db128af71a5c2482b36e588fc8a6b95e498750da4b17065947c16070f4055", size = 64057, upload-time = "2026-01-14T15:26:24.42Z" }, ] [[package]] @@ -655,28 +613,6 @@ name = "psycopg-binary" version = "3.3.2" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/25/d7/edfb0d9e56081246fd88490f99b1bafebd3588480cca601a4de0c41a3e08/psycopg_binary-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0768c5f32934bb52a5df098317eca9bdcf411de627c5dca2ee57662b64b54b41", size = 4597785, upload-time = "2025-12-06T17:31:44.867Z" }, - { url = "https://files.pythonhosted.org/packages/71/45/8458201d9573dd851263a05cefddd4bfd31e8b3c6434b3e38d62aea9f15a/psycopg_binary-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:09b3014013f05cd89828640d3a1db5f829cc24ad8fa81b6e42b2c04685a0c9d4", size = 4664440, upload-time = "2025-12-06T17:31:49.1Z" }, - { url = "https://files.pythonhosted.org/packages/d1/33/484260d87456cfe88dc219c1919026f11949b9d1de8a6371ddbe027d4d60/psycopg_binary-3.3.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3789d452a9d17a841c7f4f97bbcba51a21f957ea35641a4c98507520e6b6a068", size = 5478355, upload-time = "2025-12-06T17:31:52.657Z" }, - { url = "https://files.pythonhosted.org/packages/34/b2/18c91630c30c83f534c2bfa75fb533293fc9c3ab31bb7f2bf1cd9579c53b/psycopg_binary-3.3.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:44e89938d36acc4495735af70a886d206a5bfdc80258f95b69b52f68b2968d9e", size = 5152398, upload-time = "2025-12-06T17:31:56.092Z" }, - { url = "https://files.pythonhosted.org/packages/c0/14/7c705e1934107196d9dca2040cf34bce2ca26de62520e43073d2673052d4/psycopg_binary-3.3.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90ed9da805e52985b0202aed4f352842c907c6b4fc6c7c109c6e646c32e2f43b", size = 6748982, upload-time = "2025-12-06T17:32:00.611Z" }, - { url = "https://files.pythonhosted.org/packages/56/18/80197c47798926f79e563af02a71d1abecab88cf45ddf8dc960700598da7/psycopg_binary-3.3.2-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c3a9ccdfee4ae59cf9bf1822777e763bc097ed208f4901e21537fca1070e1391", size = 4991214, upload-time = "2025-12-06T17:32:03.897Z" }, - { url = "https://files.pythonhosted.org/packages/7e/2e/e88e2f678f5d1a968d87e57b30915061c1157e916b8aaa9b0b78bca95e25/psycopg_binary-3.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:de9173f8cc0efd88ac2a89b3b6c287a9a0011cdc2f53b2a12c28d6fd55f9f81c", size = 4517421, upload-time = "2025-12-06T17:32:07.287Z" }, - { url = "https://files.pythonhosted.org/packages/80/9e/d56813b24370723bcd62bf73871aee4d5fca0536f3476c4c4d5b037e3c7f/psycopg_binary-3.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0611f4822674f3269e507a307236efb62ae5a828fcfc923ac85fe22ca19fd7c8", size = 4206124, upload-time = "2025-12-06T17:32:10.374Z" }, - { url = "https://files.pythonhosted.org/packages/91/81/5a11a898969edf0ee43d0613a6dfd689a0aa12d418c69e148a8ff153fbc7/psycopg_binary-3.3.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:522b79c7db547767ca923e441c19b97a2157f2f494272a119c854bba4804e186", size = 3937067, upload-time = "2025-12-06T17:32:13.852Z" }, - { url = "https://files.pythonhosted.org/packages/a1/33/a6180ff1e747a0395876d985e8e295c9d7cbe956a2d66f165e7c67cffe55/psycopg_binary-3.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1ea41c0229f3f5a3844ad0857a83a9f869aa7b840448fa0c200e6bcf85d33d19", size = 4243731, upload-time = "2025-12-06T17:32:16.803Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5b/9c1b6fbc900d5b525946ed9a477865c5016a5306080c0557248bb04f1a5b/psycopg_binary-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:8ea05b499278790a8fa0ff9854ab0de2542aca02d661ddff94e830df971ff640", size = 3546403, upload-time = "2025-12-06T17:32:19.621Z" }, - { url = "https://files.pythonhosted.org/packages/57/d9/49640360fc090d27afc4655021544aa71d5393ebae124ffa53a04474b493/psycopg_binary-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:94503b79f7da0b65c80d0dbb2f81dd78b300319ec2435d5e6dcf9622160bc2fa", size = 4597890, upload-time = "2025-12-06T17:32:23.087Z" }, - { url = "https://files.pythonhosted.org/packages/85/cf/99634bbccc8af0dd86df4bce705eea5540d06bb7f5ab3067446ae9ffdae4/psycopg_binary-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07a5f030e0902ec3e27d0506ceb01238c0aecbc73ecd7fa0ee55f86134600b5b", size = 4664396, upload-time = "2025-12-06T17:32:26.421Z" }, - { url = "https://files.pythonhosted.org/packages/40/db/6035dff6d5c6dfca3a4ab0d2ac62ede623646e327e9f99e21e0cf08976c6/psycopg_binary-3.3.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e09d0d93d35c134704a2cb2b15f81ffc8174fd602f3e08f7b1a3d8896156cf0", size = 5478743, upload-time = "2025-12-06T17:32:29.901Z" }, - { url = "https://files.pythonhosted.org/packages/03/0f/fc06bbc8e87f09458d2ce04a59cd90565e54e8efca33e0802daee6d2b0e6/psycopg_binary-3.3.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:649c1d33bedda431e0c1df646985fbbeb9274afa964e1aef4be053c0f23a2924", size = 5151820, upload-time = "2025-12-06T17:32:33.562Z" }, - { url = "https://files.pythonhosted.org/packages/86/ab/bcc0397c96a0ad29463e33ed03285826e0fabc43595c195f419d9291ee70/psycopg_binary-3.3.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5774272f754605059521ff037a86e680342e3847498b0aa86b0f3560c70963c", size = 6747711, upload-time = "2025-12-06T17:32:38.074Z" }, - { url = "https://files.pythonhosted.org/packages/96/eb/7450bc75c31d5be5f7a6d02d26beef6989a4ca6f5efdec65eea6cf612d0e/psycopg_binary-3.3.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d391b70c9cc23f6e1142729772a011f364199d2c5ddc0d596f5f43316fbf982d", size = 4991626, upload-time = "2025-12-06T17:32:41.373Z" }, - { url = "https://files.pythonhosted.org/packages/dc/85/65f14453804c82a7fba31cd1a984b90349c0f327b809102c4b99115c0930/psycopg_binary-3.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f3f601f32244a677c7b029ec39412db2772ad04a28bc2cbb4b1f0931ed0ffad7", size = 4516760, upload-time = "2025-12-06T17:32:44.921Z" }, - { url = "https://files.pythonhosted.org/packages/24/8c/3105f00a91d73d9a443932f95156eae8159d5d9cb68a9d2cf512710d484f/psycopg_binary-3.3.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:0ae60e910531cfcc364a8f615a7941cac89efeb3f0fffe0c4824a6d11461eef7", size = 4204028, upload-time = "2025-12-06T17:32:48.355Z" }, - { url = "https://files.pythonhosted.org/packages/1e/dd/74f64a383342ef7c22d1eb2768ed86411c7f877ed2580cd33c17f436fe3c/psycopg_binary-3.3.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7c43a773dd1a481dbb2fe64576aa303d80f328cce0eae5e3e4894947c41d1da7", size = 3935780, upload-time = "2025-12-06T17:32:51.347Z" }, - { url = "https://files.pythonhosted.org/packages/85/30/f3f207d1c292949a26cdea6727c9c325b4ee41e04bf2736a4afbe45eb61f/psycopg_binary-3.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5a327327f1188b3fbecac41bf1973a60b86b2eb237db10dc945bd3dc97ec39e4", size = 4243239, upload-time = "2025-12-06T17:32:54.924Z" }, - { url = "https://files.pythonhosted.org/packages/b3/08/8f1b5d6231338bf7bc46f635c4d4965facec52e1c9a7952ca8a70cb57dc0/psycopg_binary-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:136c43f185244893a527540307167f5d3ef4e08786508afe45d6f146228f5aa9", size = 3548102, upload-time = "2025-12-06T17:32:57.944Z" }, { url = "https://files.pythonhosted.org/packages/4e/1e/8614b01c549dd7e385dacdcd83fe194f6b3acb255a53cc67154ee6bf00e7/psycopg_binary-3.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a9387ab615f929e71ef0f4a8a51e986fa06236ccfa9f3ec98a88f60fbf230634", size = 4579832, upload-time = "2025-12-06T17:33:01.388Z" }, { url = "https://files.pythonhosted.org/packages/26/97/0bb093570fae2f4454d42c1ae6000f15934391867402f680254e4a7def54/psycopg_binary-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3ff7489df5e06c12d1829544eaec64970fe27fe300f7cf04c8495fe682064688", size = 4658786, upload-time = "2025-12-06T17:33:05.022Z" }, { url = "https://files.pythonhosted.org/packages/61/20/1d9383e3f2038826900a14137b0647d755f67551aab316e1021443105ed5/psycopg_binary-3.3.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:9742580ecc8e1ac45164e98d32ca6df90da509c2d3ff26be245d94c430f92db4", size = 5454896, upload-time = "2025-12-06T17:33:09.023Z" }, @@ -726,32 +662,32 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.1" +version = "0.6.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/e9/01f1a64245b89f039897cb0130016d79f77d52669aae6ee7b159a6c4c018/pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034", size = 145322, upload-time = "2024-09-10T22:41:42.55Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/b6/6e630dff89739fcd427e3f72b3d905ce0acb85a45d4ec3e2678718a3487f/pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b", size = 146586, upload-time = "2026-01-16T18:04:18.534Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", size = 83135, upload-time = "2024-09-11T16:00:36.122Z" }, + { url = "https://files.pythonhosted.org/packages/44/b5/a96872e5184f354da9c84ae119971a0a4c221fe9b27a4d94bd43f2596727/pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", size = 83371, upload-time = "2026-01-16T18:04:17.174Z" }, ] [[package]] name = "pyasn1-modules" -version = "0.4.1" +version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyasn1" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/67/6afbf0d507f73c32d21084a79946bfcfca5fbc62a72057e9c23797a737c9/pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c", size = 310028, upload-time = "2024-09-10T22:42:08.349Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/77/89/bc88a6711935ba795a679ea6ebee07e128050d6382eaa35a0a47c8032bdc/pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", size = 181537, upload-time = "2024-09-11T16:02:10.336Z" }, + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, ] [[package]] name = "pygments" -version = "2.18.0" +version = "2.19.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/62/8336eff65bcbc8e4cb5d05b55faf041285951b6e80f33e2bff2024788f31/pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199", size = 4891905, upload-time = "2024-05-04T13:42:02.013Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a", size = 1205513, upload-time = "2024-05-04T13:41:57.345Z" }, + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] [[package]] @@ -768,29 +704,42 @@ wheels = [ [[package]] name = "pytest" -version = "8.3.3" +version = "9.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, + { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8b/6c/62bbd536103af674e227c41a8f3dcd022d591f6eed5facb5a0f31ee33bbc/pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181", size = 1442487, upload-time = "2024-09-10T10:52:15.003Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/77/7440a06a8ead44c7757a64362dd22df5760f9b12dc5f11b6188cd2fc27a0/pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2", size = 342341, upload-time = "2024-09-10T10:52:12.54Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, ] [[package]] name = "pytest-django" -version = "4.9.0" +version = "4.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/c0/43c8b2528c24d7f1a48a47e3f7381f5ab2ae8c64634b0c3f4bd843063955/pytest_django-4.9.0.tar.gz", hash = "sha256:8bf7bc358c9ae6f6fc51b6cebb190fe20212196e6807121f11bd6a3b03428314", size = 84067, upload-time = "2024-09-02T15:49:18.407Z" } +sdist = { url = "https://files.pythonhosted.org/packages/13/2b/db9a193df89e5660137f5428063bcc2ced7ad790003b26974adf5c5ceb3b/pytest_django-4.12.0.tar.gz", hash = "sha256:df94ec819a83c8979c8f6de13d9cdfbe76e8c21d39473cfe2b40c9fc9be3c758", size = 91156, upload-time = "2026-02-14T18:40:49.235Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/a5/41d091f697c09609e7ef1d5d61925494e0454ebf51de7de05f0f0a728f1d/pytest_django-4.12.0-py3-none-any.whl", hash = "sha256:3ff300c49f8350ba2953b90297d23bf5f589db69545f56f1ec5f8cff5da83e85", size = 26123, upload-time = "2026-02-14T18:40:47.381Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/fe/54f387ee1b41c9ad59e48fb8368a361fad0600fe404315e31a12bacaea7d/pytest_django-4.9.0-py3-none-any.whl", hash = "sha256:1d83692cb39188682dbb419ff0393867e9904094a549a7d38a3154d5731b2b99", size = 23723, upload-time = "2024-09-02T15:49:17.127Z" }, + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] [[package]] @@ -805,51 +754,53 @@ sdist = { url = "https://files.pythonhosted.org/packages/0c/88/8d2797decc42e1c1c [[package]] name = "pyyaml" -version = "6.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, - { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, - { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, - { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, - { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, - { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, - { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, - { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, - { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, - { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, - { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, - { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, - { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, - { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, - { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, - { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, - { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, - { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, - { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, - { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, - { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, - { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, - { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, - { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, - { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, - { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] [[package]] name = "requests" -version = "2.32.3" +version = "2.32.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -857,137 +808,76 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218, upload-time = "2024-05-29T15:37:49.536Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928, upload-time = "2024-05-29T15:37:47.027Z" }, + { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] [[package]] name = "rich" -version = "14.2.0" +version = "14.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fb/d2/8920e102050a0de7bfabeb4c4614a49248cf8d5d7a8d01885fbb24dc767a/rich-14.2.0.tar.gz", hash = "sha256:73ff50c7c0c1c77c8243079283f4edb376f0f6442433aecb8ce7e6d0b92d1fe4", size = 219990, upload-time = "2025-10-09T14:16:53.064Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/99/a4cab2acbb884f80e558b0771e97e21e939c5dfb460f488d19df485e8298/rich-14.3.2.tar.gz", hash = "sha256:e712f11c1a562a11843306f5ed999475f09ac31ffb64281f73ab29ffdda8b3b8", size = 230143, upload-time = "2026-02-01T16:20:47.908Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/25/7a/b0178788f8dc6cafce37a212c99565fa1fe7872c70c6c9c1e1a372d9d88f/rich-14.2.0-py3-none-any.whl", hash = "sha256:76bc51fe2e57d2b1be1f96c524b890b816e334ab4c1e45888799bfaab0021edd", size = 243393, upload-time = "2025-10-09T14:16:51.245Z" }, + { url = "https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl", hash = "sha256:08e67c3e90884651da3239ea668222d19bea7b589149d8014a21c633420dbb69", size = 309963, upload-time = "2026-02-01T16:20:46.078Z" }, ] [[package]] name = "sentry-sdk" -version = "2.48.0" +version = "2.52.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/f0/0e9dc590513d5e742d7799e2038df3a05167cba084c6ca4f3cdd75b55164/sentry_sdk-2.48.0.tar.gz", hash = "sha256:5213190977ff7fdff8a58b722fb807f8d5524a80488626ebeda1b5676c0c1473", size = 384828, upload-time = "2025-12-16T14:55:41.722Z" } +sdist = { url = "https://files.pythonhosted.org/packages/59/eb/1b497650eb564701f9a7b8a95c51b2abe9347ed2c0b290ba78f027ebe4ea/sentry_sdk-2.52.0.tar.gz", hash = "sha256:fa0bec872cfec0302970b2996825723d67390cdd5f0229fb9efed93bd5384899", size = 410273, upload-time = "2026-02-04T15:03:54.706Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/19/8d77f9992e5cbfcaa9133c3bf63b4fbbb051248802e1e803fed5c552fbb2/sentry_sdk-2.48.0-py2.py3-none-any.whl", hash = "sha256:6b12ac256769d41825d9b7518444e57fa35b5642df4c7c5e322af4d2c8721172", size = 414555, upload-time = "2025-12-16T14:55:40.152Z" }, + { url = "https://files.pythonhosted.org/packages/ca/63/2c6daf59d86b1c30600bff679d039f57fd1932af82c43c0bde1cbc55e8d4/sentry_sdk-2.52.0-py2.py3-none-any.whl", hash = "sha256:931c8f86169fc6f2752cb5c4e6480f0d516112e78750c312e081ababecbaf2ed", size = 435547, upload-time = "2026-02-04T15:03:51.567Z" }, ] [[package]] name = "six" -version = "1.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", size = 34041, upload-time = "2021-05-05T14:18:18.379Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254", size = 11053, upload-time = "2021-05-05T14:18:17.237Z" }, -] - -[[package]] -name = "sniffio" -version = "1.3.1" +version = "1.17.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload_time = "2024-02-25T23:20:04.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload_time = "2024-02-25T23:20:01.196Z" }, + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] [[package]] name = "sniffio" version = "1.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload_time = "2024-02-25T23:20:04.057Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload_time = "2024-02-25T23:20:01.196Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] [[package]] name = "sqlparse" -version = "0.5.1" +version = "0.5.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/73/82/dfa23ec2cbed08a801deab02fe7c904bfb00765256b155941d789a338c68/sqlparse-0.5.1.tar.gz", hash = "sha256:bb6b4df465655ef332548e24f08e205afc81b9ab86cb1c45657a7ff173a3a00e", size = 84502, upload-time = "2024-07-15T19:30:27.085Z" } +sdist = { url = "https://files.pythonhosted.org/packages/90/76/437d71068094df0726366574cf3432a4ed754217b436eb7429415cf2d480/sqlparse-0.5.5.tar.gz", hash = "sha256:e20d4a9b0b8585fdf63b10d30066c7c94c5d7a7ec47c889a2d83a3caa93ff28e", size = 120815, upload-time = "2025-12-19T07:17:45.073Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a5/b2860373aa8de1e626b2bdfdd6df4355f0565b47e51f7d0c54fe70faf8fe/sqlparse-0.5.1-py3-none-any.whl", hash = "sha256:773dcbf9a5ab44a090f3441e2180efe2560220203dc2f8c0b0fa141e18b505e4", size = 44156, upload-time = "2024-07-15T19:30:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/49/4b/359f28a903c13438ef59ebeee215fb25da53066db67b305c125f1c6d2a25/sqlparse-0.5.5-py3-none-any.whl", hash = "sha256:12a08b3bf3eec877c519589833aed092e2444e68240a3577e8e26148acc7b1ba", size = 46138, upload-time = "2025-12-19T07:17:46.573Z" }, ] [[package]] name = "structlog" version = "25.5.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] sdist = { url = "https://files.pythonhosted.org/packages/ef/52/9ba0f43b686e7f3ddfeaa78ac3af750292662284b3661e91ad5494f21dbc/structlog-25.5.0.tar.gz", hash = "sha256:098522a3bebed9153d4570c6d0288abf80a031dfdb2048d59a49e9dc2190fc98", size = 1460830, upload-time = "2025-10-27T08:28:23.028Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a8/45/a132b9074aa18e799b891b91ad72133c98d8042c70f6240e4c5f9dabee2f/structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f", size = 72510, upload-time = "2025-10-27T08:28:21.535Z" }, ] -[[package]] -name = "tomli" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/ed/3f73f72945444548f33eba9a87fc7a6e969915e7b1acc8260b30e1f76a2f/tomli-2.3.0.tar.gz", hash = "sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549", size = 17392, upload-time = "2025-10-08T22:01:47.119Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/2e/299f62b401438d5fe1624119c723f5d877acc86a4c2492da405626665f12/tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45", size = 153236, upload-time = "2025-10-08T22:01:00.137Z" }, - { url = "https://files.pythonhosted.org/packages/86/7f/d8fffe6a7aefdb61bced88fcb5e280cfd71e08939da5894161bd71bea022/tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba", size = 148084, upload-time = "2025-10-08T22:01:01.63Z" }, - { url = "https://files.pythonhosted.org/packages/47/5c/24935fb6a2ee63e86d80e4d3b58b222dafaf438c416752c8b58537c8b89a/tomli-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf", size = 234832, upload-time = "2025-10-08T22:01:02.543Z" }, - { url = "https://files.pythonhosted.org/packages/89/da/75dfd804fc11e6612846758a23f13271b76d577e299592b4371a4ca4cd09/tomli-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441", size = 242052, upload-time = "2025-10-08T22:01:03.836Z" }, - { url = "https://files.pythonhosted.org/packages/70/8c/f48ac899f7b3ca7eb13af73bacbc93aec37f9c954df3c08ad96991c8c373/tomli-2.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845", size = 239555, upload-time = "2025-10-08T22:01:04.834Z" }, - { url = "https://files.pythonhosted.org/packages/ba/28/72f8afd73f1d0e7829bfc093f4cb98ce0a40ffc0cc997009ee1ed94ba705/tomli-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c", size = 245128, upload-time = "2025-10-08T22:01:05.84Z" }, - { url = "https://files.pythonhosted.org/packages/b6/eb/a7679c8ac85208706d27436e8d421dfa39d4c914dcf5fa8083a9305f58d9/tomli-2.3.0-cp311-cp311-win32.whl", hash = "sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456", size = 96445, upload-time = "2025-10-08T22:01:06.896Z" }, - { url = "https://files.pythonhosted.org/packages/0a/fe/3d3420c4cb1ad9cb462fb52967080575f15898da97e21cb6f1361d505383/tomli-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be", size = 107165, upload-time = "2025-10-08T22:01:08.107Z" }, - { url = "https://files.pythonhosted.org/packages/ff/b7/40f36368fcabc518bb11c8f06379a0fd631985046c038aca08c6d6a43c6e/tomli-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac", size = 154891, upload-time = "2025-10-08T22:01:09.082Z" }, - { url = "https://files.pythonhosted.org/packages/f9/3f/d9dd692199e3b3aab2e4e4dd948abd0f790d9ded8cd10cbaae276a898434/tomli-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22", size = 148796, upload-time = "2025-10-08T22:01:10.266Z" }, - { url = "https://files.pythonhosted.org/packages/60/83/59bff4996c2cf9f9387a0f5a3394629c7efa5ef16142076a23a90f1955fa/tomli-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f", size = 242121, upload-time = "2025-10-08T22:01:11.332Z" }, - { url = "https://files.pythonhosted.org/packages/45/e5/7c5119ff39de8693d6baab6c0b6dcb556d192c165596e9fc231ea1052041/tomli-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52", size = 250070, upload-time = "2025-10-08T22:01:12.498Z" }, - { url = "https://files.pythonhosted.org/packages/45/12/ad5126d3a278f27e6701abde51d342aa78d06e27ce2bb596a01f7709a5a2/tomli-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8", size = 245859, upload-time = "2025-10-08T22:01:13.551Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a1/4d6865da6a71c603cfe6ad0e6556c73c76548557a8d658f9e3b142df245f/tomli-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6", size = 250296, upload-time = "2025-10-08T22:01:14.614Z" }, - { url = "https://files.pythonhosted.org/packages/a0/b7/a7a7042715d55c9ba6e8b196d65d2cb662578b4d8cd17d882d45322b0d78/tomli-2.3.0-cp312-cp312-win32.whl", hash = "sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876", size = 97124, upload-time = "2025-10-08T22:01:15.629Z" }, - { url = "https://files.pythonhosted.org/packages/06/1e/f22f100db15a68b520664eb3328fb0ae4e90530887928558112c8d1f4515/tomli-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878", size = 107698, upload-time = "2025-10-08T22:01:16.51Z" }, - { url = "https://files.pythonhosted.org/packages/89/48/06ee6eabe4fdd9ecd48bf488f4ac783844fd777f547b8d1b61c11939974e/tomli-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b", size = 154819, upload-time = "2025-10-08T22:01:17.964Z" }, - { url = "https://files.pythonhosted.org/packages/f1/01/88793757d54d8937015c75dcdfb673c65471945f6be98e6a0410fba167ed/tomli-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae", size = 148766, upload-time = "2025-10-08T22:01:18.959Z" }, - { url = "https://files.pythonhosted.org/packages/42/17/5e2c956f0144b812e7e107f94f1cc54af734eb17b5191c0bbfb72de5e93e/tomli-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b", size = 240771, upload-time = "2025-10-08T22:01:20.106Z" }, - { url = "https://files.pythonhosted.org/packages/d5/f4/0fbd014909748706c01d16824eadb0307115f9562a15cbb012cd9b3512c5/tomli-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf", size = 248586, upload-time = "2025-10-08T22:01:21.164Z" }, - { url = "https://files.pythonhosted.org/packages/30/77/fed85e114bde5e81ecf9bc5da0cc69f2914b38f4708c80ae67d0c10180c5/tomli-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f", size = 244792, upload-time = "2025-10-08T22:01:22.417Z" }, - { url = "https://files.pythonhosted.org/packages/55/92/afed3d497f7c186dc71e6ee6d4fcb0acfa5f7d0a1a2878f8beae379ae0cc/tomli-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05", size = 248909, upload-time = "2025-10-08T22:01:23.859Z" }, - { url = "https://files.pythonhosted.org/packages/f8/84/ef50c51b5a9472e7265ce1ffc7f24cd4023d289e109f669bdb1553f6a7c2/tomli-2.3.0-cp313-cp313-win32.whl", hash = "sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606", size = 96946, upload-time = "2025-10-08T22:01:24.893Z" }, - { url = "https://files.pythonhosted.org/packages/b2/b7/718cd1da0884f281f95ccfa3a6cc572d30053cba64603f79d431d3c9b61b/tomli-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999", size = 107705, upload-time = "2025-10-08T22:01:26.153Z" }, - { url = "https://files.pythonhosted.org/packages/19/94/aeafa14a52e16163008060506fcb6aa1949d13548d13752171a755c65611/tomli-2.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e", size = 154244, upload-time = "2025-10-08T22:01:27.06Z" }, - { url = "https://files.pythonhosted.org/packages/db/e4/1e58409aa78eefa47ccd19779fc6f36787edbe7d4cd330eeeedb33a4515b/tomli-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3", size = 148637, upload-time = "2025-10-08T22:01:28.059Z" }, - { url = "https://files.pythonhosted.org/packages/26/b6/d1eccb62f665e44359226811064596dd6a366ea1f985839c566cd61525ae/tomli-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc", size = 241925, upload-time = "2025-10-08T22:01:29.066Z" }, - { url = "https://files.pythonhosted.org/packages/70/91/7cdab9a03e6d3d2bb11beae108da5bdc1c34bdeb06e21163482544ddcc90/tomli-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0", size = 249045, upload-time = "2025-10-08T22:01:31.98Z" }, - { url = "https://files.pythonhosted.org/packages/15/1b/8c26874ed1f6e4f1fcfeb868db8a794cbe9f227299402db58cfcc858766c/tomli-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879", size = 245835, upload-time = "2025-10-08T22:01:32.989Z" }, - { url = "https://files.pythonhosted.org/packages/fd/42/8e3c6a9a4b1a1360c1a2a39f0b972cef2cc9ebd56025168c4137192a9321/tomli-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005", size = 253109, upload-time = "2025-10-08T22:01:34.052Z" }, - { url = "https://files.pythonhosted.org/packages/22/0c/b4da635000a71b5f80130937eeac12e686eefb376b8dee113b4a582bba42/tomli-2.3.0-cp314-cp314-win32.whl", hash = "sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463", size = 97930, upload-time = "2025-10-08T22:01:35.082Z" }, - { url = "https://files.pythonhosted.org/packages/b9/74/cb1abc870a418ae99cd5c9547d6bce30701a954e0e721821df483ef7223c/tomli-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8", size = 107964, upload-time = "2025-10-08T22:01:36.057Z" }, - { url = "https://files.pythonhosted.org/packages/54/78/5c46fff6432a712af9f792944f4fcd7067d8823157949f4e40c56b8b3c83/tomli-2.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77", size = 163065, upload-time = "2025-10-08T22:01:37.27Z" }, - { url = "https://files.pythonhosted.org/packages/39/67/f85d9bd23182f45eca8939cd2bc7050e1f90c41f4a2ecbbd5963a1d1c486/tomli-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf", size = 159088, upload-time = "2025-10-08T22:01:38.235Z" }, - { url = "https://files.pythonhosted.org/packages/26/5a/4b546a0405b9cc0659b399f12b6adb750757baf04250b148d3c5059fc4eb/tomli-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530", size = 268193, upload-time = "2025-10-08T22:01:39.712Z" }, - { url = "https://files.pythonhosted.org/packages/42/4f/2c12a72ae22cf7b59a7fe75b3465b7aba40ea9145d026ba41cb382075b0e/tomli-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b", size = 275488, upload-time = "2025-10-08T22:01:40.773Z" }, - { url = "https://files.pythonhosted.org/packages/92/04/a038d65dbe160c3aa5a624e93ad98111090f6804027d474ba9c37c8ae186/tomli-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67", size = 272669, upload-time = "2025-10-08T22:01:41.824Z" }, - { url = "https://files.pythonhosted.org/packages/be/2f/8b7c60a9d1612a7cbc39ffcca4f21a73bf368a80fc25bccf8253e2563267/tomli-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f", size = 279709, upload-time = "2025-10-08T22:01:43.177Z" }, - { url = "https://files.pythonhosted.org/packages/7e/46/cc36c679f09f27ded940281c38607716c86cf8ba4a518d524e349c8b4874/tomli-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0", size = 107563, upload-time = "2025-10-08T22:01:44.233Z" }, - { url = "https://files.pythonhosted.org/packages/84/ff/426ca8683cf7b753614480484f6437f568fd2fda2edbdf57a2d3d8b27a0b/tomli-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba", size = 119756, upload-time = "2025-10-08T22:01:45.234Z" }, - { url = "https://files.pythonhosted.org/packages/77/b8/0135fadc89e73be292b473cb820b4f5a08197779206b33191e801feeae40/tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b", size = 14408, upload-time = "2025-10-08T22:01:46.04Z" }, -] - [[package]] name = "tox" -version = "4.32.0" +version = "4.35.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cachetools" }, @@ -1000,21 +890,21 @@ dependencies = [ { name = "pyproject-api" }, { name = "virtualenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/59/bf/0e4dbd42724cbae25959f0e34c95d0c730df03ab03f54d52accd9abfc614/tox-4.32.0.tar.gz", hash = "sha256:1ad476b5f4d3679455b89a992849ffc3367560bbc7e9495ee8a3963542e7c8ff", size = 203330, upload-time = "2025-10-24T18:03:38.132Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/7c/d2b9d58c7fe3a36224a71c86fc7072936af126425b7af40ca18103f6f691/tox-4.35.0.tar.gz", hash = "sha256:74d2fe33eb37233d506f854196bd7bd7e2fbb79e8d9b4bed214ab3da98740876", size = 205701, upload-time = "2026-02-12T22:47:29.036Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/cc/e09c0d663a004945f82beecd4f147053567910479314e8d01ba71e5d5dea/tox-4.32.0-py3-none-any.whl", hash = "sha256:451e81dc02ba8d1ed20efd52ee409641ae4b5d5830e008af10fe8823ef1bd551", size = 175905, upload-time = "2025-10-24T18:03:36.337Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5f/8df349c4e9ea0747cfc12a44c2e952c2f7a1a12fb54f36543dae17638a57/tox-4.35.0-py3-none-any.whl", hash = "sha256:282aa2e1f96328ad197ee09878ff241610426cd8ec01e62a04eb51c987da922d", size = 176999, upload-time = "2026-02-12T22:47:27.768Z" }, ] [[package]] name = "tox-gh-actions" -version = "3.2.0" +version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tox" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5b/a1/3e7b3185031f905ddfc0c65dfd85949dae612a4387cbacb286b9f2931314/tox-gh-actions-3.2.0.tar.gz", hash = "sha256:ac6fa3b8da51bc90dd77985fd55f09e746c6558c55910c0a93d643045a2b0ccc", size = 18834, upload-time = "2024-01-03T13:58:36.114Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/11/6c3f818887c37a144a4b48bfc440150f006bdac7a68d92de1046680f578f/tox_gh_actions-3.5.0.tar.gz", hash = "sha256:cc8e148c4513042e5019973e5672594c3df241b035e7fb550f6f778588110051", size = 18815, upload-time = "2025-10-22T14:12:24.846Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/7a/d59f4eb08d156787af19fb44371586fd8b25580a1837ce92ae203a847c49/tox_gh_actions-3.2.0-py2.py3-none-any.whl", hash = "sha256:821b66a4751a788fa3e9617bd796d696507b08c6e1d929ee4faefba06b73b694", size = 9951, upload-time = "2024-01-03T13:58:33.77Z" }, + { url = "https://files.pythonhosted.org/packages/fd/9e/8d50f3b3fc4af8c73154f64d4a2293bfa2d517a19000e70ef2d614254084/tox_gh_actions-3.5.0-py3-none-any.whl", hash = "sha256:070790114c92f4c94337047515ca5e077ceb269a5cb9366e0a0b3440a024eca1", size = 9910, upload-time = "2025-10-22T14:12:23.405Z" }, ] [[package]] @@ -1023,37 +913,24 @@ version = "1.29.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "tox" }, { name = "uv" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4f/90/06752775b8cfadba8856190f5beae9f552547e0f287e0246677972107375/tox_uv-1.29.0.tar.gz", hash = "sha256:30fa9e6ad507df49d3c6a2f88894256bcf90f18e240a00764da6ecab1db24895", size = 23427, upload-time = "2025-10-09T20:40:27.384Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/8e/94afb25547f5e4987801e8f6aa11e357190f72f31eb363267a3cb2fa6a88/tox_uv-1.13.1-py3-none-any.whl", hash = "sha256:b163dd28ca37a9f4c6d8cbac11153be27c2e929b58bcae62e323ffa8f71c327d", size = 13383, upload-time = "2024-10-11T16:14:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/5c/17/221d62937c4130b044bb437caac4181e7e13d5536bbede65264db1f0ac9f/tox_uv-1.29.0-py3-none-any.whl", hash = "sha256:b1d251286edeeb4bc4af1e24c8acfdd9404700143c2199ccdbb4ea195f7de6cc", size = 17254, upload-time = "2025-10-09T20:40:25.885Z" }, ] [[package]] name = "treetop-client" -version = "0.0.2" +version = "0.0.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fa/a1/ae06eb362cdc44ccf2bea1c2af6856e038cebac63ceb4908434482e1323a/treetop_client-0.0.2.tar.gz", hash = "sha256:c89794d09fc9c31d39cb0e13ee03f1559bbe2ec893a0647698f48747a1759f8f", size = 5103, upload_time = "2025-07-28T19:34:36.677Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/00/44a4a3f8957c9a490fc4033226f49ae8c6cf0e072d43a53985683b335eb5/treetop_client-0.0.7.tar.gz", hash = "sha256:01b0d8dde4f2830ec9556d3373b01a4c09f1048145deb9781c00d8d3515df38f", size = 28056, upload-time = "2026-02-12T12:47:52.203Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/e4/9587921902ece105f616310b66cae9e4f40e38c8673d5e74ff2a66b61d57/treetop_client-0.0.2-py3-none-any.whl", hash = "sha256:af08d71d9d74a3913c7655fe3a5e1dc579e335ddecf562710ca2443b5227d16a", size = 6108, upload_time = "2025-07-28T19:34:35.405Z" }, -] - -[[package]] -name = "treetop-client" -version = "0.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/87/2e/1920bff4d2f621cbab9d97a54eff9bc71a3a4c9eaabf16a4164a56ac7565/treetop_client-0.0.3.tar.gz", hash = "sha256:863b3a7f01e794e03674b4c7e011655c753432621c71e2f909ab966f4a7b5dda", size = 5409, upload_time = "2025-09-01T10:35:26.513Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/c6/77e4dd3519bab99832d7b45869d57230f1028352c5a6942b7c33484b83e3/treetop_client-0.0.3-py3-none-any.whl", hash = "sha256:cf294f607401755ef02813a26cb8228fdba06835d1081ee43412e53fe4a8a21c", size = 6443, upload_time = "2025-09-01T10:35:25.446Z" }, + { url = "https://files.pythonhosted.org/packages/60/0c/ea9f569072ff9fcad1025fffa4876b6cc2c8eecacec838e7d7acce724ed3/treetop_client-0.0.7-py3-none-any.whl", hash = "sha256:4c158a8563a8438e4dc2a5e79965be572aa4c350b8a12e513691b94bae017ada", size = 10058, upload-time = "2026-02-12T12:47:53.692Z" }, ] [[package]] @@ -1076,20 +953,20 @@ wheels = [ [[package]] name = "unittest-parametrize" -version = "1.4.0" +version = "1.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f5/40/6311475f636738bc10726559cc42b271c11f5a7574f1c2c5aa75fac5d221/unittest_parametrize-1.4.0.tar.gz", hash = "sha256:f9dcaed821ad19724964a49e7f595311b9a2c5253c61cdb1fb2c01b899af67a2", size = 12546, upload-time = "2023-10-12T10:00:25.999Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/5a/32bb766239d938f6030dd3920387dedc30b1c89aca01cb5281c3075bdf87/unittest_parametrize-1.8.0.tar.gz", hash = "sha256:90818ddeb28648e795eb53983f42d44e273b26ab03a1db49c27f6830ed419236", size = 15060, upload-time = "2025-09-09T11:04:26.317Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/0e/64feadbd3ced20c1da29aac9dc32b9df2a730cd34618b204d22f5673ce7f/unittest_parametrize-1.4.0-py3-none-any.whl", hash = "sha256:8b9554fbcfd36bfef6c5c522b9146f13647eb4ec589c17af7f67f22793cecb82", size = 8118, upload-time = "2023-10-12T10:00:24.353Z" }, + { url = "https://files.pythonhosted.org/packages/ab/62/193690ddc422c7af943b3f371ef34742dbb6a061e0b348e356399a52004d/unittest_parametrize-1.8.0-py3-none-any.whl", hash = "sha256:10a580aa04d13cf4f1c8e5e07be5694392b7ecef3b4d585bfc2bddce65bc3275", size = 9261, upload-time = "2025-09-09T11:04:25.025Z" }, ] [[package]] name = "uritemplate" -version = "4.1.1" +version = "4.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d2/5a/4742fdba39cd02a56226815abfa72fe0aa81c33bed16ed045647d6000eba/uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", size = 273898, upload-time = "2021-10-13T11:15:14.84Z" } +sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/c0/7461b49cd25aeece13766f02ee576d1db528f1c37ce69aee300e075b485b/uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e", size = 10356, upload-time = "2021-10-13T11:15:12.316Z" }, + { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, ] [[package]] @@ -1103,41 +980,39 @@ wheels = [ [[package]] name = "uv" -version = "0.9.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/03/1afff9e6362dc9d3a9e03743da0a4b4c7a0809f859c79eb52bbae31ea582/uv-0.9.18.tar.gz", hash = "sha256:17b5502f7689c4dc1fdeee9d8437a9a6664dcaa8476e70046b5f4753559533f5", size = 3824466, upload-time = "2025-12-16T15:45:11.81Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/9c/92fad10fcee8ea170b66442d95fd2af308fe9a107909ded4b3cc384fdc69/uv-0.9.18-py3-none-linux_armv6l.whl", hash = "sha256:e9e4915bb280c1f79b9a1c16021e79f61ed7c6382856ceaa99d53258cb0b4951", size = 21345538, upload-time = "2025-12-16T15:45:13.992Z" }, - { url = "https://files.pythonhosted.org/packages/81/b1/b0e5808e05acb54aa118c625d9f7b117df614703b0cbb89d419d03d117f3/uv-0.9.18-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d91abfd2649987996e3778729140c305ef0f6ff5909f55aac35c3c372544a24f", size = 20439572, upload-time = "2025-12-16T15:45:26.397Z" }, - { url = "https://files.pythonhosted.org/packages/b7/0b/9487d83adf5b7fd1e20ced33f78adf84cb18239c3d7e91f224cedba46c08/uv-0.9.18-py3-none-macosx_11_0_arm64.whl", hash = "sha256:cf33f4146fd97e94cdebe6afc5122208eea8c55b65ca4127f5a5643c9717c8b8", size = 18952907, upload-time = "2025-12-16T15:44:48.399Z" }, - { url = "https://files.pythonhosted.org/packages/58/92/c8f7ae8900eff8e4ce1f7826d2e1e2ad5a95a5f141abdb539865aff79930/uv-0.9.18-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:edf965e9a5c55f74020ac82285eb0dfe7fac4f325ad0a7afc816290269ecfec1", size = 20772495, upload-time = "2025-12-16T15:45:29.614Z" }, - { url = "https://files.pythonhosted.org/packages/5a/28/9831500317c1dd6cde5099e3eb3b22b88ac75e47df7b502f6aef4df5750e/uv-0.9.18-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae10a941bd7ca1ee69edbe3998c34dce0a9fc2d2406d98198343daf7d2078493", size = 20949623, upload-time = "2025-12-16T15:44:57.482Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ff/1fe1ffa69c8910e54dd11f01fb0765d4fd537ceaeb0c05fa584b6b635b82/uv-0.9.18-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1669a95b588f613b13dd10e08ced6d5bcd79169bba29a2240eee87532648790", size = 21920580, upload-time = "2025-12-16T15:44:39.009Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ee/eed3ec7679ee80e16316cfc95ed28ef6851700bcc66edacfc583cbd2cc47/uv-0.9.18-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:11e1e406590d3159138288203a41ff8a8904600b8628a57462f04ff87d62c477", size = 23491234, upload-time = "2025-12-16T15:45:32.59Z" }, - { url = "https://files.pythonhosted.org/packages/78/58/64b15df743c79ad03ea7fbcbd27b146ba16a116c57f557425dd4e44d6684/uv-0.9.18-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e82078d3c622cb4c60da87f156168ffa78b9911136db7ffeb8e5b0a040bf30e", size = 23095438, upload-time = "2025-12-16T15:45:17.916Z" }, - { url = "https://files.pythonhosted.org/packages/43/6d/3d3dae71796961603c3871699e10d6b9de2e65a3c327b58d4750610a5f93/uv-0.9.18-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704abaf6e76b4d293fc1f24bef2c289021f1df0de9ed351f476cbbf67a7edae0", size = 22140992, upload-time = "2025-12-16T15:44:45.527Z" }, - { url = "https://files.pythonhosted.org/packages/31/91/1042d0966a30e937df500daed63e1f61018714406ce4023c8a6e6d2dcf7c/uv-0.9.18-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3332188fd8d96a68e5001409a52156dced910bf1bc41ec3066534cffcd46eb68", size = 22229626, upload-time = "2025-12-16T15:45:20.712Z" }, - { url = "https://files.pythonhosted.org/packages/5a/1f/0a4a979bb2bf6e1292cc57882955bf1d7757cad40b1862d524c59c2a2ad8/uv-0.9.18-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:b7295e6d505f1fd61c54b1219e3b18e11907396333a9fa61cefe489c08fc7995", size = 20896524, upload-time = "2025-12-16T15:45:06.799Z" }, - { url = "https://files.pythonhosted.org/packages/a5/3c/24f92e56af00cac7d9bed2888d99a580f8093c8745395ccf6213bfccf20b/uv-0.9.18-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:62ea0e518dd4ab76e6f06c0f43a25898a6342a3ecf996c12f27f08eb801ef7f1", size = 22077340, upload-time = "2025-12-16T15:44:51.271Z" }, - { url = "https://files.pythonhosted.org/packages/9c/3e/73163116f748800e676bf30cee838448e74ac4cc2f716c750e1705bc3fe4/uv-0.9.18-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:8bd073e30030211ba01206caa57b4d63714e1adee2c76a1678987dd52f72d44d", size = 20932956, upload-time = "2025-12-16T15:45:00.3Z" }, - { url = "https://files.pythonhosted.org/packages/59/1b/a26990b51a17de1ffe41fbf2e30de3a98f0e0bce40cc60829fb9d9ed1a8a/uv-0.9.18-py3-none-musllinux_1_1_i686.whl", hash = "sha256:f248e013d10e1fc7a41f94310628b4a8130886b6d683c7c85c42b5b36d1bcd02", size = 21357247, upload-time = "2025-12-16T15:45:23.575Z" }, - { url = "https://files.pythonhosted.org/packages/5f/20/b6ba14fdd671e9237b22060d7422aba4a34503e3e42d914dbf925eff19aa/uv-0.9.18-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:17bedf2b0791e87d889e1c7f125bd5de77e4b7579aec372fa06ba832e07c957e", size = 22443585, upload-time = "2025-12-16T15:44:42.213Z" }, - { url = "https://files.pythonhosted.org/packages/5e/da/1b3dd596964f90a122cfe94dcf5b6b89cf5670eb84434b8c23864382576f/uv-0.9.18-py3-none-win32.whl", hash = "sha256:de6f0bb3e9c18e484545bd1549ec3c956968a141a393d42e2efb25281cb62787", size = 20091088, upload-time = "2025-12-16T15:45:03.225Z" }, - { url = "https://files.pythonhosted.org/packages/11/0b/50e13ebc1eedb36d88524b7740f78351be33213073e3faf81ac8925d0c6e/uv-0.9.18-py3-none-win_amd64.whl", hash = "sha256:c82b0e2e36b33e2146fba5f0ae6906b9679b3b5fe6a712e5d624e45e441e58e9", size = 22181193, upload-time = "2025-12-16T15:44:54.394Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d4/0bf338d863a3d9e5545e268d77a8e6afdd75d26bffc939603042f2e739f9/uv-0.9.18-py3-none-win_arm64.whl", hash = "sha256:4c4ce0ed080440bbda2377488575d426867f94f5922323af6d4728a1cd4d091d", size = 20564933, upload-time = "2025-12-16T15:45:09.819Z" }, +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/9a/fe74aa0127cdc26141364e07abf25e5d69b4bf9788758fad9cfecca637aa/uv-0.10.2.tar.gz", hash = "sha256:b5016f038e191cc9ef00e17be802f44363d1b1cc3ef3454d1d76839a4246c10a", size = 3858864, upload-time = "2026-02-10T19:17:51.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/b5/aea88f66284d220be56ef748ed5e1bd11d819be14656a38631f4b55bfd48/uv-0.10.2-py3-none-linux_armv6l.whl", hash = "sha256:69e35aa3e91a245b015365e5e6ca383ecf72a07280c6d00c17c9173f2d3b68ab", size = 22215714, upload-time = "2026-02-10T19:17:34.281Z" }, + { url = "https://files.pythonhosted.org/packages/7f/72/947ba7737ae6cd50de61d268781b9e7717caa3b07e18238ffd547f9fc728/uv-0.10.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0b7eef95c36fe92e7aac399c0dce555474432cbfeaaa23975ed83a63923f78fd", size = 21276485, upload-time = "2026-02-10T19:18:15.415Z" }, + { url = "https://files.pythonhosted.org/packages/d3/38/5c3462b927a93be4ccaaa25138926a5fb6c9e1b72884efd7af77e451d82e/uv-0.10.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:acc08e420abab21de987151059991e3f04bc7f4044d94ca58b5dd547995b4843", size = 20048620, upload-time = "2026-02-10T19:17:26.481Z" }, + { url = "https://files.pythonhosted.org/packages/03/51/d4509b0f5b7740c1af82202e9c69b700d5848b8bd0faa25229e8edd2c19c/uv-0.10.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:aefbcd749ab2ad48bb533ec028607607f7b03be11c83ea152dbb847226cd6285", size = 21870454, upload-time = "2026-02-10T19:17:21.838Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7e/2bcbafcb424bb885817a7e58e6eec9314c190c55935daaafab1858bb82cd/uv-0.10.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:fad554c38d9988409ceddfac69a465e6e5f925a8b689e7606a395c20bb4d1d78", size = 21839508, upload-time = "2026-02-10T19:17:59.211Z" }, + { url = "https://files.pythonhosted.org/packages/60/08/16df2c1f8ad121a595316b82f6e381447e8974265b2239c9135eb874f33b/uv-0.10.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6dd2dc41043e92b3316d7124a7bf48c2affe7117c93079419146f083df71933c", size = 21841283, upload-time = "2026-02-10T19:17:41.419Z" }, + { url = "https://files.pythonhosted.org/packages/76/27/a869fec4c03af5e43db700fabe208d8ee8dbd56e0ff568ba792788d505cd/uv-0.10.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111c05182c5630ac523764e0ec2e58d7b54eb149dbe517b578993a13c2f71aff", size = 23111967, upload-time = "2026-02-10T19:18:11.764Z" }, + { url = "https://files.pythonhosted.org/packages/2a/4a/fb38515d966acfbd80179e626985aab627898ffd02c70205850d6eb44df1/uv-0.10.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45c3deaba0343fd27ab5385d6b7cde0765df1a15389ee7978b14a51c32895662", size = 23911019, upload-time = "2026-02-10T19:18:26.947Z" }, + { url = "https://files.pythonhosted.org/packages/dd/5f/51bcbb490ddb1dcb06d767f0bde649ad2826686b9e30efa57f8ab2750a1d/uv-0.10.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bb2cac4f3be60b64a23d9f035019c30a004d378b563c94f60525c9591665a56b", size = 23030217, upload-time = "2026-02-10T19:17:37.789Z" }, + { url = "https://files.pythonhosted.org/packages/46/69/144f6db851d49aa6f25b040dc5c8c684b8f92df9e8d452c7abc619c6ec23/uv-0.10.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937687df0380d636ceafcb728cf6357f0432588e721892128985417b283c3b54", size = 23036452, upload-time = "2026-02-10T19:18:18.97Z" }, + { url = "https://files.pythonhosted.org/packages/66/29/3c7c4559c9310ed478e3d6c585ee0aad2852dc4d5fb14f4d92a2a12d1728/uv-0.10.2-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:f90bca8703ae66bccfcfb7313b4b697a496c4d3df662f4a1a2696a6320c47598", size = 21941903, upload-time = "2026-02-10T19:17:30.575Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5a/42883b5ef2ef0b1bc5b70a1da12a6854a929ff824aa8eb1a5571fb27a39b/uv-0.10.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:cca026c2e584788e1264879a123bf499dd8f169b9cafac4a2065a416e09d3823", size = 22651571, upload-time = "2026-02-10T19:18:22.74Z" }, + { url = "https://files.pythonhosted.org/packages/e8/b8/e4f1dda1b3b0cc6c8ac06952bfe7bc28893ff016fb87651c8fafc6dfca96/uv-0.10.2-py3-none-musllinux_1_1_i686.whl", hash = "sha256:9f878837938103ee1307ed3ed5d9228118e3932816ab0deb451e7e16dc8ce82a", size = 22321279, upload-time = "2026-02-10T19:17:49.402Z" }, + { url = "https://files.pythonhosted.org/packages/2c/4b/baa16d46469e024846fc1a8aa0cfa63f1f89ad0fd3eaa985359a168c3fb0/uv-0.10.2-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:6ec75cfe638b316b329474aa798c3988e5946ead4d9e977fe4dc6fc2ea3e0b8b", size = 23252208, upload-time = "2026-02-10T19:17:54.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/6a74e5ec2ee90e4314905e6d1d1708d473e06405e492ec38868b42645388/uv-0.10.2-py3-none-win32.whl", hash = "sha256:f7f3c7e09bf53b81f55730a67dd86299158f470dffb2bd279b6432feb198d231", size = 21118543, upload-time = "2026-02-10T19:18:07.296Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f9/e5cc6cf3a578b87004e857274df97d3cdecd8e19e965869b9b67c094c20c/uv-0.10.2-py3-none-win_amd64.whl", hash = "sha256:7b3685aa1da15acbe080b4cba8684afbb6baf11c9b04d4d4b347cc18b7b9cfa0", size = 23620790, upload-time = "2026-02-10T19:17:45.204Z" }, + { url = "https://files.pythonhosted.org/packages/df/7a/99979dc08ae6a65f4f7a44c5066699016c6eecdc4e695b7512c2efb53378/uv-0.10.2-py3-none-win_arm64.whl", hash = "sha256:abdd5b3c6b871b17bf852a90346eb7af881345706554fd082346b000a9393afd", size = 22035199, upload-time = "2026-02-10T19:18:03.679Z" }, ] [[package]] name = "virtualenv" -version = "20.35.4" +version = "20.36.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, { name = "filelock" }, { name = "platformdirs" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/28/e6f1a6f655d620846bd9df527390ecc26b3805a0c5989048c210e22c5ca9/virtualenv-20.35.4.tar.gz", hash = "sha256:643d3914d73d3eeb0c552cbb12d7e82adf0e504dbf86a3182f8771a153a1971c", size = 6028799, upload-time = "2025-10-29T06:57:40.511Z" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/a3/4d310fa5f00863544e1d0f4de93bddec248499ccf97d4791bc3122c9d4f3/virtualenv-20.36.1.tar.gz", hash = "sha256:8befb5c81842c641f8ee658481e42641c68b5eab3521d8e092d18320902466ba", size = 6032239, upload-time = "2026-01-09T18:21:01.296Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/0c/c05523fa3181fdf0c9c52a6ba91a23fbf3246cc095f26f6516f9c60e6771/virtualenv-20.35.4-py3-none-any.whl", hash = "sha256:c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b", size = 6005095, upload-time = "2025-10-29T06:57:37.598Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2a/dc2228b2888f51192c7dc766106cd475f1b768c10caaf9727659726f7391/virtualenv-20.36.1-py3-none-any.whl", hash = "sha256:575a8d6b124ef88f6f51d56d656132389f961062a9177016a50e4f507bbcc19f", size = 6008258, upload-time = "2026-01-09T18:20:59.425Z" }, ] From 0f62e743caf19f34ef66d25a801346515a86cdb6 Mon Sep 17 00:00:00 2001 From: pederhan Date: Fri, 9 Jan 2026 13:01:10 +0100 Subject: [PATCH 05/34] Bump version to 1.4.0 From 283e06513d6b5351c435b264383f16e00d622cd5 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Thu, 29 Jan 2026 11:29:30 +0100 Subject: [PATCH 06/34] Add Prometheus metrics middleware and expose metrics endpoint (#605) * Add Prometheus metrics middleware and expose metrics endpoint - Implement PrometheusRequestMiddleware to track a selection of HTTP, DB, and LDAP metrics. - Create MetricsView to serve metrics at /api/meta/metrics. - Update URL routing to include metrics endpoint. - Add tests for metrics endpoint and metrics recording. - Include prometheus-client dependency in project configuration. --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9de8dd5c..6dec75d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dev = [ "coverage[toml]", "pytest", "pytest-django", - "uv>=0.9", + "uv>=0.10", ] ci = [ # Explictly include dev group for non-uv package managers diff --git a/uv.lock b/uv.lock index 445c38a4..c6484f77 100644 --- a/uv.lock +++ b/uv.lock @@ -523,14 +523,14 @@ ci = [ { name = "pytest-django" }, { name = "tox-gh-actions" }, { name = "tox-uv", specifier = ">=1.29" }, - { name = "uv", specifier = ">=0.9" }, + { name = "uv", specifier = ">=0.10" }, ] dev = [ { name = "coverage", extras = ["toml"] }, { name = "pytest" }, { name = "pytest-django" }, { name = "tox-uv", specifier = ">=1.29" }, - { name = "uv", specifier = ">=0.9" }, + { name = "uv", specifier = ">=0.10" }, ] [[package]] From 9c2e3a0c0d4a4aa84105aeb85dca50389e82777e Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Mon, 11 Aug 2025 12:41:05 +0200 Subject: [PATCH 07/34] Initial commit to the branch. --- mreg/api/permissions.py | 21 ++++++- mreg/api/treetop.py | 109 +++++++------------------------------ pyproject.toml | 2 +- treetop/data/mreg.cedar | 62 ++++++++++++--------- treetop/docker-compose.yml | 1 - 5 files changed, 77 insertions(+), 118 deletions(-) diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index a2be74cd..3b0d4353 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -384,7 +384,16 @@ def has_permission(self, request, view): if not super().has_permission(request, view): return False - return self.user_is_superuser(request=request, view=view) + + return policy_parity( + User.from_request(request).is_mreg_superuser, + request=request, + view=view, + permission_class=self.__class__.__name__, + action="is_superuser", + resource_kind="Generic", + resource_attrs={"kind": "Any", "id": "any"}, + ) class IsSuperOrAdminOrReadOnly(IsAuthenticated): @@ -397,7 +406,15 @@ def has_permission(self, request, view): return False if request.method in SAFE_METHODS: return True - return self.user_is_admin(request=request, view=view) + return policy_parity( + User.from_request(request).is_mreg_superuser_or_admin, + request=request, + view=view, + permission_class=self.__class__.__name__, + action="is_admin", # Superadmins don't care what the action is + resource_kind="Generic", + resource_attrs={"kind": "Any", "id": "any"}, + ) diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index 77f87088..645d35f0 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -1,10 +1,6 @@ from __future__ import annotations import logging -from typing import Any, Optional, Mapping -import ipaddress -import json -import threading -from contextlib import contextmanager +from typing import Any, Optional from django.conf import settings from rest_framework.request import Request @@ -13,13 +9,10 @@ from mreg.models.auth import User as MregUser # your request->user wrapper from treetop_client.client import TreeTopClient -from treetop_client.models import Request as TreeTopRequest, User as TreeTopUser, Action, Resource, ResourceAttribute, ResourceAttributeType +from treetop_client.models import Request as TreeTopRequest, User as TreeTopUser, Action as TreeTopAction, Resource as TreeTopResource logger = logging.getLogger("mreg.policy.parity") -# Thread-local storage for parity checking bypass flag -_thread_local = threading.local() - # Configure these in settings.py POLICY_PARITY_ENABLED = getattr(settings, "POLICY_PARITY_ENABLED", True) POLICY_BASE_URL = getattr(settings, "POLICY_BASE_URL", "http://localhost:9999") @@ -33,34 +26,6 @@ treetopclient = TreeTopClient(base_url=POLICY_BASE_URL) -@contextmanager -def disable_policy_parity(): - """Context manager to temporarily disable policy parity checking. - - Useful for tests that modify permissions/state mid-test, which would - cause the legacy and policy systems to be out of sync. - - Example: - def test_permission_changes(self): - with disable_policy_parity(): - # Modify permissions here - user.groups.add(some_group) - # Make API calls - parity checking will be skipped - """ - old_value = getattr(_thread_local, "skip_parity", False) - _thread_local.skip_parity = True - try: - yield - finally: - _thread_local.skip_parity = old_value - -def _is_parity_enabled() -> bool: - """Check if parity checking should be performed in current context.""" - if not POLICY_PARITY_ENABLED: - return False - # Skip parity checking if we're in a disabled context - return not getattr(_thread_local, "skip_parity", False) - def _corr_id(request: Request) -> Optional[str]: return request.headers.get("X-Correlation-ID") or request.META.get("HTTP_X_CORRELATION_ID") @@ -80,54 +45,31 @@ def policy_parity( permission_class: Optional[str] = None, action: str, resource_kind: str, - resource_id: str, - resource_attrs: Mapping[str, str], + resource_attrs: dict[str, Any], ) -> bool: """ Log legacy-vs-policy parity and return `decision` unchanged. Use this anywhere you currently 'return True/False'. """ - if not _is_parity_enabled(): + if not POLICY_PARITY_ENABLED: return decision # Build policy request muser = MregUser.from_request(request) - principal = TreeTopUser.new(str(muser.username), POLICY_NAMESPACE, groups=list(muser.group_list)) - pol_action = Action.new(action, POLICY_NAMESPACE) - - attrs = {} - - for k, v in resource_attrs.items(): - try: - ip = ipaddress.ip_address(v) - attrs[k] = ResourceAttribute.new(str(ip), ResourceAttributeType.IP) - except ValueError: - attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) -# if v.isdigit(): -# attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.NUMBER) -# elif v.lower() in ("true", "false"): -# attrs[k] = ResourceAttribute.new(v.lower(), ResourceAttributeType.BOOLEAN) -# else: -# attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) - - res = Resource.new(str(resource_kind), resource_id, attrs=attrs) - - if len(pol_action.id.namespace) > 0: - fully_qualified_action = "::".join(pol_action.id.namespace) + f"::{pol_action.id.id}" - else: - fully_qualified_action = f"{pol_action.id.id}" + principal = TreeTopUser.new(muser.username, POLICY_NAMESPACE, groups=list(muser.group_list)) + pol_action = TreeTopAction.new(action, POLICY_NAMESPACE) + res = TreeTopResource.new(resource_kind, resource_attrs) context = { "path": request.path, "method": request.method, "permission": permission_class or (view and view.__class__.__name__), "view": view and view.__class__.__name__, - "model": _model_name_from_view(view), + "resource_kind": resource_kind, + "action": getattr(pol_action, "name", str(pol_action)), "principal": muser.username, "groups": list(muser.group_list), - "action": fully_qualified_action, - "resource_kind": resource_kind, - "resource_attrs": resource_attrs, + "model": _model_name_from_view(view), "correlation_id": _corr_id(request), } @@ -137,18 +79,6 @@ def policy_parity( pol_allowed = bool(resp.is_allowed()) except Exception as exc: error = repr(exc) - # Log policy server errors prominently - logger.error( - f"Policy server error: {type(exc).__name__}: {exc}", - extra={ - "error_type": type(exc).__name__, - "error_msg": str(exc), - "path": request.path, - "correlation_id": _corr_id(request), - }, - ) - # If policy server fails, we cannot determine parity. Return legacy decision - # but flag this in the payload for monitoring. parity = False if bool(decision) and pol_allowed: @@ -156,24 +86,25 @@ def policy_parity( elif not bool(decision) and not pol_allowed: parity = True - payload: dict[str, Any] = { - "parity": parity, + payload: dict[str, object] = { + **context, "legacy_decision": bool(decision), "policy_decision": pol_allowed, + "parity": parity, + "resource_attrs": resource_attrs, "error": error, - "context": context, } if parity: - logger.info("policy_parity_ok", extra=payload) - log_policy_parity(payload) - else: logger.warning("policy_parity_mismatch", extra=payload) - log_policy_parity(payload) + log_policy_parity("OK", payload) + else: + logger.info("policy_parity_ok", extra=payload) + log_policy_parity("MISMATCH", payload) return decision # Log data to a file in addition to normal logging -def log_policy_parity(payload: dict[str, Any]): +def log_policy_parity(result: str, payload: dict[str, Any]): with open(POLICY_EXTRA_LOG_FILE_NAME, "a") as log_file: - log_file.write(f"{json.dumps(payload)}\n") + log_file.write(f"{result}: {payload}\n") diff --git a/pyproject.toml b/pyproject.toml index 6dec75d2..8a280eff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "pyyaml", # For testing inside Docker image "unittest-parametrize", - "treetop-client>=0.0.7", + "treetop-client>=0.0.7", "prometheus-client>=0.24", ] dynamic = ["version"] diff --git a/treetop/data/mreg.cedar b/treetop/data/mreg.cedar index 33bc5fe8..20d19ebe 100644 --- a/treetop/data/mreg.cedar +++ b/treetop/data/mreg.cedar @@ -3,7 +3,11 @@ @id("MREG.admins_policy") permit ( principal in MREG::Group::"admins", - action in MREG::Action::"host_access", + action in + [MREG::Action::"create_host", + MREG::Action::"delete_host", + MREG::Action::"view_host", + MREG::Action::"edit_host"], resource is Host ); @@ -12,15 +16,34 @@ permit ( @id("MREG.webadmins_policy") permit ( principal in MREG::Group::"webadmins", - action in MREG::Action::"host_access", + action in + [MREG::Action::"edit_host", + MREG::Action::"delete_host", + MREG::Action::"create_host"], resource is Host ) when { resource.nameLabels.contains("webserver") && - resource.ip.isInRange(ip("192.168.1.0/24")) + resource.ip.isInRange("192.168.1.0/24") }; +// Users can only view hosts +@id("MREG.users_policy") +permit ( + principal in MREG::Group::"users", + action == MREG::Action::"view_host", + resource is Host +); + +// Charlie does not get to delete hosts, no matter what. +@id("MREG.charlie_forbid_delete_host_policy") +forbid ( + principal == MREG::User::"charlie", + action == MREG::Action::"delete_host", + resource is Host +); + // Admins can manipulate any IP address, even if it is a gw, a broadcast address, // the network address, reserved. These three groups are unified as "restricted" IPs. @id("MREG.admins_ip_policy") @@ -35,17 +58,6 @@ permit ( resource is IPAddress ); - -/// Test group access, used during testing. -@id("MREG.test_group_policy") -permit ( - principal in MREG::Group::"testgroup", - action in [MREG::Action::"host_access"], - resource is Host -) when { - resource.ip.isInRange(ip("10.0.0.0/24")) -}; - /// Network Admins can manage any IP in any network @id("MREG.network_admins_ip_network_policy") permit ( @@ -63,8 +75,8 @@ permit ( ) when { - resource.ip.isInRange(ip("192.168.1.0/24")) || - resource.ip.isInRange(ip("10.0.0.0/8")) + resource.ip.isInRange("192.168.1.0/24") || + resource.ip.isInRange("10.0.0.0/8") }; /// Admins can do whatever with labels. @@ -80,7 +92,7 @@ permit ( ); /// Superadmins -@id("MREG.superadmin") +@id("MREG.is_superuser") permit ( principal in MREG::Group::"default-super-group", action, // Superadmins don't care what the action is @@ -88,34 +100,34 @@ permit ( ); /// Normal (?) admins -@id("MREG.admin") +@id("MREG.is_admin") permit ( principal in MREG::Group::"default-admin-group", - action == MREG::Action::"admin_access", + action == MREG::Action::"is_admin", resource ); /// Host Policy Admins -@id("MREG.hostpolicy_admin") +@id("MREG.is_hostpolicy_admin") permit ( principal in MREG::Group::"default-hostpolicyadmin-group", - action == MREG::Action::"hostpolicy_admin_access", + action == MREG::Action::"is_hostpolicy_admin", resource ); /// DNS Wildcard Admins -@id("MREG.dns_wildcard_admin") +@id("MREG.is_dns_wildcard_admin") permit ( principal in MREG::Group::"default-dns-wildcard-group", - action == MREG::Action::"dns_wildcard_admin_access", + action == MREG::Action::"is_dns_wildcard_admin", resource ); /// DNS Underscore Admins -@id("MREG.dns_underscore_admin") +@id("MREG.is_dns_underscore_admin") permit ( principal in MREG::Group::"default-dns-underscore-group", - action == MREG::Action::"dns_underscore_admin_access", + action == MREG::Action::"is_dns_underscore_admin", resource ); diff --git a/treetop/docker-compose.yml b/treetop/docker-compose.yml index 381e1fa6..fc7f817e 100644 --- a/treetop/docker-compose.yml +++ b/treetop/docker-compose.yml @@ -10,7 +10,6 @@ services: treetop-server: image: ghcr.io/terjekv/treetop-rest:develop - pull_policy: "always" ports: - "9999:9999" environment: From fb00fd6d1aa51bfef2a9cbc4bb772e683cbea710 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 12 Nov 2025 11:40:26 +0100 Subject: [PATCH 08/34] Refactoring, flattning permissions for now. --- mreg/api/permissions.py | 74 +++++--------------------------------- mreg/api/treetop.py | 54 ++++++++++++++++++++-------- treetop/data/mreg.cedar | 62 +++++++++++++------------------- treetop/docker-compose.yml | 1 + 4 files changed, 73 insertions(+), 118 deletions(-) diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index 3b0d4353..4e4e8a09 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -9,6 +9,7 @@ from structlog import get_logger + from mreg.api.v1.serializers import HostSerializer from mreg.models.host import HostGroup from mreg.models.network import NetGroupRegexPermission, Network @@ -31,14 +32,9 @@ class ParityMixin: - """Small helpers to reduce repetition around policy_parity. - - The public pp() method logs every call. For cases where multiple checks - feed into a single decision (pp_any, pp_all), use _pp() internally to avoid - nested logging and only log the final result. - """ + """Small helpers to reduce repetition around policy_parity.""" - def _pp( + def pp( self, *, decision: bool, @@ -48,12 +44,7 @@ def _pp( resource_kind: str = "Generic", resource_id: str = "any", resource_attrs: Optional[Mapping[str, str]] = None, - log: bool = True, ) -> bool: - """Internal policy parity check. Set log=False to skip logging.""" - if not log: - # For internal use: return decision without calling policy_parity - return decision return policy_parity( decision, request=request, @@ -65,28 +56,6 @@ def _pp( resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, ) - def pp( - self, - *, - decision: bool, - action: str, - request: Request, - view: "GenericAPIView", - resource_kind: str = "Generic", - resource_id: str = "any", - resource_attrs: Optional[Mapping[str, str]] = None, - ) -> bool: - return self._pp( - decision=decision, - action=action, - request=request, - view=view, - resource_kind=resource_kind, - resource_id=resource_id, - resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, - log=True, - ) - def pp_host( self, *, @@ -125,16 +94,14 @@ def pp_any( resource_kind: str = "Generic", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: - # Use internal _pp with log=False to avoid nested logging for each check for decision, action in checks: - if self._pp( + if self.pp( decision=decision, action=action, request=request, view=view, resource_kind=resource_kind, resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, - log=False, ): return True return False @@ -148,30 +115,19 @@ def pp_all( resource_kind: str = "Generic", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: - # Use internal _pp with log=False to avoid nested logging for each check for decision, action in checks: - if not self._pp( + if not self.pp( decision=decision, action=action, request=request, view=view, resource_kind=resource_kind, resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, - log=False, ): return False return True - def pp_generic_action( - self, - attrs: Mapping[str, str], - decision: bool, - action: str, - request: Request, - view: GenericAPIView, - kind: str = "Generic", - id: str = "Any" - ) -> bool: + def pp_generic_action(self, attrs: Mapping[str, str], decision: bool, action: str, request: Request, view: GenericAPIView, kind: str = "Generic", id: str = "Any") -> bool: return self.pp( decision=decision, action=action, @@ -182,13 +138,7 @@ def pp_generic_action( resource_attrs={ kind: kind, **attrs } ) - def user_has_permission( - self, - membership: MregAdminGroup, - request: Request, - view: GenericAPIView, - exclude_superuser: bool = False - ) -> bool: + def user_has_permission(self, membership: MregAdminGroup, request: Request, view: GenericAPIView, exclude_superuser: bool = False) -> bool: """ Check if the user has a given generic permission level. """ @@ -406,15 +356,7 @@ def has_permission(self, request, view): return False if request.method in SAFE_METHODS: return True - return policy_parity( - User.from_request(request).is_mreg_superuser_or_admin, - request=request, - view=view, - permission_class=self.__class__.__name__, - action="is_admin", # Superadmins don't care what the action is - resource_kind="Generic", - resource_attrs={"kind": "Any", "id": "any"}, - ) + return self.user_is_admin(request=request, view=view) diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index 645d35f0..5ee7f2fa 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -1,6 +1,8 @@ from __future__ import annotations import logging -from typing import Any, Optional +from typing import Any, Optional, Mapping +import ipaddress +import json from django.conf import settings from rest_framework.request import Request @@ -9,7 +11,7 @@ from mreg.models.auth import User as MregUser # your request->user wrapper from treetop_client.client import TreeTopClient -from treetop_client.models import Request as TreeTopRequest, User as TreeTopUser, Action as TreeTopAction, Resource as TreeTopResource +from treetop_client.models import Request as TreeTopRequest, User as TreeTopUser, Action, Resource, ResourceAttribute, ResourceAttributeType logger = logging.getLogger("mreg.policy.parity") @@ -45,7 +47,8 @@ def policy_parity( permission_class: Optional[str] = None, action: str, resource_kind: str, - resource_attrs: dict[str, Any], + resource_id: str, + resource_attrs: Mapping[str, str], ) -> bool: """ Log legacy-vs-policy parity and return `decision` unchanged. @@ -57,19 +60,41 @@ def policy_parity( # Build policy request muser = MregUser.from_request(request) principal = TreeTopUser.new(muser.username, POLICY_NAMESPACE, groups=list(muser.group_list)) - pol_action = TreeTopAction.new(action, POLICY_NAMESPACE) - res = TreeTopResource.new(resource_kind, resource_attrs) + pol_action = Action.new(action, POLICY_NAMESPACE) + + attrs = {} + + for k, v in resource_attrs.items(): + try: + ip = ipaddress.ip_address(v) + attrs[k] = ResourceAttribute.new(str(ip), ResourceAttributeType.IP) + except ValueError: + attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) +# if v.isdigit(): +# attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.NUMBER) +# elif v.lower() in ("true", "false"): +# attrs[k] = ResourceAttribute.new(v.lower(), ResourceAttributeType.BOOLEAN) +# else: +# attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) + + res = Resource.new(resource_kind, resource_id, attrs=attrs) + + if len(pol_action.id.namespace) > 0: + fully_qualified_action = "::".join(pol_action.id.namespace) + f"::{pol_action.id.id}" + else: + fully_qualified_action = f"{pol_action.id.id}" context = { "path": request.path, "method": request.method, "permission": permission_class or (view and view.__class__.__name__), "view": view and view.__class__.__name__, - "resource_kind": resource_kind, - "action": getattr(pol_action, "name", str(pol_action)), + "model": _model_name_from_view(view), "principal": muser.username, "groups": list(muser.group_list), - "model": _model_name_from_view(view), + "action": fully_qualified_action, + "resource_kind": resource_kind, + "resource_attrs": resource_attrs, "correlation_id": _corr_id(request), } @@ -87,24 +112,23 @@ def policy_parity( parity = True payload: dict[str, object] = { - **context, + "parity": parity, "legacy_decision": bool(decision), "policy_decision": pol_allowed, - "parity": parity, - "resource_attrs": resource_attrs, "error": error, + "context": context, } if parity: logger.warning("policy_parity_mismatch", extra=payload) - log_policy_parity("OK", payload) + log_policy_parity(payload) else: logger.info("policy_parity_ok", extra=payload) - log_policy_parity("MISMATCH", payload) + log_policy_parity(payload) return decision # Log data to a file in addition to normal logging -def log_policy_parity(result: str, payload: dict[str, Any]): +def log_policy_parity(payload: dict[str, Any]): with open(POLICY_EXTRA_LOG_FILE_NAME, "a") as log_file: - log_file.write(f"{result}: {payload}\n") + log_file.write(f"{json.dumps(payload)}\n") diff --git a/treetop/data/mreg.cedar b/treetop/data/mreg.cedar index 20d19ebe..33bc5fe8 100644 --- a/treetop/data/mreg.cedar +++ b/treetop/data/mreg.cedar @@ -3,11 +3,7 @@ @id("MREG.admins_policy") permit ( principal in MREG::Group::"admins", - action in - [MREG::Action::"create_host", - MREG::Action::"delete_host", - MREG::Action::"view_host", - MREG::Action::"edit_host"], + action in MREG::Action::"host_access", resource is Host ); @@ -16,34 +12,15 @@ permit ( @id("MREG.webadmins_policy") permit ( principal in MREG::Group::"webadmins", - action in - [MREG::Action::"edit_host", - MREG::Action::"delete_host", - MREG::Action::"create_host"], + action in MREG::Action::"host_access", resource is Host ) when { resource.nameLabels.contains("webserver") && - resource.ip.isInRange("192.168.1.0/24") + resource.ip.isInRange(ip("192.168.1.0/24")) }; -// Users can only view hosts -@id("MREG.users_policy") -permit ( - principal in MREG::Group::"users", - action == MREG::Action::"view_host", - resource is Host -); - -// Charlie does not get to delete hosts, no matter what. -@id("MREG.charlie_forbid_delete_host_policy") -forbid ( - principal == MREG::User::"charlie", - action == MREG::Action::"delete_host", - resource is Host -); - // Admins can manipulate any IP address, even if it is a gw, a broadcast address, // the network address, reserved. These three groups are unified as "restricted" IPs. @id("MREG.admins_ip_policy") @@ -58,6 +35,17 @@ permit ( resource is IPAddress ); + +/// Test group access, used during testing. +@id("MREG.test_group_policy") +permit ( + principal in MREG::Group::"testgroup", + action in [MREG::Action::"host_access"], + resource is Host +) when { + resource.ip.isInRange(ip("10.0.0.0/24")) +}; + /// Network Admins can manage any IP in any network @id("MREG.network_admins_ip_network_policy") permit ( @@ -75,8 +63,8 @@ permit ( ) when { - resource.ip.isInRange("192.168.1.0/24") || - resource.ip.isInRange("10.0.0.0/8") + resource.ip.isInRange(ip("192.168.1.0/24")) || + resource.ip.isInRange(ip("10.0.0.0/8")) }; /// Admins can do whatever with labels. @@ -92,7 +80,7 @@ permit ( ); /// Superadmins -@id("MREG.is_superuser") +@id("MREG.superadmin") permit ( principal in MREG::Group::"default-super-group", action, // Superadmins don't care what the action is @@ -100,34 +88,34 @@ permit ( ); /// Normal (?) admins -@id("MREG.is_admin") +@id("MREG.admin") permit ( principal in MREG::Group::"default-admin-group", - action == MREG::Action::"is_admin", + action == MREG::Action::"admin_access", resource ); /// Host Policy Admins -@id("MREG.is_hostpolicy_admin") +@id("MREG.hostpolicy_admin") permit ( principal in MREG::Group::"default-hostpolicyadmin-group", - action == MREG::Action::"is_hostpolicy_admin", + action == MREG::Action::"hostpolicy_admin_access", resource ); /// DNS Wildcard Admins -@id("MREG.is_dns_wildcard_admin") +@id("MREG.dns_wildcard_admin") permit ( principal in MREG::Group::"default-dns-wildcard-group", - action == MREG::Action::"is_dns_wildcard_admin", + action == MREG::Action::"dns_wildcard_admin_access", resource ); /// DNS Underscore Admins -@id("MREG.is_dns_underscore_admin") +@id("MREG.dns_underscore_admin") permit ( principal in MREG::Group::"default-dns-underscore-group", - action == MREG::Action::"is_dns_underscore_admin", + action == MREG::Action::"dns_underscore_admin_access", resource ); diff --git a/treetop/docker-compose.yml b/treetop/docker-compose.yml index fc7f817e..381e1fa6 100644 --- a/treetop/docker-compose.yml +++ b/treetop/docker-compose.yml @@ -10,6 +10,7 @@ services: treetop-server: image: ghcr.io/terjekv/treetop-rest:develop + pull_policy: "always" ports: - "9999:9999" environment: From c09f3c6ba0ca58e24519c29bb13472317c1cd4d0 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Thu, 8 Jan 2026 09:15:44 +0100 Subject: [PATCH 09/34] Update towards master. --- mreg/api/permissions.py | 63 +++++++++++++++++++++++++++++++++++++---- mreg/api/treetop.py | 57 +++++++++++++++++++++++++++++++++---- 2 files changed, 108 insertions(+), 12 deletions(-) diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index 4e4e8a09..121a1b4b 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -32,9 +32,14 @@ class ParityMixin: - """Small helpers to reduce repetition around policy_parity.""" + """Small helpers to reduce repetition around policy_parity. - def pp( + The public pp() method logs every call. For cases where multiple checks + feed into a single decision (pp_any, pp_all), use _pp() internally to avoid + nested logging and only log the final result. + """ + + def _pp( self, *, decision: bool, @@ -44,7 +49,12 @@ def pp( resource_kind: str = "Generic", resource_id: str = "any", resource_attrs: Optional[Mapping[str, str]] = None, + log: bool = True, ) -> bool: + """Internal policy parity check. Set log=False to skip logging.""" + if not log: + # For internal use: return decision without calling policy_parity + return decision return policy_parity( decision, request=request, @@ -56,6 +66,28 @@ def pp( resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, ) + def pp( + self, + *, + decision: bool, + action: str, + request: Request, + view: "GenericAPIView", + resource_kind: str = "Generic", + resource_id: str = "any", + resource_attrs: Optional[Mapping[str, str]] = None, + ) -> bool: + return self._pp( + decision=decision, + action=action, + request=request, + view=view, + resource_kind=resource_kind, + resource_id=resource_id, + resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + log=True, + ) + def pp_host( self, *, @@ -94,14 +126,16 @@ def pp_any( resource_kind: str = "Generic", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: + # Use internal _pp with log=False to avoid nested logging for each check for decision, action in checks: - if self.pp( + if self._pp( decision=decision, action=action, request=request, view=view, resource_kind=resource_kind, resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + log=False, ): return True return False @@ -115,19 +149,30 @@ def pp_all( resource_kind: str = "Generic", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: + # Use internal _pp with log=False to avoid nested logging for each check for decision, action in checks: - if not self.pp( + if not self._pp( decision=decision, action=action, request=request, view=view, resource_kind=resource_kind, resource_attrs=resource_attrs or DEFAULT_RESOURCE_ATTRS, + log=False, ): return False return True - def pp_generic_action(self, attrs: Mapping[str, str], decision: bool, action: str, request: Request, view: GenericAPIView, kind: str = "Generic", id: str = "Any") -> bool: + def pp_generic_action( + self, + attrs: Mapping[str, str], + decision: bool, + action: str, + request: Request, + view: GenericAPIView, + kind: str = "Generic", + id: str = "Any" + ) -> bool: return self.pp( decision=decision, action=action, @@ -138,7 +183,13 @@ def pp_generic_action(self, attrs: Mapping[str, str], decision: bool, action: st resource_attrs={ kind: kind, **attrs } ) - def user_has_permission(self, membership: MregAdminGroup, request: Request, view: GenericAPIView, exclude_superuser: bool = False) -> bool: + def user_has_permission( + self, + membership: MregAdminGroup, + request: Request, + view: GenericAPIView, + exclude_superuser: bool = False + ) -> bool: """ Check if the user has a given generic permission level. """ diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index 5ee7f2fa..77f87088 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -3,6 +3,8 @@ from typing import Any, Optional, Mapping import ipaddress import json +import threading +from contextlib import contextmanager from django.conf import settings from rest_framework.request import Request @@ -15,6 +17,9 @@ logger = logging.getLogger("mreg.policy.parity") +# Thread-local storage for parity checking bypass flag +_thread_local = threading.local() + # Configure these in settings.py POLICY_PARITY_ENABLED = getattr(settings, "POLICY_PARITY_ENABLED", True) POLICY_BASE_URL = getattr(settings, "POLICY_BASE_URL", "http://localhost:9999") @@ -28,6 +33,34 @@ treetopclient = TreeTopClient(base_url=POLICY_BASE_URL) +@contextmanager +def disable_policy_parity(): + """Context manager to temporarily disable policy parity checking. + + Useful for tests that modify permissions/state mid-test, which would + cause the legacy and policy systems to be out of sync. + + Example: + def test_permission_changes(self): + with disable_policy_parity(): + # Modify permissions here + user.groups.add(some_group) + # Make API calls - parity checking will be skipped + """ + old_value = getattr(_thread_local, "skip_parity", False) + _thread_local.skip_parity = True + try: + yield + finally: + _thread_local.skip_parity = old_value + +def _is_parity_enabled() -> bool: + """Check if parity checking should be performed in current context.""" + if not POLICY_PARITY_ENABLED: + return False + # Skip parity checking if we're in a disabled context + return not getattr(_thread_local, "skip_parity", False) + def _corr_id(request: Request) -> Optional[str]: return request.headers.get("X-Correlation-ID") or request.META.get("HTTP_X_CORRELATION_ID") @@ -54,12 +87,12 @@ def policy_parity( Log legacy-vs-policy parity and return `decision` unchanged. Use this anywhere you currently 'return True/False'. """ - if not POLICY_PARITY_ENABLED: + if not _is_parity_enabled(): return decision # Build policy request muser = MregUser.from_request(request) - principal = TreeTopUser.new(muser.username, POLICY_NAMESPACE, groups=list(muser.group_list)) + principal = TreeTopUser.new(str(muser.username), POLICY_NAMESPACE, groups=list(muser.group_list)) pol_action = Action.new(action, POLICY_NAMESPACE) attrs = {} @@ -77,7 +110,7 @@ def policy_parity( # else: # attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) - res = Resource.new(resource_kind, resource_id, attrs=attrs) + res = Resource.new(str(resource_kind), resource_id, attrs=attrs) if len(pol_action.id.namespace) > 0: fully_qualified_action = "::".join(pol_action.id.namespace) + f"::{pol_action.id.id}" @@ -104,6 +137,18 @@ def policy_parity( pol_allowed = bool(resp.is_allowed()) except Exception as exc: error = repr(exc) + # Log policy server errors prominently + logger.error( + f"Policy server error: {type(exc).__name__}: {exc}", + extra={ + "error_type": type(exc).__name__, + "error_msg": str(exc), + "path": request.path, + "correlation_id": _corr_id(request), + }, + ) + # If policy server fails, we cannot determine parity. Return legacy decision + # but flag this in the payload for monitoring. parity = False if bool(decision) and pol_allowed: @@ -111,7 +156,7 @@ def policy_parity( elif not bool(decision) and not pol_allowed: parity = True - payload: dict[str, object] = { + payload: dict[str, Any] = { "parity": parity, "legacy_decision": bool(decision), "policy_decision": pol_allowed, @@ -120,10 +165,10 @@ def policy_parity( } if parity: - logger.warning("policy_parity_mismatch", extra=payload) + logger.info("policy_parity_ok", extra=payload) log_policy_parity(payload) else: - logger.info("policy_parity_ok", extra=payload) + logger.warning("policy_parity_mismatch", extra=payload) log_policy_parity(payload) return decision From a7ca240520d730f1bc96b3cdc17110988004c0dd Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sat, 14 Feb 2026 18:29:31 +0100 Subject: [PATCH 10/34] Resolve outstanding parity mismatches and return to granular permissions. - Also adds documentation for the policy framework and setup. --- docs/env.md | 16 ++ docs/parity_testing.md | 59 +++++ docs/policies.md | 133 +++++++++++ mreg/api/permissions.py | 434 ++++++++++++++++++++++++++++++---- mreg/api/treetop.py | 7 +- mregsite/settings.py | 19 ++ pyproject.toml | 3 +- treetop/data/host_labels.json | 18 -- treetop/data/labels.json | 25 ++ treetop/data/mreg.cedar | 265 ++++++++++++++++++++- treetop/docker-compose.yml | 12 +- uv.lock | 17 +- 12 files changed, 929 insertions(+), 79 deletions(-) create mode 100644 docs/policies.md delete mode 100644 treetop/data/host_labels.json create mode 100644 treetop/data/labels.json diff --git a/docs/env.md b/docs/env.md index ebcb3ef7..59af27ce 100644 --- a/docs/env.md +++ b/docs/env.md @@ -12,6 +12,22 @@ Must be one of the following: - `ERROR` - `CRITICAL` +## `MREG_POLICY_PARITY_LOG_LEVEL` + +Log level for the dedicated `mreg.policy.parity` logger. Default: `WARNING` + +This controls parity discrepancy logs independently from `MREG_LOG_LEVEL`, so +legacy-vs-policy mismatches can be surfaced even when the general app logger is +more restrictive. + +Must be one of the following: + +- `DEBUG` +- `INFO` +- `WARNING` +- `ERROR` +- `CRITICAL` + ## `MREG_LOG_FILE_SIZE` Maximum file size of the log file in bytes. Default: `52428800` (50MB). diff --git a/docs/parity_testing.md b/docs/parity_testing.md index 65589381..f1959961 100644 --- a/docs/parity_testing.md +++ b/docs/parity_testing.md @@ -1,5 +1,9 @@ # Disabling Parity Checking in Tests +Related documentation: + +- Policy actions and resource/action contracts: [`policies.md`](./policies.md) + ## Problem Tests that modify permissions or group memberships mid-test cause the legacy permission system and the TreeTop policy engine to be out of sync. Since TreeTop's policy content is immutable, these tests cannot maintain parity between the two systems. @@ -94,6 +98,61 @@ Do NOT disable parity checking for: - Tests that modify non-permission data (hosts, networks, etc.) - Tests where both legacy and policy systems should agree +## Scope Rules (Enforcement Guidance) + +Keep parity disable scope as narrow as possible: + +- Prefer wrapping only the exact mutation and requests that depend on that mutation. +- Do not wrap an entire test module unless the whole module genuinely mutates permission state. +- Do not wrap whole suites by default; this hides real policy regressions. +- Re-enable parity immediately after the mutation scenario has been asserted. + ## Implementation Details The `disable_policy_parity()` context manager uses thread-local storage to safely disable parity checking for the current thread only, ensuring test isolation in parallel test execution. + +## Parity Runbook + +Use this sequence when validating parity changes: + +1. Run full tests with coverage and parity logging enabled. + +```bash +source .env; .venv/bin/tox -e coverage +``` + +2. Count parity mismatches. + +```bash +jq -s '[.[] | select(.parity == false)] | length' policy_parity.log +``` + +3. List mismatch lines for triage. + +```bash +rg -n '"parity": false' policy_parity.log +``` + +4. Optional: inspect actions seen in the run. + +```bash +jq -r '.context.action // empty' policy_parity.log | sort | uniq -c | sort -nr +``` + +## Mismatch Triage Guide + +Use the payload fields `legacy_decision`, `policy_decision`, `context.action`, and `context.resource_attrs`. + +- `legacy_decision=true`, `policy_decision=false`: + - Missing/too-narrow Cedar allow rule. + - Action name mismatch (for example wrong CRUD token). + - Missing required attributes for Cedar conditions. +- `legacy_decision=false`, `policy_decision=true`: + - Cedar rule is broader than legacy behavior. + - Resource kind fallback produced a more permissive policy path than intended. +- `error` present: + - Policy client/server failure. Resolve connectivity/config first before triaging semantics. +- Unexpected `context.resource_kind` (for example view name fallback): + - Fix serializer `Meta.model` usage or explicit resource kind dispatch in permission code. + +When fixing mismatches, update code and Cedar together, then rerun the runbook until mismatch count is zero. diff --git a/docs/policies.md b/docs/policies.md new file mode 100644 index 00000000..1ea8c833 --- /dev/null +++ b/docs/policies.md @@ -0,0 +1,133 @@ + +# Policy Actions + +This section documents the policy actions currently used by MREG permission checks. + +Related documentation: + +- Parity test workflow and triage: [`parity_testing.md`](./parity_testing.md) + +## Source of Truth + +- Policy definitions: `treetop/data/mreg.cedar` +- Action generation in code: `mreg/api/permissions.py` (`ParityMixin._crud_action`) +- Parity transport/logging: `mreg/api/treetop.py` + +## Adding a New Protected Resource + +When introducing a new resource that should be parity-checked, use this checklist: + +1. Ensure the permission path reaches `ParityMixin.pp()` or `pp_generic_action()`. +2. Confirm CRUD action dispatch is used (`_`). +3. Verify resource kind resolution works for the endpoint: + - Prefer serializer `Meta.model`. + - Fallbacks should remain stable for non-model views. +4. Verify resource ID resolution produces stable IDs for list/detail/custom views. +5. Add or update Cedar actions/rules in `treetop/data/mreg.cedar`. +6. If policy conditions depend on derived labels, update `treetop/data/labels.json`. +7. Add tests for create/read/update/delete behavior and group/admin overrides. +8. Run parity checks and confirm zero mismatches. +9. If tests mutate permissions mid-test, scope `disable_policy_parity()` as narrowly as possible. + +## Resource Kind and ID Resolution + +`ParityMixin` resolves resource kind and ID using deterministic fallbacks. + +Resource kind fallback order (`_resource_kind_from_view`): + +1. `obj.__class__.__name__` when object is available +2. `validated_serializer.Meta.model.__name__` +3. `view.get_serializer_class().Meta.model.__name__` +4. `validated_serializer.instance.__class__.__name__` +5. View class name with suffixes (`List`, `Detail`, `View`) stripped + +Resource ID fallback order (`_resource_id_from_view`): + +1. Object attributes: `pk`, `id`, `name` +2. Request/serializer data keys: `pk`, `id`, `name` +3. `validated_serializer.instance` attributes: `pk`, `id`, `name` +4. URL kwargs: `pk`, `id`, `name`, `cpk`, `hostpk`, `network` +5. Default `"any"` + +## CRUD Action Naming + +For model-backed checks, action names are generated as: + +`_` + +Where operation is mapped from HTTP method: + +- `GET`, `HEAD`, `OPTIONS` -> `read` +- `POST` -> `create` +- `PUT`, `PATCH` -> `update` +- `DELETE` -> `delete` + +## CRUD Actions Declared in Cedar + +- `host_create`, `host_read`, `host_update`, `host_delete` +- `host_contacts_read` +- `ipaddress_create`, `ipaddress_read`, `ipaddress_update`, `ipaddress_delete` +- `cname_create`, `cname_read`, `cname_update`, `cname_delete` +- `hinfo_create`, `hinfo_read`, `hinfo_update`, `hinfo_delete` +- `loc_create`, `loc_read`, `loc_update`, `loc_delete` +- `mx_create`, `mx_read`, `mx_update`, `mx_delete` +- `naptr_create`, `naptr_read`, `naptr_update`, `naptr_delete` +- `name_server_create`, `name_server_read`, `name_server_update`, `name_server_delete` +- `ptr_override_create`, `ptr_override_read`, `ptr_override_update`, `ptr_override_delete` +- `sshfp_create`, `sshfp_read`, `sshfp_update`, `sshfp_delete` +- `srv_create`, `srv_read`, `srv_update`, `srv_delete` +- `txt_create`, `txt_read`, `txt_update`, `txt_delete` +- `bacnet_id_create`, `bacnet_id_read`, `bacnet_id_update`, `bacnet_id_delete` +- `community_create`, `community_read`, `community_update`, `community_delete` + +## Non-CRUD Actions Declared in Cedar + +- `admin_access` +- `network_admin_access` +- `hostgroup_admin_access` +- `hostpolicy_admin_access` +- `dns_wildcard_admin_access` +- `dns_underscore_admin_access` +- `ip_gw_management` +- `ip_broadcast_management` +- `ip_network_management` +- `ip_reserved_management` +- `ip_restricted_management` +- `create_label` +- `delete_label` +- `view_label` +- `edit_label` + +## Attribute Contract for Policy Checks + +All resource attributes are normalized through `ParityMixin._normalize_resource_attrs`: + +- `kind` is always added using snake_case resource kind. +- Attribute values are stringified. +- In `policy_parity`, string values that parse as IPs are sent as IP-typed attributes, otherwise as string attributes. + +Common attribute payloads in current checks: + +| Context | Typical action(s) | Attributes sent | +| --- | --- | --- | +| Safe/read precheck in `IsGrantedNetGroupRegexPermission.has_permission` | `_read` | `kind`, `path` | +| Host/IP netgroup evaluation (`has_perm`) | CRUD action from method | `kind`, `hostname`, optional `ip` | +| Create admin parity check | `_create` | `kind` + flattened serializer data | +| Update admin parity check | `_update` | `kind` + stringified validated data | +| Destroy admin parity check | `_delete` | `kind`, `id` | + +## Wildcard Action Rules + +These rules do not enumerate action names and therefore match any action: + +- `MREG.superadmin`: principal in `default-super-group` may perform any action. +- `global.super_admin_allow_all_policy`: principal `User::"super"` may perform any action. + +## Code-Emitted Parity Actions + +The code also emits parity checks for: + +- `superuser_access` +- `is_superuser` + +These are covered by wildcard superadmin rules in Cedar. diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index 121a1b4b..0980d330 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -1,6 +1,7 @@ from __future__ import annotations import ipaddress +import re from django.db import models from typing import TYPE_CHECKING, Iterable, Mapping, Optional, Tuple, Any from rest_framework import exceptions @@ -39,6 +40,146 @@ class ParityMixin: nested logging and only log the final result. """ + _CRUD_METHOD_TO_OPERATION = { + "GET": "read", + "HEAD": "read", + "OPTIONS": "read", + "POST": "create", + "PUT": "update", + "PATCH": "update", + "DELETE": "delete", + } + + @staticmethod + def _stringify_attr_value(value: Any) -> str: + """Convert attribute values to strings for TreeTop resource attributes.""" + return "" if value is None else str(value) + + @staticmethod + def _snake_case(value: str) -> str: + """Normalize model/resource names to snake_case action/resource tokens.""" + if value.startswith("BACnet"): + value = f"Bacnet{value[len('BACnet'):]}" + value = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", value) + value = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value) + value = value.replace("-", "_") + value = re.sub(r"[^a-zA-Z0-9_]+", "_", value).strip("_").lower() + return value or "generic" + + @staticmethod + def _resource_name_from_model(model: Any) -> Optional[str]: + """Return a model class name if available, otherwise None.""" + name = getattr(model, "__name__", None) + return str(name) if name else None + + def _resource_kind_from_view( + self, + *, + view: "GenericAPIView", + validated_serializer: Optional["Serializer"] = None, + obj: Any = None, + ) -> str: + """Resolve a resource kind for parity from object/serializer/view metadata. + + Resolution order: + 1. concrete object class + 2. serializer Meta.model (validated serializer) + 3. view serializer Meta.model + 4. validated serializer instance class + 5. view class name with common DRF suffixes stripped + """ + if obj is not None: + return obj.__class__.__name__ + + if validated_serializer is not None: + serializer_model = self._resource_name_from_model( + getattr(getattr(validated_serializer, "Meta", None), "model", None) + ) + if serializer_model: + return serializer_model + + try: + serializer_class = view.get_serializer_class() + view_model = self._resource_name_from_model( + getattr(getattr(serializer_class, "Meta", None), "model", None) + ) + if view_model: + return view_model + except Exception: + pass + + if validated_serializer is not None: + instance = getattr(validated_serializer, "instance", None) + if instance is not None: + return instance.__class__.__name__ + + view_name = view.__class__.__name__ + for suffix in ("List", "Detail", "View"): + if view_name.endswith(suffix): + view_name = view_name[: -len(suffix)] + break + return view_name or "Generic" + + def _resource_id_from_view( + self, + *, + view: "GenericAPIView", + validated_serializer: Optional["Serializer"] = None, + obj: Any = None, + data: Optional[Mapping[str, Any]] = None, + default: str = "any", + ) -> str: + """Resolve a stable resource identifier for parity logging/evaluation.""" + if obj is not None: + for key in ("pk", "id", "name"): + val = getattr(obj, key, None) + if val is not None: + return str(val) + + if data: + for key in ("pk", "id", "name"): + val = data.get(key) + if val is not None: + return str(val) + + if validated_serializer is not None: + instance = getattr(validated_serializer, "instance", None) + if instance is not None: + for key in ("pk", "id", "name"): + val = getattr(instance, key, None) + if val is not None: + return str(val) + + view_kwargs = getattr(view, "kwargs", {}) + if isinstance(view_kwargs, Mapping): + for key in ("pk", "id", "name", "cpk", "hostpk", "network"): + if key in view_kwargs and view_kwargs[key] is not None: + return str(view_kwargs[key]) + + return default + + def _crud_operation_from_method(self, method: str) -> str: + """Map an HTTP method to a CRUD operation token.""" + return self._CRUD_METHOD_TO_OPERATION.get(method.upper(), "read") + + def _crud_action(self, resource_kind: str, operation: str) -> str: + """Build a policy action name like `_`.""" + return f"{self._snake_case(resource_kind)}_{operation}" + + def _normalize_resource_attrs( + self, + *, + resource_kind: str, + attrs: Optional[Mapping[str, Any]], + ) -> dict[str, str]: + """Normalize resource attributes to string values with a canonical kind.""" + normalized = {"kind": self._snake_case(resource_kind)} + if attrs: + normalized.update( + {str(k): self._stringify_attr_value(v) for k, v in attrs.items()} + ) + return normalized + def _pp( self, *, @@ -77,6 +218,7 @@ def pp( resource_id: str = "any", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: + """Run one parity check and emit a single parity log record.""" return self._pp( decision=decision, action=action, @@ -95,16 +237,16 @@ def pp_host( request: Request, view: "GenericAPIView", resource_id: str = "", - action: str = "host_access", + action: str = "host_read", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: """Helper for host-related actions. - Assumes `resource_kind="Host"` and `action="host_access"`, and tries to extract the resource ID from + Assumes `resource_kind="Host"` and tries to extract the resource ID from `resource_attrs["hostname"]` if not explicitly given. """ - if not resource_id and resource_attrs and hasattr(resource_attrs, "hostname"): + if not resource_id and resource_attrs and "hostname" in resource_attrs: resource_id = resource_attrs["hostname"] return self.pp( @@ -126,6 +268,7 @@ def pp_any( resource_kind: str = "Generic", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: + """Return True if any candidate check succeeds, without nested logging.""" # Use internal _pp with log=False to avoid nested logging for each check for decision, action in checks: if self._pp( @@ -149,6 +292,7 @@ def pp_all( resource_kind: str = "Generic", resource_attrs: Optional[Mapping[str, str]] = None, ) -> bool: + """Return True if all candidate checks succeed, without nested logging.""" # Use internal _pp with log=False to avoid nested logging for each check for decision, action in checks: if not self._pp( @@ -165,22 +309,23 @@ def pp_all( def pp_generic_action( self, - attrs: Mapping[str, str], + attrs: Mapping[str, Any], decision: bool, action: str, request: Request, view: GenericAPIView, kind: str = "Generic", - id: str = "Any" + id: str = "any" ) -> bool: + """Convenience wrapper that normalizes attrs and forwards to pp().""" return self.pp( decision=decision, action=action, request=request, view=view, resource_kind=kind, - resource_id=id, - resource_attrs={ kind: kind, **attrs } + resource_id=str(id), + resource_attrs=self._normalize_resource_attrs(resource_kind=kind, attrs=attrs), ) def user_has_permission( @@ -393,6 +538,7 @@ def has_permission(self, request, view): permission_class=self.__class__.__name__, action="is_superuser", resource_kind="Generic", + resource_id="any", resource_attrs={"kind": "Any", "id": "any"}, ) @@ -451,11 +597,23 @@ def has_permission(self, request, view): # just do some preliminary checks. if not super().has_permission(request, view): return False + user = User.from_request(request) if request.method in SAFE_METHODS: - return True + resource_kind = self._resource_kind_from_view(view=view) + return self.pp_generic_action( + decision=True, + action=self._crud_action(resource_kind, "read"), + kind=resource_kind, + id=self._resource_id_from_view(view=view), + attrs={"path": request.path}, + request=request, + view=view, + ) + if user.is_mreg_superuser_or_admin: return True + # Will do do more object checks later, but initially refuse any # unwarranted requests. qs = NetGroupRegexPermission.objects.filter(group__in=user.group_list) @@ -469,22 +627,76 @@ def has_permission(self, request, view): return True return False - def has_perm(self, user, hostname, ips, request: Request, view: GenericAPIView, require_ip=True): + def has_perm( + self, + user, + hostname, + ips, + request: Request, + view: GenericAPIView, + require_ip=True, + action: Optional[str] = None, + resource_kind: str = "Host", + resource_id: Optional[str] = None, + ): + """Evaluate NetGroupRegexPermission and parity for hostname/IP tuples.""" legacy = bool(NetGroupRegexPermission.find_perm(user.group_list, hostname, ips, require_ip)) + operation = self._crud_operation_from_method(request.method) + resolved_action = action or self._crud_action(resource_kind, operation) + resolved_resource_id = str(resource_id or hostname or "any") policy: list[bool] = [] if ips: # This will perform one policy lookup per IP for the host. This should probably be optimized server side. for ip in ips: - policy.append(self.pp_host(decision=legacy, request=request, view=view, resource_attrs={"hostname": hostname, "ip": ip})) + policy.append( + self.pp( + decision=legacy, + action=resolved_action, + request=request, + view=view, + resource_kind=resource_kind, + resource_id=resolved_resource_id, + resource_attrs={"hostname": str(hostname), "ip": str(ip)}, + ) + ) else: - policy.append(self.pp_host(decision=legacy, request=request, view=view, resource_attrs={"hostname": hostname})) + policy.append( + self.pp( + decision=legacy, + action=resolved_action, + request=request, + view=view, + resource_kind=resource_kind, + resource_id=resolved_resource_id, + resource_attrs={"hostname": str(hostname)}, + ) + ) return any(policy) - def has_obj_perm(self, user: User, obj: str, request: Request, view: GenericAPIView) -> bool: - return self.has_perm(user, *self._get_hostname_and_ips(obj), request=request, view=view) + def has_obj_perm( + self, + user: User, + obj: str, + request: Request, + view: GenericAPIView, + action: Optional[str] = None, + resource_kind: str = "Host", + resource_id: Optional[str] = None, + ) -> bool: + """Resolve hostname/IPs from an object and delegate to has_perm().""" + return self.has_perm( + user, + *self._get_hostname_and_ips(obj), + request=request, + view=view, + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + ) def has_create_permission(self, request, view, validated_serializer): + """Authorize create operations using CRUD parity actions and legacy rules.""" import mreg.api.v1.views as v1_views user = User.from_request(request) @@ -496,8 +708,18 @@ def has_create_permission(self, request, view, validated_serializer): hostname = None ips = [] - attrs: dict[str, Any] = {} + attrs: dict[str, str] = {} data: dict[str, Any] = validated_serializer.validated_data # type: ignore + resource_kind = self._resource_kind_from_view( + view=view, + validated_serializer=validated_serializer, + ) + action = self._crud_action(resource_kind, "create") + resource_id = self._resource_id_from_view( + view=view, + validated_serializer=validated_serializer, + data=data, + ) # Convert all data from the serializer to strings to feed as attributes to the policy engine. # We also introspect BaseModel instances to flatten them out (one level deep). @@ -506,34 +728,69 @@ def has_create_permission(self, request, view, validated_serializer): if data: for key, value in data.items(): if isinstance(value, (str, int, float, bool)): - attrs[key] = value + attrs[key] = self._stringify_attr_value(value) elif isinstance(value, models.Model): for field in value._meta.fields: - attrs[f"{key}_{field.name}"] = str(getattr(value, field.name, '')) + attrs[f"{key}_{field.name}"] = self._stringify_attr_value( + getattr(value, field.name, "") + ) else: - attrs[key] = str(value) + attrs[key] = self._stringify_attr_value(value) ipaddress = data.get('ipaddress', None) host = data.get('host', None) - object_type = validated_serializer.instance.__class__.__name__.lower() # First check if we are asking for a restricted name. if self.deny_superuser_only_names(data=data, view=view, request=request): return False # Then check if we are asking for an IP address *and* it is reserved. if ipaddress and self.deny_reserved_ipaddress(ip=ipaddress, view=view, request=request): return False + + handled_by_view = isinstance( + view, + (v1_views.CnameList, v1_views.HostList, v1_views.IpaddressList, v1_views.PtrOverrideList), + ) + if not handled_by_view and 'host' not in data: + raise exceptions.PermissionDenied(f"Unhandled view: {view}") + # If the user is an admin, they are now free to create (minus the above checks). - if self.pp_generic_action(decision=user.is_mreg_admin, action="create", kind=object_type, attrs=attrs, request=request, view=view): + if self.pp_generic_action( + decision=user.is_mreg_admin, + action=action, + kind=resource_kind, + id=resource_id, + attrs=attrs, + request=request, + view=view, + ): return True # Now check if the user has permission to the host object (if any). if isinstance(view, (v1_views.IpaddressList, v1_views.PtrOverrideList)): - if host and not self.has_obj_perm(user, host, request=request, view=view): + if host and not self.has_obj_perm( + user, + host, + request=request, + view=view, + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + ): return False # CNAMEs are special, we check only the cname, not the ip addresses. if isinstance(view, v1_views.CnameList): - return self.has_perm(user, data['name'], (), require_ip=False, request=request, view=view) + return self.has_perm( + user, + data['name'], + (), + require_ip=False, + request=request, + view=view, + action=action, + resource_kind=resource_kind, + resource_id=self._stringify_attr_value(data['name']), + ) # For hosts and other objects, we need to check the host and its IPs. if isinstance(view, (v1_views.HostList, v1_views.IpaddressList, v1_views.PtrOverrideList)): # HostList does not require ipaddress, but if none, the permissions will not match, so just refuse it. @@ -549,75 +806,167 @@ def has_create_permission(self, request, view, validated_serializer): raise exceptions.PermissionDenied(f"Unhandled view: {view}") if ips and hostname: - return self.has_perm(user, hostname, ips, request=request, view=view) + return self.has_perm( + user, + hostname, + ips, + request=request, + view=view, + action=action, + resource_kind=resource_kind, + resource_id=self._stringify_attr_value(hostname), + ) return False def has_destroy_permission(self, request, view, validated_serializer): - import mreg.api.v1.views + """Authorize delete operations using CRUD parity actions and legacy rules.""" + import mreg.api.v1.views as v1_views user = User.from_request(request) if user.is_mreg_superuser: return True - obj = view.get_object() - if isinstance(view, mreg.api.v1.views.HostDetail): + + target_obj = view.get_object() + resource_kind = self._resource_kind_from_view(view=view, obj=target_obj) + action = self._crud_action(resource_kind, "delete") + resource_id = self._resource_id_from_view(view=view, obj=target_obj) + + host_obj = target_obj + if isinstance(view, v1_views.HostDetail): pass - elif hasattr(obj, 'host'): - obj = obj.host + elif hasattr(target_obj, 'host'): + host_obj = target_obj.host else: raise exceptions.PermissionDenied(f"Unhandled view: {view}") - if self.deny_superuser_only_names(name=obj.name, view=view, request=request): + if self.deny_superuser_only_names(name=host_obj.name, view=view, request=request): return False - if hasattr(obj, 'ipaddress'): - if self.deny_reserved_ipaddress(ip=obj.ipaddress, view=view, request=request): + if hasattr(host_obj, 'ipaddress'): + if self.deny_reserved_ipaddress(ip=host_obj.ipaddress, view=view, request=request): return False - - object_type = obj.__class__.__name__.lower() + if self.pp_generic_action( decision=user.is_mreg_admin, - action="destroy", - kind=object_type, - attrs={"id": str(obj)}, + action=action, + kind=resource_kind, + id=resource_id, + attrs={"id": resource_id}, request=request, view=view ): return True - return self.has_obj_perm(user, obj, request=request, view=view) + return self.has_obj_perm( + user, + host_obj, + request=request, + view=view, + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + ) def has_update_permission(self, request, view, validated_serializer): + """Authorize update operations using CRUD parity actions and legacy rules.""" import mreg.api.v1.views as v1_views user = User.from_request(request) if user.is_mreg_superuser: return True + data: dict[str, Any] = validated_serializer.validated_data # type: ignore + target_obj = view.get_object() + resource_kind = self._resource_kind_from_view( + view=view, + validated_serializer=validated_serializer, + obj=target_obj, + ) + action = self._crud_action(resource_kind, "update") + resource_id = self._resource_id_from_view( + view=view, + validated_serializer=validated_serializer, + obj=target_obj, + data=data, + ) + if self.deny_superuser_only_names(data=data, view=view, request=request): return False if 'ipaddress' in data: if self.deny_reserved_ipaddress(ip=data['ipaddress'], view=view, request=request): return False - if self.user_is_admin(request=request, view=view): + + if not isinstance(view, v1_views.HostDetail) and not hasattr(target_obj, 'host'): + raise exceptions.PermissionDenied(f"Unhandled view: {view}") + + admin_attrs = { + str(key): self._stringify_attr_value(value) + for key, value in data.items() + } + if self.pp_generic_action( + decision=user.is_mreg_admin, + action=action, + kind=resource_kind, + id=resource_id, + attrs=admin_attrs, + request=request, + view=view, + ): return True - obj = view.get_object() + + obj = target_obj if isinstance(view, v1_views.HostDetail): hostname, ips = self._get_hostname_and_ips(obj) # If renaming a host, make sure the user has permission to both the # new and and old hostname. if 'name' in data: - if not self.has_perm(user, data['name'], ips, request=request, view=view): + if not self.has_perm( + user, + data['name'], + ips, + request=request, + view=view, + action=action, + resource_kind=resource_kind, + resource_id=self._stringify_attr_value(data['name']), + ): return False - return self.has_perm(user, hostname, ips, request=request, view=view) + return self.has_perm( + user, + hostname, + ips, + request=request, + view=view, + action=action, + resource_kind=resource_kind, + resource_id=self._stringify_attr_value(hostname), + ) elif hasattr(obj, 'host'): # If changing host object, make sure the user has permission the # new one. if 'host' in data and data['host'] != obj.host: - if not self.has_obj_perm(user, data['host'], request=request, view=view): + if not self.has_obj_perm( + user, + data['host'], + request=request, + view=view, + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + ): return False - return self.has_obj_perm(user, obj.host, request=request, view=view) + return self.has_obj_perm( + user, + obj.host, + request=request, + view=view, + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + ) # Testing these kinds of should-never-happen codepaths is hard. # We have to basically mock a complete API call and then break it. raise exceptions.PermissionDenied(f"Unhandled view: {view}") # pragma: no cover def _get_hostname_and_ips(self, hostobject): + """Extract a host's canonical name and all attached IP addresses.""" ips = [] host = HostSerializer(hostobject) for i in host.data['ipaddresses']: @@ -709,4 +1058,3 @@ def has_destroy_permission(self, request: Request, view: GenericAPIView, validat # in a `BaseModel` instance instead of a serializer when checking # destroy permissions, so we cannot access any sort of validated data. return self.has_permission(request, view) - diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index 77f87088..d98989ee 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -62,9 +62,11 @@ def _is_parity_enabled() -> bool: return not getattr(_thread_local, "skip_parity", False) def _corr_id(request: Request) -> Optional[str]: + """Return request correlation ID from standard header variants.""" return request.headers.get("X-Correlation-ID") or request.META.get("HTTP_X_CORRELATION_ID") def _model_name_from_view(view) -> str: # type: ignore + """Best-effort view model name, preferring serializer Meta.model.""" # Best effort: try serializer model, else view class name try: sc = view.get_serializer_class() @@ -133,8 +135,8 @@ def policy_parity( pol_allowed, error = None, None try: - resp = treetopclient.check(TreeTopRequest(principal=principal, action=pol_action, resource=res)) - pol_allowed = bool(resp.is_allowed()) + resp = treetopclient.authorize(TreeTopRequest(principal=principal, action=pol_action, resource=res)) + pol_allowed = bool(resp.all_allowed()) except Exception as exc: error = repr(exc) # Log policy server errors prominently @@ -175,5 +177,6 @@ def policy_parity( # Log data to a file in addition to normal logging def log_policy_parity(payload: dict[str, Any]): + """Append one JSON parity event to the configured parity log file.""" with open(POLICY_EXTRA_LOG_FILE_NAME, "a") as log_file: log_file.write(f"{json.dumps(payload)}\n") diff --git a/mregsite/settings.py b/mregsite/settings.py index ab1dca30..76d26add 100644 --- a/mregsite/settings.py +++ b/mregsite/settings.py @@ -79,6 +79,7 @@ def parse_protected_attrs(raw: str) -> list[dict]: SECRET_KEY = ")e#67040xjxar=zl^y#@#b*zilv2dxtraj582$^(e6!wf++_n#" LOG_LEVEL = envvar("MREG_LOG_LEVEL", "CRITICAL").upper() +POLICY_PARITY_LOG_LEVEL = envvar("MREG_POLICY_PARITY_LOG_LEVEL", "WARNING").upper() REQUESTS_THRESHOLD_SLOW = envvar("MREG_REQUESTS_THRESHOLD_SLOW", 1000) REQUESTS_LOG_LEVEL_SLOW = envvar("MREG_REQUESTS_LOG_LEVEL_SLOW", "WARNING") @@ -405,6 +406,19 @@ def parse_protected_attrs(raw: str) -> list[dict]: "filename": LOG_FILE_NAME, "formatter": "plain", }, + "policy_parity_default": { + "level": POLICY_PARITY_LOG_LEVEL, + "class": "logging.StreamHandler", + "formatter": "colored", + }, + "policy_parity_file": { + "level": POLICY_PARITY_LOG_LEVEL, + "class": "logging.handlers.RotatingFileHandler", + "maxBytes": LOG_FILE_SIZE, + "backupCount": LOG_FILE_COUNT, + "filename": LOG_FILE_NAME, + "formatter": "plain", + }, }, "loggers": { "": { @@ -412,6 +426,11 @@ def parse_protected_attrs(raw: str) -> list[dict]: "level": "DEBUG", "propagate": True, }, + "mreg.policy.parity": { + "handlers": ["policy_parity_default", "policy_parity_file"], + "level": POLICY_PARITY_LOG_LEVEL, + "propagate": False, + }, }, } ) diff --git a/pyproject.toml b/pyproject.toml index 8a280eff..7305a346 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "pyyaml", # For testing inside Docker image "unittest-parametrize", - "treetop-client>=0.0.7", + "treetop-client>=0.0.1", "prometheus-client>=0.24", ] dynamic = ["version"] @@ -39,6 +39,7 @@ dev = [ "pytest", "pytest-django", "uv>=0.10", + "tblib>=3", ] ci = [ # Explictly include dev group for non-uv package managers diff --git a/treetop/data/host_labels.json b/treetop/data/host_labels.json deleted file mode 100644 index 619c36dd..00000000 --- a/treetop/data/host_labels.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "name": "in_domain", - "regex": "example\\.com$" - }, - { - "name": "valid_webserver_name", - "regex": "^web-\\d+" - }, - { - "name": "admin_subdomain", - "regex": "^admin\\." - }, - { - "name": "staging_environment", - "regex": "^staging\\." - } -] \ No newline at end of file diff --git a/treetop/data/labels.json b/treetop/data/labels.json new file mode 100644 index 00000000..9494941b --- /dev/null +++ b/treetop/data/labels.json @@ -0,0 +1,25 @@ +[ + { + "kind": "Host", + "field": "name", + "output": "nameLabels", + "patterns": [ + { + "name": "in_domain", + "regex": "example\\.com$" + }, + { + "name": "valid_webserver_name", + "regex": "^web-\\d+" + }, + { + "name": "admin_subdomain", + "regex": "^admin\\." + }, + { + "name": "staging_environment", + "regex": "^staging\\." + } + ] + } +] \ No newline at end of file diff --git a/treetop/data/mreg.cedar b/treetop/data/mreg.cedar index 33bc5fe8..2501ab6b 100644 --- a/treetop/data/mreg.cedar +++ b/treetop/data/mreg.cedar @@ -1,9 +1,87 @@ // MREG permissions example. +// Common CRUD action sets for resource-managed APIs. +// Keep this list in sync with mreg/api/permissions.py::_crud_action. + +@id("MREG.read_all") +permit ( + principal, + action in + [MREG::Action::"host_read", + MREG::Action::"host_contacts_read", + MREG::Action::"ipaddress_read", + MREG::Action::"cname_read", + MREG::Action::"hinfo_read", + MREG::Action::"loc_read", + MREG::Action::"mx_read", + MREG::Action::"naptr_read", + MREG::Action::"name_server_read", + MREG::Action::"ptr_override_read", + MREG::Action::"sshfp_read", + MREG::Action::"srv_read", + MREG::Action::"txt_read", + MREG::Action::"bacnet_id_read", + MREG::Action::"community_read"], + resource +); + +@id("MREG.admin_crud") +permit ( + principal in MREG::Group::"default-admin-group", + action in + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"host_delete", + MREG::Action::"ipaddress_create", + MREG::Action::"ipaddress_update", + MREG::Action::"ipaddress_delete", + MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete", + MREG::Action::"hinfo_create", + MREG::Action::"hinfo_update", + MREG::Action::"hinfo_delete", + MREG::Action::"loc_create", + MREG::Action::"loc_update", + MREG::Action::"loc_delete", + MREG::Action::"mx_create", + MREG::Action::"mx_update", + MREG::Action::"mx_delete", + MREG::Action::"naptr_create", + MREG::Action::"naptr_update", + MREG::Action::"naptr_delete", + MREG::Action::"name_server_create", + MREG::Action::"name_server_update", + MREG::Action::"name_server_delete", + MREG::Action::"ptr_override_create", + MREG::Action::"ptr_override_update", + MREG::Action::"ptr_override_delete", + MREG::Action::"sshfp_create", + MREG::Action::"sshfp_update", + MREG::Action::"sshfp_delete", + MREG::Action::"srv_create", + MREG::Action::"srv_update", + MREG::Action::"srv_delete", + MREG::Action::"txt_create", + MREG::Action::"txt_update", + MREG::Action::"txt_delete", + MREG::Action::"bacnet_id_create", + MREG::Action::"bacnet_id_update", + MREG::Action::"bacnet_id_delete", + MREG::Action::"community_create", + MREG::Action::"community_update", + MREG::Action::"community_delete"], + resource +); + @id("MREG.admins_policy") permit ( principal in MREG::Group::"admins", - action in MREG::Action::"host_access", + action in + [MREG::Action::"host_create", + MREG::Action::"host_read", + MREG::Action::"host_update", + MREG::Action::"host_delete"], resource is Host ); @@ -12,7 +90,11 @@ permit ( @id("MREG.webadmins_policy") permit ( principal in MREG::Group::"webadmins", - action in MREG::Action::"host_access", + action in + [MREG::Action::"host_create", + MREG::Action::"host_read", + MREG::Action::"host_update", + MREG::Action::"host_delete"], resource is Host ) when @@ -40,10 +122,159 @@ permit ( @id("MREG.test_group_policy") permit ( principal in MREG::Group::"testgroup", - action in [MREG::Action::"host_access"], - resource is Host + action in + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"host_delete", + MREG::Action::"ipaddress_create", + MREG::Action::"ipaddress_update", + MREG::Action::"ipaddress_delete", + MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete", + MREG::Action::"hinfo_create", + MREG::Action::"hinfo_update", + MREG::Action::"hinfo_delete", + MREG::Action::"loc_create", + MREG::Action::"loc_update", + MREG::Action::"loc_delete", + MREG::Action::"mx_create", + MREG::Action::"mx_update", + MREG::Action::"mx_delete", + MREG::Action::"naptr_create", + MREG::Action::"naptr_update", + MREG::Action::"naptr_delete", + MREG::Action::"name_server_create", + MREG::Action::"name_server_update", + MREG::Action::"name_server_delete", + MREG::Action::"ptr_override_create", + MREG::Action::"ptr_override_update", + MREG::Action::"ptr_override_delete", + MREG::Action::"sshfp_create", + MREG::Action::"sshfp_update", + MREG::Action::"sshfp_delete", + MREG::Action::"srv_create", + MREG::Action::"srv_update", + MREG::Action::"srv_delete", + MREG::Action::"txt_create", + MREG::Action::"txt_update", + MREG::Action::"txt_delete", + MREG::Action::"bacnet_id_create", + MREG::Action::"bacnet_id_update", + MREG::Action::"bacnet_id_delete", + MREG::Action::"community_create", + MREG::Action::"community_update", + MREG::Action::"community_delete"], + resource ) when { - resource.ip.isInRange(ip("10.0.0.0/24")) + resource has hostname && + ( + ( + action in + [MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete"] && + !(resource has ip) && + resource.hostname like "ho*.example.org" + ) || + ( + !(action in + [MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete"]) && + resource.hostname like "*.example.org" && + resource has ip && + ( + resource.ip.isInRange(ip("10.0.0.0/24")) || + resource.ip.isInRange(ip("10.1.0.0/25")) || + resource.ip.isInRange(ip("192.168.1.0/24")) || + resource.ip.isInRange(ip("192.168.2.1/32")) || + resource.ip.isInRange(ip("2001:db8::/64")) || + resource.ip.isInRange(ip("2001:db8::1/128")) || + resource.ip.isInRange(ip("2002:db9::/64")) + ) + ) + ) +}; + +/// Network admin permissions used in tests where network-admin users also +/// receive NetGroupRegexPermission entries. +@id("MREG.network_admin_group_policy") +permit ( + principal in MREG::Group::"default-networkadmin-group", + action in + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"host_delete", + MREG::Action::"ipaddress_create", + MREG::Action::"ipaddress_update", + MREG::Action::"ipaddress_delete", + MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete", + MREG::Action::"hinfo_create", + MREG::Action::"hinfo_update", + MREG::Action::"hinfo_delete", + MREG::Action::"loc_create", + MREG::Action::"loc_update", + MREG::Action::"loc_delete", + MREG::Action::"mx_create", + MREG::Action::"mx_update", + MREG::Action::"mx_delete", + MREG::Action::"naptr_create", + MREG::Action::"naptr_update", + MREG::Action::"naptr_delete", + MREG::Action::"name_server_create", + MREG::Action::"name_server_update", + MREG::Action::"name_server_delete", + MREG::Action::"ptr_override_create", + MREG::Action::"ptr_override_update", + MREG::Action::"ptr_override_delete", + MREG::Action::"sshfp_create", + MREG::Action::"sshfp_update", + MREG::Action::"sshfp_delete", + MREG::Action::"srv_create", + MREG::Action::"srv_update", + MREG::Action::"srv_delete", + MREG::Action::"txt_create", + MREG::Action::"txt_update", + MREG::Action::"txt_delete", + MREG::Action::"bacnet_id_create", + MREG::Action::"bacnet_id_update", + MREG::Action::"bacnet_id_delete", + MREG::Action::"community_create", + MREG::Action::"community_update", + MREG::Action::"community_delete"], + resource +) when { + resource has hostname && + ( + ( + action in + [MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete"] && + !(resource has ip) && + resource.hostname like "ho*.example.org" + ) || + ( + !(action in + [MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete"]) && + resource.hostname like "*.example.org" && + resource has ip && + ( + resource.ip.isInRange(ip("10.0.0.0/24")) || + resource.ip.isInRange(ip("10.1.0.0/25")) || + resource.ip.isInRange(ip("192.168.1.0/24")) || + resource.ip.isInRange(ip("192.168.2.1/32")) || + resource.ip.isInRange(ip("2001:db8::/64")) || + resource.ip.isInRange(ip("2001:db8::1/128")) || + resource.ip.isInRange(ip("2002:db9::/64")) + ) + ) + ) }; /// Network Admins can manage any IP in any network @@ -87,7 +318,7 @@ permit ( resource ); -/// Normal (?) admins +/// Normal admins @id("MREG.admin") permit ( principal in MREG::Group::"default-admin-group", @@ -95,7 +326,23 @@ permit ( resource ); -/// Host Policy Admins +/// Network admins (group-membership permission check) +@id("MREG.network_admin") +permit ( + principal in MREG::Group::"default-networkadmin-group", + action == MREG::Action::"network_admin_access", + resource +); + +/// Host group admins (group-membership permission check) +@id("MREG.hostgroup_admin") +permit ( + principal in MREG::Group::"default-groupadmin-group", + action == MREG::Action::"hostgroup_admin_access", + resource +); + +/// Host Policy Admins @id("MREG.hostpolicy_admin") permit ( principal in MREG::Group::"default-hostpolicyadmin-group", @@ -103,7 +350,7 @@ permit ( resource ); -/// DNS Wildcard Admins +/// DNS Wildcard Admins @id("MREG.dns_wildcard_admin") permit ( principal in MREG::Group::"default-dns-wildcard-group", @@ -126,4 +373,4 @@ permit ( principal == User::"super", action, resource -); \ No newline at end of file +); diff --git a/treetop/docker-compose.yml b/treetop/docker-compose.yml index 381e1fa6..07345501 100644 --- a/treetop/docker-compose.yml +++ b/treetop/docker-compose.yml @@ -14,7 +14,11 @@ services: ports: - "9999:9999" environment: - - APP_POLICY_URL=http://cedar-server:8080/mreg.cedar - - APP_HOST_LABEL_URL=http://cedar-server:8080/host_labels.json - - RUST_LOG=info,treetop=debug - command: ["server", "--host", "0.0.0.0"] + - TREETOP_POLICY_URL=http://cedar-server:8080/mreg.cedar + - TREETOP_LABELS_URL=http://cedar-server:8080/labels.json + - TREETOP_PORT=9999 + - TREETOP_CLIENT_ALLOWLIST=* + - TREETOP_LISTEN=0.0.0.0 + - RUST_LOG=mio=warn,actix_server=warn,actix_http=warn,hyper_util=warn,reqwest=warn,info + healthcheck: + test: ["NONE"] diff --git a/uv.lock b/uv.lock index c6484f77..f7017b63 100644 --- a/uv.lock +++ b/uv.lock @@ -479,6 +479,7 @@ ci = [ { name = "coveralls" }, { name = "pytest" }, { name = "pytest-django" }, + { name = "tblib" }, { name = "tox-gh-actions" }, { name = "tox-uv" }, { name = "uv" }, @@ -487,6 +488,7 @@ dev = [ { name = "coverage" }, { name = "pytest" }, { name = "pytest-django" }, + { name = "tblib" }, { name = "tox-uv" }, { name = "uv" }, ] @@ -503,13 +505,13 @@ requires-dist = [ { name = "gunicorn", specifier = ">=23.0.0" }, { name = "idna", specifier = ">=3.11" }, { name = "pika", specifier = ">=1.3.2" }, - { name = "prometheus-client", specifier = ">=0.24" }, + { name = "prometheus-client", specifier = ">=0.20" }, { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3" }, { name = "pyyaml" }, { name = "rich", specifier = ">=14" }, { name = "sentry-sdk", specifier = ">=2.48.0" }, { name = "structlog", specifier = ">=25" }, - { name = "treetop-client", specifier = ">=0.0.7" }, + { name = "treetop-client", specifier = ">=0.0.1" }, { name = "tzdata", specifier = ">=2025.3" }, { name = "unittest-parametrize" }, { name = "uritemplate" }, @@ -521,6 +523,7 @@ ci = [ { name = "coveralls" }, { name = "pytest" }, { name = "pytest-django" }, + { name = "tblib", specifier = ">=3" }, { name = "tox-gh-actions" }, { name = "tox-uv", specifier = ">=1.29" }, { name = "uv", specifier = ">=0.10" }, @@ -529,6 +532,7 @@ dev = [ { name = "coverage", extras = ["toml"] }, { name = "pytest" }, { name = "pytest-django" }, + { name = "tblib", specifier = ">=3" }, { name = "tox-uv", specifier = ">=1.29" }, { name = "uv", specifier = ">=0.10" }, ] @@ -875,6 +879,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/45/a132b9074aa18e799b891b91ad72133c98d8042c70f6240e4c5f9dabee2f/structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f", size = 72510, upload-time = "2025-10-27T08:28:21.535Z" }, ] +[[package]] +name = "tblib" +version = "3.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/8a/14c15ae154895cc131174f858c707790d416c444fc69f93918adfd8c4c0b/tblib-3.2.2.tar.gz", hash = "sha256:e9a652692d91bf4f743d4a15bc174c0b76afc750fe8c7b6d195cc1c1d6d2ccec", size = 35046, upload-time = "2025-11-12T12:21:16.572Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/be/5d2d47b1fb58943194fb59dcf222f7c4e35122ec0ffe8c36e18b5d728f0b/tblib-3.2.2-py3-none-any.whl", hash = "sha256:26bdccf339bcce6a88b2b5432c988b266ebbe63a4e593f6b578b1d2e723d2b76", size = 12893, upload-time = "2025-11-12T12:21:14.407Z" }, +] + [[package]] name = "tox" version = "4.35.0" From a501885b381cce1fbf62ba94f4baeb772fa577e2 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sat, 14 Feb 2026 18:43:08 +0100 Subject: [PATCH 11/34] Batching support (optional, defaults to True). --- docs/env.md | 8 + mreg/api/treetop.py | 226 ++++++++++++++++++++++------ mreg/api/v1/tests/test_logging.py | 19 +++ mreg/middleware/logging_http.py | 14 +- mreg/tests/test_treetop_batching.py | 184 ++++++++++++++++++++++ mregsite/settings.py | 1 + 6 files changed, 402 insertions(+), 50 deletions(-) create mode 100644 mreg/tests/test_treetop_batching.py diff --git a/docs/env.md b/docs/env.md index 59af27ce..bdc145e8 100644 --- a/docs/env.md +++ b/docs/env.md @@ -28,6 +28,14 @@ Must be one of the following: - `ERROR` - `CRITICAL` +## `MREG_POLICY_PARITY_BATCH_ENABLED` + +Boolean flag controlling request-scoped batching of parity authorize checks. +Default: `True` + +When enabled, parity checks are queued during request handling and flushed as a +single batch call to the policy `authorize` endpoint. + ## `MREG_LOG_FILE_SIZE` Maximum file size of the log file in bytes. Default: `52428800` (50MB). diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index d98989ee..07ef0410 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging -from typing import Any, Optional, Mapping +from dataclasses import dataclass +from typing import Any, Optional, Mapping, Sequence import ipaddress import json import threading @@ -26,6 +27,7 @@ POLICY_NAMESPACE = getattr(settings, "POLICY_NAMESPACE", ["MREG"]) POLICY_EXTRA_LOG_FILE_NAME = getattr(settings, "POLICY_EXTRA_LOG_FILE_NAME", "policy_parity.log") POLICY_TRUNCATE_LOG_FILE = getattr(settings, "POLICY_TRUNCATE_LOG_FILE", True) +POLICY_PARITY_BATCH_ENABLED = getattr(settings, "POLICY_PARITY_BATCH_ENABLED", True) if POLICY_TRUNCATE_LOG_FILE: with open(POLICY_EXTRA_LOG_FILE_NAME, "w"): @@ -33,6 +35,56 @@ treetopclient = TreeTopClient(base_url=POLICY_BASE_URL) + +@dataclass(slots=True) +class _ParityBatchItem: + """One queued parity check, to be sent via batch authorize.""" + + decision: bool + policy_request: TreeTopRequest + context: dict[str, Any] + + +def _batch_depth() -> int: + return int(getattr(_thread_local, "batch_depth", 0)) + + +def _batch_queue() -> list[_ParityBatchItem]: + queue = getattr(_thread_local, "batch_queue", None) + if queue is None: + queue = [] + _thread_local.batch_queue = queue + return queue + + +def _is_batching() -> bool: + return _batch_depth() > 0 and POLICY_PARITY_BATCH_ENABLED + + +@contextmanager +def batch_policy_parity(): + """Batch parity checks in the current thread and flush on scope exit.""" + if not POLICY_PARITY_BATCH_ENABLED: + yield + return + + current_depth = _batch_depth() + if current_depth == 0: + _thread_local.batch_queue = [] + _thread_local.batch_depth = current_depth + 1 + + try: + yield + finally: + new_depth = _batch_depth() - 1 + _thread_local.batch_depth = max(new_depth, 0) + if new_depth <= 0: + try: + flush_policy_parity_batch() + finally: + _thread_local.batch_queue = [] + _thread_local.batch_depth = 0 + @contextmanager def disable_policy_parity(): """Context manager to temporarily disable policy parity checking. @@ -74,6 +126,112 @@ def _model_name_from_view(view) -> str: # type: ignore except Exception: return view.__class__.__name__ + +def _build_resource_attrs(resource_attrs: Mapping[str, str]) -> dict[str, ResourceAttribute]: + attrs: dict[str, ResourceAttribute] = {} + for key, value in resource_attrs.items(): + try: + ip = ipaddress.ip_address(value) + attrs[key] = ResourceAttribute.new(str(ip), ResourceAttributeType.IP) + except ValueError: + attrs[key] = ResourceAttribute.new(value, ResourceAttributeType.STRING) + return attrs + + +def _fully_qualified_action(action: Action) -> str: + if len(action.id.namespace) > 0: + return "::".join(action.id.namespace) + f"::{action.id.id}" + return f"{action.id.id}" + + +def _compute_parity_payload( + *, + decision: bool, + pol_allowed: Optional[bool], + error: Optional[str], + context: dict[str, Any], +) -> dict[str, Any]: + parity = False + if bool(decision) and pol_allowed: + parity = True + elif not bool(decision) and pol_allowed is False: + parity = True + + return { + "parity": parity, + "legacy_decision": bool(decision), + "policy_decision": pol_allowed, + "error": error, + "context": context, + } + + +def _log_parity_payload(payload: dict[str, Any]) -> None: + if payload["parity"]: + logger.info("policy_parity_ok", extra=payload) + else: + logger.warning("policy_parity_mismatch", extra=payload) + log_policy_parity(payload) + + +def _result_to_decision_and_error( + results: Sequence[Any], + index: int, +) -> tuple[Optional[bool], Optional[str]]: + if index >= len(results): + return None, f"Missing policy result at index {index}" + + result = results[index] + if hasattr(result, "is_success") and result.is_success(): + return bool(result.is_allowed()), None + + status = getattr(result, "status", "unknown") + error = getattr(result, "error", None) or f"Authorization failed with status={status}" + return None, str(error) + + +def flush_policy_parity_batch() -> None: + """Flush queued parity checks as one authorize batch call.""" + queue = _batch_queue() + if not queue: + return + + correlation_id = queue[0].context.get("correlation_id") + pol_allowed_by_index: list[Optional[bool]] = [None] * len(queue) + error_by_index: list[Optional[str]] = [None] * len(queue) + + try: + response = treetopclient.authorize( + [item.policy_request for item in queue], + correlation_id=correlation_id, + ) + results = list(getattr(response, "results", [])) + for idx in range(len(queue)): + pol_allowed_by_index[idx], error_by_index[idx] = _result_to_decision_and_error(results, idx) + except Exception as exc: + error = repr(exc) + logger.error( + f"Policy server error: {type(exc).__name__}: {exc}", + extra={ + "error_type": type(exc).__name__, + "error_msg": str(exc), + "path": queue[0].context.get("path"), + "correlation_id": correlation_id, + "batch_size": len(queue), + }, + ) + for idx in range(len(queue)): + error_by_index[idx] = error + + for idx, item in enumerate(queue): + payload = _compute_parity_payload( + decision=item.decision, + pol_allowed=pol_allowed_by_index[idx], + error=error_by_index[idx], + context=item.context, + ) + _log_parity_payload(payload) + def policy_parity( decision: bool, *, @@ -96,28 +254,10 @@ def policy_parity( muser = MregUser.from_request(request) principal = TreeTopUser.new(str(muser.username), POLICY_NAMESPACE, groups=list(muser.group_list)) pol_action = Action.new(action, POLICY_NAMESPACE) - - attrs = {} - - for k, v in resource_attrs.items(): - try: - ip = ipaddress.ip_address(v) - attrs[k] = ResourceAttribute.new(str(ip), ResourceAttributeType.IP) - except ValueError: - attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) -# if v.isdigit(): -# attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.NUMBER) -# elif v.lower() in ("true", "false"): -# attrs[k] = ResourceAttribute.new(v.lower(), ResourceAttributeType.BOOLEAN) -# else: -# attrs[k] = ResourceAttribute.new(v, ResourceAttributeType.STRING) - + attrs = _build_resource_attrs(resource_attrs) res = Resource.new(str(resource_kind), resource_id, attrs=attrs) - - if len(pol_action.id.namespace) > 0: - fully_qualified_action = "::".join(pol_action.id.namespace) + f"::{pol_action.id.id}" - else: - fully_qualified_action = f"{pol_action.id.id}" + pol_request = TreeTopRequest(principal=principal, action=pol_action, resource=res) + fully_qualified_action = _fully_qualified_action(pol_action) context = { "path": request.path, @@ -133,10 +273,21 @@ def policy_parity( "correlation_id": _corr_id(request), } + if _is_batching(): + _batch_queue().append( + _ParityBatchItem( + decision=bool(decision), + policy_request=pol_request, + context=context, + ) + ) + return decision + pol_allowed, error = None, None try: - resp = treetopclient.authorize(TreeTopRequest(principal=principal, action=pol_action, resource=res)) - pol_allowed = bool(resp.all_allowed()) + resp = treetopclient.authorize(pol_request, correlation_id=context["correlation_id"]) + results = list(getattr(resp, "results", [])) + pol_allowed, error = _result_to_decision_and_error(results, 0) except Exception as exc: error = repr(exc) # Log policy server errors prominently @@ -152,26 +303,13 @@ def policy_parity( # If policy server fails, we cannot determine parity. Return legacy decision # but flag this in the payload for monitoring. - parity = False - if bool(decision) and pol_allowed: - parity = True - elif not bool(decision) and not pol_allowed: - parity = True - - payload: dict[str, Any] = { - "parity": parity, - "legacy_decision": bool(decision), - "policy_decision": pol_allowed, - "error": error, - "context": context, - } - - if parity: - logger.info("policy_parity_ok", extra=payload) - log_policy_parity(payload) - else: - logger.warning("policy_parity_mismatch", extra=payload) - log_policy_parity(payload) + payload = _compute_parity_payload( + decision=decision, + pol_allowed=pol_allowed, + error=error, + context=context, + ) + _log_parity_payload(payload) return decision diff --git a/mreg/api/v1/tests/test_logging.py b/mreg/api/v1/tests/test_logging.py index 60352090..c82fac12 100644 --- a/mreg/api/v1/tests/test_logging.py +++ b/mreg/api/v1/tests/test_logging.py @@ -143,6 +143,25 @@ def mock_get_response(_): # Check that the body was logged as '' self.assertEqual(cap_logs[0]["content"], "") + def test_middleware_uses_policy_parity_batching_context(self) -> None: + """Ensure request handling is wrapped in the parity batching context.""" + middleware = LoggingMiddleware(MagicMock()) + + def mock_get_response(_): + return HttpResponse(status=200) + + middleware.get_response = mock_get_response + + request = HttpRequest() + request._body = b"Some request body" + request.user = get_user_model().objects.get(username="superuser") + + with patch("mreg.middleware.logging_http.batch_policy_parity") as mock_batch: + middleware(request) + mock_batch.assert_called_once() + mock_batch.return_value.__enter__.assert_called_once() + mock_batch.return_value.__exit__.assert_called_once() + class TestLoggingMiddleware(MregAPITestCase): """Test logging middleware.""" diff --git a/mreg/middleware/logging_http.py b/mreg/middleware/logging_http.py index a7e9724e..46dd5edb 100644 --- a/mreg/middleware/logging_http.py +++ b/mreg/middleware/logging_http.py @@ -10,6 +10,7 @@ import traceback from django.conf import settings from django.http import HttpRequest, HttpResponse +from mreg.api.treetop import batch_policy_parity mreg_logger = structlog.getLogger("mreg.http") @@ -47,11 +48,12 @@ def __call__(self, request: HttpRequest) -> HttpResponse: self.log_request(request) - try: - response = self.get_response(request) - except Exception as e: # pragma: no cover (this is somewhat tricky to properly test) - self.log_exception(request, e, start_time) - raise + with batch_policy_parity(): + try: + response = self.get_response(request) + except Exception as e: # pragma: no cover (this is somewhat tricky to properly test) + self.log_exception(request, e, start_time) + raise self.log_response(request, response, start_time) return response @@ -219,4 +221,4 @@ def log_exception(self, request: HttpRequest, exception: Exception, start_time: scope.set_extra("request_body", self._get_body(request)) # Capture the exception - sentry_sdk.capture_exception(exception) \ No newline at end of file + sentry_sdk.capture_exception(exception) diff --git a/mreg/tests/test_treetop_batching.py b/mreg/tests/test_treetop_batching.py new file mode 100644 index 00000000..e3ad9f67 --- /dev/null +++ b/mreg/tests/test_treetop_batching.py @@ -0,0 +1,184 @@ +from types import SimpleNamespace +from unittest.mock import patch + +from django.http import HttpRequest, HttpResponse +from django.test import SimpleTestCase + +from mreg.api.treetop import _thread_local, batch_policy_parity, policy_parity +from mreg.middleware.logging_http import LoggingMiddleware + + +class _DummyAuthorizeResult: + def __init__(self, allowed: bool) -> None: + self._allowed = allowed + self.status = "success" + self.error = None + + def is_success(self) -> bool: + return True + + def is_allowed(self) -> bool: + return self._allowed + + +class _DummyAuthorizeResponse: + def __init__(self, decisions: list[bool]) -> None: + self.results = [_DummyAuthorizeResult(decision) for decision in decisions] + + +class TreeTopParityBatchingTests(SimpleTestCase): + def tearDown(self) -> None: + _thread_local.batch_queue = [] + _thread_local.batch_depth = 0 + super().tearDown() + + @staticmethod + def _request() -> HttpRequest: + request = HttpRequest() + request.method = "GET" + request.path = "/api/v1/hosts/" + request.META["HTTP_X_CORRELATION_ID"] = "test-correlation-id" + request.user = SimpleNamespace(is_authenticated=True) + return request + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop.log_policy_parity") + def test_batch_policy_parity_uses_single_authorize_call( + self, + mock_log_policy_parity, + mock_from_request, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + + calls: list[tuple[int, str | None]] = [] + + def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-def] + request_list = requests if isinstance(requests, list) else [requests] + calls.append((len(request_list), correlation_id)) + return _DummyAuthorizeResponse([True] * len(request_list)) + + request = self._request() + with ( + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), + patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), + batch_policy_parity(), + ): + self.assertTrue( + policy_parity( + True, + request=request, + action="host_read", + resource_kind="Host", + resource_id="host1.example.org", + resource_attrs={"kind": "host", "hostname": "host1.example.org"}, + ) + ) + self.assertFalse( + policy_parity( + False, + request=request, + action="host_read", + resource_kind="Host", + resource_id="host2.example.org", + resource_attrs={"kind": "host", "hostname": "host2.example.org"}, + ) + ) + + self.assertEqual(calls, [(2, "test-correlation-id")]) + self.assertEqual(mock_log_policy_parity.call_count, 2) + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop.log_policy_parity") + def test_policy_parity_without_batch_context_calls_authorize_per_check( + self, + mock_log_policy_parity, + mock_from_request, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + + calls: list[int] = [] + + def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-def] + request_list = requests if isinstance(requests, list) else [requests] + calls.append(len(request_list)) + return _DummyAuthorizeResponse([True] * len(request_list)) + + request = self._request() + with ( + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), + patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), + ): + policy_parity( + True, + request=request, + action="host_read", + resource_kind="Host", + resource_id="host1.example.org", + resource_attrs={"kind": "host", "hostname": "host1.example.org"}, + ) + policy_parity( + True, + request=request, + action="host_read", + resource_kind="Host", + resource_id="host2.example.org", + resource_attrs={"kind": "host", "hostname": "host2.example.org"}, + ) + + self.assertEqual(calls, [1, 1]) + self.assertEqual(mock_log_policy_parity.call_count, 2) + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop.log_policy_parity") + def test_single_http_request_flushes_one_authorize_batch( + self, + mock_log_policy_parity, + mock_from_request, + ) -> None: + """Verify one policy-engine query for one request with multiple parity checks.""" + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + + calls: list[tuple[int, str | None]] = [] + + def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-def] + request_list = requests if isinstance(requests, list) else [requests] + calls.append((len(request_list), correlation_id)) + return _DummyAuthorizeResponse([True] * len(request_list)) + + request = self._request() + request.path_info = request.path + request._body = b"" + request.user = SimpleNamespace(username="tester") + + def mock_get_response(http_request: HttpRequest) -> HttpResponse: + policy_parity( + True, + request=http_request, + action="host_read", + resource_kind="Host", + resource_id="host1.example.org", + resource_attrs={"kind": "host", "hostname": "host1.example.org"}, + ) + policy_parity( + True, + request=http_request, + action="host_read", + resource_kind="Host", + resource_id="host2.example.org", + resource_attrs={"kind": "host", "hostname": "host2.example.org"}, + ) + return HttpResponse(status=200) + + middleware = LoggingMiddleware(mock_get_response) + + with ( + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), + patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), + ): + middleware(request) + + self.assertEqual(calls, [(2, "test-correlation-id")]) + self.assertEqual(mock_log_policy_parity.call_count, 2) diff --git a/mregsite/settings.py b/mregsite/settings.py index 76d26add..691846cf 100644 --- a/mregsite/settings.py +++ b/mregsite/settings.py @@ -80,6 +80,7 @@ def parse_protected_attrs(raw: str) -> list[dict]: LOG_LEVEL = envvar("MREG_LOG_LEVEL", "CRITICAL").upper() POLICY_PARITY_LOG_LEVEL = envvar("MREG_POLICY_PARITY_LOG_LEVEL", "WARNING").upper() +POLICY_PARITY_BATCH_ENABLED = envvar("MREG_POLICY_PARITY_BATCH_ENABLED", True) REQUESTS_THRESHOLD_SLOW = envvar("MREG_REQUESTS_THRESHOLD_SLOW", 1000) REQUESTS_LOG_LEVEL_SLOW = envvar("MREG_REQUESTS_LOG_LEVEL_SLOW", "WARNING") From db8200f9250081f809e362ff021bf514ed4b01ab Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sun, 15 Feb 2026 02:12:25 +0100 Subject: [PATCH 12/34] Add policy engine metrics and improve log file handling - Introduced new Prometheus metrics for policy decisions, legacy decisions, and parity results. - Log file management ensures the parity log is truncated only once in the main process. - Updated tests to validate the new metrics and log file behavior. --- docs/metrics.md | 56 +++++++++ docs/parity_testing.md | 2 + mreg/api/tests/test_metrics.py | 18 +-- mreg/api/treetop.py | 123 ++++++++++++++++++-- mreg/tests/prometheus_test_utils.py | 41 +++++++ mreg/tests/test_treetop_batching.py | 174 +++++++++++++++++++++++++++- uv.lock | 2 +- 7 files changed, 388 insertions(+), 28 deletions(-) create mode 100644 mreg/tests/prometheus_test_utils.py diff --git a/docs/metrics.md b/docs/metrics.md index 975c5e8d..b6a0f08c 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -101,6 +101,62 @@ Metrics are exposed at the following endpoint: `/api/meta/metrics`. - Unit: failures - Description: LDAP operation failures by operation and exception class (e.g., bind LDAPError). Useful to see if LDAP is flapping or credential/ACL issues arise. +## Policy Engine Metrics + +- Name: mreg_policy_decisions_total + - Type: Counter + - Labels: decision + - Unit: decisions + - Description: Policy-engine decisions recorded during parity checks. + - Label values: `allow`, `deny`, `error` + +- Name: mreg_policy_legacy_decisions_total + - Type: Counter + - Labels: decision + - Unit: decisions + - Description: Legacy permission decisions compared against policy parity. + - Label values: `allow`, `deny` + +- Name: mreg_policy_parity_results_total + - Type: Counter + - Labels: result + - Unit: comparisons + - Description: Outcome of legacy vs policy parity comparisons. + - Label values: `match`, `mismatch`, `error` + +- Name: mreg_policy_authorize_calls_total + - Type: Counter + - Labels: status + - Unit: calls + - Description: Calls made to the policy `authorize` endpoint. + - Label values: `success`, `exception` + +- Name: mreg_policy_authorize_duration_seconds + - Type: Histogram + - Labels: status + - Unit: seconds + - Description: Duration of policy `authorize` calls. + - Buckets/ranges: `0-1ms`, `1-2.5ms`, `2.5-5ms`, `5-10ms`, `10-25ms`, `25-50ms`, `50-100ms`, `100-250ms`, `250-500ms`, `500ms-1s`, `1-2.5s`, `2.5-5s`, `5s+` + - Prometheus boundaries: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, +Inf] + +- Name: mreg_policy_requests_per_authorize + - Type: Histogram + - Labels: none + - Unit: policy requests + - Description: Number of policy requests included in each `authorize` call. + - Buckets/ranges: `0`, `1`, `2`, `3`, `4-5`, `6-8`, `9+` + - Prometheus boundaries: [0, 1, 2, 3, 5, 8, +Inf] + +- Name: mreg_policy_queries_per_request + - Type: Histogram + - Labels: none + - Unit: authorize queries + - Description: Number of policy `authorize` queries made per HTTP request. + - Buckets/ranges: `0`, `1`, `2`, `3`, `4-5`, `6-8`, `9+` + - Prometheus boundaries: [0, 1, 2, 3, 5, 8, +Inf] + +When request batching is enabled (default), `mreg_policy_queries_per_request` should usually be `0` (no parity checks queued) or `1` (one batched authorize call). + ## Labeling Strategy - path: normalized using Django URL resolution to view name (preferred) or route pattern. Falls back to "unresolved" to avoid cardinality explosion from raw paths with IDs. diff --git a/docs/parity_testing.md b/docs/parity_testing.md index f1959961..6113ecba 100644 --- a/docs/parity_testing.md +++ b/docs/parity_testing.md @@ -111,6 +111,8 @@ Keep parity disable scope as narrow as possible: The `disable_policy_parity()` context manager uses thread-local storage to safely disable parity checking for the current thread only, ensuring test isolation in parallel test execution. +`policy_parity.log` truncation now runs once in the main process only. Parallel test workers append without re-truncating, so a full `tox -e coverage` run keeps one consistent parity log. + ## Parity Runbook Use this sequence when validating parity changes: diff --git a/mreg/api/tests/test_metrics.py b/mreg/api/tests/test_metrics.py index 645f9442..0c33e536 100644 --- a/mreg/api/tests/test_metrics.py +++ b/mreg/api/tests/test_metrics.py @@ -4,26 +4,11 @@ from rest_framework.test import APIClient from django.contrib.auth import get_user_model -import re from typing import Any from mreg.models.host import Host, Ipaddress from mreg.middleware.metrics import PrometheusRequestMiddleware - - -def _parse_prometheus_metric(content: str, metric_name: str) -> dict: - """Parse Prometheus text format and extract metrics by name.""" - result = {} - pattern = rf"^{re.escape(metric_name)}(\{{[^}}]*\}})?\s+([0-9.e+-]+)$" - for line in content.split('\n'): - if line.startswith('#'): - continue - match = re.match(pattern, line) - if match: - labels = match.group(1) or '' - value = float(match.group(2)) - result[labels] = value - return result +from mreg.tests.prometheus_test_utils import parse_prometheus_metric as _parse_prometheus_metric @@ -409,4 +394,3 @@ def test_ldap_metrics_failure_counter(mock_backend: Any) -> None: "operation=\"bind\"" in k and "exception=\"LDAPError\"" in k for k in failures.keys() ), f"Expected LDAPError bind failure metric: {failures}" - diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index 07ef0410..e7775544 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -4,12 +4,16 @@ from typing import Any, Optional, Mapping, Sequence import ipaddress import json +import multiprocessing +import os import threading from contextlib import contextmanager +from time import monotonic from django.conf import settings from rest_framework.request import Request from django.views import View +from prometheus_client import Counter, Histogram from mreg.models.auth import User as MregUser # your request->user wrapper @@ -28,13 +32,73 @@ POLICY_EXTRA_LOG_FILE_NAME = getattr(settings, "POLICY_EXTRA_LOG_FILE_NAME", "policy_parity.log") POLICY_TRUNCATE_LOG_FILE = getattr(settings, "POLICY_TRUNCATE_LOG_FILE", True) POLICY_PARITY_BATCH_ENABLED = getattr(settings, "POLICY_PARITY_BATCH_ENABLED", True) +_POLICY_PARITY_LOG_INITIALIZED_ENV = "MREG_POLICY_PARITY_LOG_INITIALIZED" -if POLICY_TRUNCATE_LOG_FILE: + +def _initialize_policy_parity_log_file() -> None: + """Truncate the parity log once in the main process. + + Parallel test workers import this module too. Guarding on the main process + and an environment marker prevents workers from re-truncating the file. + """ + if not POLICY_TRUNCATE_LOG_FILE: + return + if multiprocessing.current_process().name != "MainProcess": + return + if os.environ.get(_POLICY_PARITY_LOG_INITIALIZED_ENV) == "1": + return with open(POLICY_EXTRA_LOG_FILE_NAME, "w"): pass + os.environ[_POLICY_PARITY_LOG_INITIALIZED_ENV] = "1" + + +_initialize_policy_parity_log_file() treetopclient = TreeTopClient(base_url=POLICY_BASE_URL) +POLICY_DECISIONS_TOTAL = Counter( + "mreg_policy_decisions_total", + "Total policy decisions from the external policy engine.", + ["decision"], +) + +POLICY_LEGACY_DECISIONS_TOTAL = Counter( + "mreg_policy_legacy_decisions_total", + "Total legacy permission decisions used for parity comparison.", + ["decision"], +) + +POLICY_PARITY_RESULTS_TOTAL = Counter( + "mreg_policy_parity_results_total", + "Parity comparison outcomes between legacy and external policy decisions.", + ["result"], +) + +POLICY_AUTHORIZE_CALLS_TOTAL = Counter( + "mreg_policy_authorize_calls_total", + "Total calls to the policy authorize endpoint.", + ["status"], +) + +POLICY_AUTHORIZE_DURATION_SECONDS = Histogram( + "mreg_policy_authorize_duration_seconds", + "Duration of policy authorize endpoint calls in seconds.", + ["status"], + buckets=[0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5], +) + +POLICY_REQUESTS_PER_AUTHORIZE = Histogram( + "mreg_policy_requests_per_authorize", + "Number of policy requests sent in each authorize call.", + buckets=[0, 1, 2, 3, 5, 8], +) + +POLICY_QUERIES_PER_REQUEST = Histogram( + "mreg_policy_queries_per_request", + "Number of policy authorize queries made per HTTP request.", + buckets=[0, 1, 2, 3, 5, 8], +) + @dataclass(slots=True) class _ParityBatchItem: @@ -61,16 +125,22 @@ def _is_batching() -> bool: return _batch_depth() > 0 and POLICY_PARITY_BATCH_ENABLED +def _request_authorize_calls() -> int: + return int(getattr(_thread_local, "policy_authorize_calls", 0)) + + +def _inc_request_authorize_calls() -> None: + _thread_local.policy_authorize_calls = _request_authorize_calls() + 1 + + @contextmanager def batch_policy_parity(): """Batch parity checks in the current thread and flush on scope exit.""" - if not POLICY_PARITY_BATCH_ENABLED: - yield - return - current_depth = _batch_depth() if current_depth == 0: - _thread_local.batch_queue = [] + _thread_local.policy_authorize_calls = 0 + if POLICY_PARITY_BATCH_ENABLED: + _thread_local.batch_queue = [] _thread_local.batch_depth = current_depth + 1 try: @@ -80,10 +150,13 @@ def batch_policy_parity(): _thread_local.batch_depth = max(new_depth, 0) if new_depth <= 0: try: - flush_policy_parity_batch() + if POLICY_PARITY_BATCH_ENABLED: + flush_policy_parity_batch() finally: + POLICY_QUERIES_PER_REQUEST.observe(float(_request_authorize_calls())) _thread_local.batch_queue = [] _thread_local.batch_depth = 0 + _thread_local.policy_authorize_calls = 0 @contextmanager def disable_policy_parity(): @@ -167,6 +240,27 @@ def _compute_parity_payload( def _log_parity_payload(payload: dict[str, Any]) -> None: + legacy_decision = payload.get("legacy_decision") + if legacy_decision is True: + POLICY_LEGACY_DECISIONS_TOTAL.labels(decision="allow").inc() + else: + POLICY_LEGACY_DECISIONS_TOTAL.labels(decision="deny").inc() + + policy_decision = payload.get("policy_decision") + if policy_decision is True: + POLICY_DECISIONS_TOTAL.labels(decision="allow").inc() + elif policy_decision is False: + POLICY_DECISIONS_TOTAL.labels(decision="deny").inc() + else: + POLICY_DECISIONS_TOTAL.labels(decision="error").inc() + + if payload.get("error") is not None or policy_decision is None: + POLICY_PARITY_RESULTS_TOTAL.labels(result="error").inc() + elif payload["parity"]: + POLICY_PARITY_RESULTS_TOTAL.labels(result="match").inc() + else: + POLICY_PARITY_RESULTS_TOTAL.labels(result="mismatch").inc() + if payload["parity"]: logger.info("policy_parity_ok", extra=payload) else: @@ -197,18 +291,25 @@ def flush_policy_parity_batch() -> None: return correlation_id = queue[0].context.get("correlation_id") + POLICY_REQUESTS_PER_AUTHORIZE.observe(float(len(queue))) + _inc_request_authorize_calls() pol_allowed_by_index: list[Optional[bool]] = [None] * len(queue) error_by_index: list[Optional[str]] = [None] * len(queue) + call_started = monotonic() try: response = treetopclient.authorize( [item.policy_request for item in queue], correlation_id=correlation_id, ) + POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="success").inc() + POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="success").observe(monotonic() - call_started) results = list(getattr(response, "results", [])) for idx in range(len(queue)): pol_allowed_by_index[idx], error_by_index[idx] = _result_to_decision_and_error(results, idx) except Exception as exc: + POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="exception").inc() + POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="exception").observe(monotonic() - call_started) error = repr(exc) logger.error( f"Policy server error: {type(exc).__name__}: {exc}", @@ -284,11 +385,19 @@ def policy_parity( return decision pol_allowed, error = None, None + _inc_request_authorize_calls() + call_started = monotonic() try: resp = treetopclient.authorize(pol_request, correlation_id=context["correlation_id"]) + POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="success").inc() + POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="success").observe(monotonic() - call_started) + POLICY_REQUESTS_PER_AUTHORIZE.observe(1.0) results = list(getattr(resp, "results", [])) pol_allowed, error = _result_to_decision_and_error(results, 0) except Exception as exc: + POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="exception").inc() + POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="exception").observe(monotonic() - call_started) + POLICY_REQUESTS_PER_AUTHORIZE.observe(1.0) error = repr(exc) # Log policy server errors prominently logger.error( diff --git a/mreg/tests/prometheus_test_utils.py b/mreg/tests/prometheus_test_utils.py new file mode 100644 index 00000000..2ae7b2ec --- /dev/null +++ b/mreg/tests/prometheus_test_utils.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import re + +from prometheus_client import generate_latest + + +def parse_prometheus_metric(content: str, metric_name: str) -> dict[str, float]: + """Parse Prometheus text exposition and return samples for one metric name.""" + result: dict[str, float] = {} + pattern = rf"^{re.escape(metric_name)}(\{{[^}}]*\}})?\s+([0-9.e+-]+)$" + for line in content.split("\n"): + if line.startswith("#"): + continue + match = re.match(pattern, line) + if match: + labels = match.group(1) or "" + result[labels] = float(match.group(2)) + return result + + +def prometheus_registry_text() -> str: + """Return the current default Prometheus registry text format.""" + return generate_latest().decode("utf-8") + + +def metric_by_label(metric_name: str, label_filter: str, *, content: str | None = None) -> float: + """Return metric value for the first sample whose label set contains label_filter.""" + raw = content if content is not None else prometheus_registry_text() + values = parse_prometheus_metric(raw, metric_name) + for labels, value in values.items(): + if label_filter in labels: + return value + return 0.0 + + +def metric_total(metric_name: str, *, content: str | None = None) -> float: + """Return sum of all samples for a metric from Prometheus text content.""" + raw = content if content is not None else prometheus_registry_text() + values = parse_prometheus_metric(raw, metric_name) + return sum(values.values()) if values else 0.0 diff --git a/mreg/tests/test_treetop_batching.py b/mreg/tests/test_treetop_batching.py index e3ad9f67..1cc3b7a8 100644 --- a/mreg/tests/test_treetop_batching.py +++ b/mreg/tests/test_treetop_batching.py @@ -1,11 +1,21 @@ from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import mock_open, patch from django.http import HttpRequest, HttpResponse from django.test import SimpleTestCase -from mreg.api.treetop import _thread_local, batch_policy_parity, policy_parity +from mreg.api.treetop import ( + _initialize_policy_parity_log_file, + _thread_local, + batch_policy_parity, + policy_parity, +) from mreg.middleware.logging_http import LoggingMiddleware +from mreg.tests.prometheus_test_utils import ( + metric_by_label as _metric_by_label, + metric_total as _metric_total, + prometheus_registry_text, +) class _DummyAuthorizeResult: @@ -41,6 +51,40 @@ def _request() -> HttpRequest: request.user = SimpleNamespace(is_authenticated=True) return request + def test_initialize_policy_log_file_truncates_only_once(self) -> None: + """Main process should truncate once and then mark initialization.""" + mocked_open = mock_open() + with ( + patch("mreg.api.treetop.POLICY_TRUNCATE_LOG_FILE", True), + patch("mreg.api.treetop.POLICY_EXTRA_LOG_FILE_NAME", "policy_parity.log"), + patch( + "mreg.api.treetop.multiprocessing.current_process", + return_value=SimpleNamespace(name="MainProcess"), + ), + patch("mreg.api.treetop.open", mocked_open), + patch.dict("mreg.api.treetop.os.environ", {}, clear=True), + ): + _initialize_policy_parity_log_file() + _initialize_policy_parity_log_file() + + mocked_open.assert_called_once_with("policy_parity.log", "w") + + def test_initialize_policy_log_file_skips_parallel_worker(self) -> None: + """Parallel workers must not truncate the shared parity log file.""" + mocked_open = mock_open() + with ( + patch("mreg.api.treetop.POLICY_TRUNCATE_LOG_FILE", True), + patch( + "mreg.api.treetop.multiprocessing.current_process", + return_value=SimpleNamespace(name="ForkPoolWorker-1"), + ), + patch("mreg.api.treetop.open", mocked_open), + patch.dict("mreg.api.treetop.os.environ", {}, clear=True), + ): + _initialize_policy_parity_log_file() + + mocked_open.assert_not_called() + @patch("mreg.api.treetop.MregUser.from_request") @patch("mreg.api.treetop.log_policy_parity") def test_batch_policy_parity_uses_single_authorize_call( @@ -55,7 +99,9 @@ def test_batch_policy_parity_uses_single_authorize_call( def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-def] request_list = requests if isinstance(requests, list) else [requests] calls.append((len(request_list), correlation_id)) - return _DummyAuthorizeResponse([True] * len(request_list)) + # Keep policy results aligned with legacy decisions in this test. + decisions = [True, False][: len(request_list)] + return _DummyAuthorizeResponse(decisions) request = self._request() with ( @@ -182,3 +228,125 @@ def mock_get_response(http_request: HttpRequest) -> HttpResponse: self.assertEqual(calls, [(2, "test-correlation-id")]) self.assertEqual(mock_log_policy_parity.call_count, 2) + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop.log_policy_parity") + @patch("mreg.api.treetop.logger.warning") + def test_policy_metrics_are_recorded_for_batched_request( + self, + _mock_warning, + mock_log_policy_parity, + mock_from_request, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + + base_calls_success = _metric_by_label("mreg_policy_authorize_calls_total", 'status="success"') + base_policy_allow = _metric_by_label("mreg_policy_decisions_total", 'decision="allow"') + base_policy_deny = _metric_by_label("mreg_policy_decisions_total", 'decision="deny"') + base_legacy_allow = _metric_by_label("mreg_policy_legacy_decisions_total", 'decision="allow"') + base_legacy_deny = _metric_by_label("mreg_policy_legacy_decisions_total", 'decision="deny"') + base_parity_match = _metric_by_label("mreg_policy_parity_results_total", 'result="match"') + base_parity_mismatch = _metric_by_label("mreg_policy_parity_results_total", 'result="mismatch"') + base_parity_error = _metric_by_label("mreg_policy_parity_results_total", 'result="error"') + base_queries_count = _metric_total("mreg_policy_queries_per_request_count") + base_queries_sum = _metric_total("mreg_policy_queries_per_request_sum") + base_req_per_auth_count = _metric_total("mreg_policy_requests_per_authorize_count") + base_req_per_auth_sum = _metric_total("mreg_policy_requests_per_authorize_sum") + + def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-def] + request_list = requests if isinstance(requests, list) else [requests] + decisions = [True, False, True][: len(request_list)] + return _DummyAuthorizeResponse(decisions) + + request = self._request() + request.path_info = request.path + request._body = b"" + request.user = SimpleNamespace(username="tester") + + def mock_get_response(http_request: HttpRequest) -> HttpResponse: + policy_parity( + True, + request=http_request, + action="host_read", + resource_kind="Host", + resource_id="host1.example.org", + resource_attrs={"kind": "host", "hostname": "host1.example.org"}, + ) + policy_parity( + True, + request=http_request, + action="host_read", + resource_kind="Host", + resource_id="host2.example.org", + resource_attrs={"kind": "host", "hostname": "host2.example.org"}, + ) + policy_parity( + False, + request=http_request, + action="host_read", + resource_kind="Host", + resource_id="host3.example.org", + resource_attrs={"kind": "host", "hostname": "host3.example.org"}, + ) + return HttpResponse(status=200) + + middleware = LoggingMiddleware(mock_get_response) + + with ( + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), + patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), + ): + middleware(request) + + self.assertEqual( + _metric_by_label("mreg_policy_authorize_calls_total", 'status="success"') - base_calls_success, + 1.0, + ) + self.assertEqual( + _metric_by_label("mreg_policy_decisions_total", 'decision="allow"') - base_policy_allow, + 2.0, + ) + self.assertEqual( + _metric_by_label("mreg_policy_decisions_total", 'decision="deny"') - base_policy_deny, + 1.0, + ) + self.assertEqual( + _metric_by_label("mreg_policy_legacy_decisions_total", 'decision="allow"') - base_legacy_allow, + 2.0, + ) + self.assertEqual( + _metric_by_label("mreg_policy_legacy_decisions_total", 'decision="deny"') - base_legacy_deny, + 1.0, + ) + self.assertEqual( + _metric_by_label("mreg_policy_parity_results_total", 'result="match"') - base_parity_match, + 1.0, + ) + self.assertEqual( + _metric_by_label("mreg_policy_parity_results_total", 'result="mismatch"') - base_parity_mismatch, + 2.0, + ) + self.assertEqual( + _metric_by_label("mreg_policy_parity_results_total", 'result="error"') - base_parity_error, + 0.0, + ) + self.assertEqual(_metric_total("mreg_policy_queries_per_request_count") - base_queries_count, 1.0) + self.assertEqual(_metric_total("mreg_policy_queries_per_request_sum") - base_queries_sum, 1.0) + self.assertEqual(_metric_total("mreg_policy_requests_per_authorize_count") - base_req_per_auth_count, 1.0) + self.assertEqual(_metric_total("mreg_policy_requests_per_authorize_sum") - base_req_per_auth_sum, 3.0) + + raw = prometheus_registry_text() + # Prometheus boundaries for buckets: 0,1,2,3,5,8,+Inf (ranges: 0,1,2,3,4-5,6-8,9+) + self.assertIn('mreg_policy_queries_per_request_bucket{le="0.0"}', raw) + self.assertIn('mreg_policy_queries_per_request_bucket{le="1.0"}', raw) + self.assertIn('mreg_policy_queries_per_request_bucket{le="2.0"}', raw) + self.assertIn('mreg_policy_queries_per_request_bucket{le="3.0"}', raw) + self.assertIn('mreg_policy_queries_per_request_bucket{le="5.0"}', raw) + self.assertIn('mreg_policy_queries_per_request_bucket{le="8.0"}', raw) + self.assertIn('mreg_policy_requests_per_authorize_bucket{le="0.0"}', raw) + self.assertIn('mreg_policy_requests_per_authorize_bucket{le="1.0"}', raw) + self.assertIn('mreg_policy_requests_per_authorize_bucket{le="2.0"}', raw) + self.assertIn('mreg_policy_requests_per_authorize_bucket{le="3.0"}', raw) + self.assertIn('mreg_policy_requests_per_authorize_bucket{le="5.0"}', raw) + self.assertIn('mreg_policy_requests_per_authorize_bucket{le="8.0"}', raw) diff --git a/uv.lock b/uv.lock index f7017b63..2fefb3a6 100644 --- a/uv.lock +++ b/uv.lock @@ -505,7 +505,7 @@ requires-dist = [ { name = "gunicorn", specifier = ">=23.0.0" }, { name = "idna", specifier = ">=3.11" }, { name = "pika", specifier = ">=1.3.2" }, - { name = "prometheus-client", specifier = ">=0.20" }, + { name = "prometheus-client", specifier = ">=0.24" }, { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3" }, { name = "pyyaml" }, { name = "rich", specifier = ">=14" }, From da8ef028e7606ab8922c17b0b8aa3ccd5707f25a Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sun, 15 Feb 2026 02:16:45 +0100 Subject: [PATCH 13/34] Add tests for policy log file initialization and batch queue handling --- mreg/tests/test_treetop_batching.py | 141 ++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/mreg/tests/test_treetop_batching.py b/mreg/tests/test_treetop_batching.py index 1cc3b7a8..263ba8fc 100644 --- a/mreg/tests/test_treetop_batching.py +++ b/mreg/tests/test_treetop_batching.py @@ -5,7 +5,11 @@ from django.test import SimpleTestCase from mreg.api.treetop import ( + _batch_queue, + _fully_qualified_action, _initialize_policy_parity_log_file, + _is_parity_enabled, + _result_to_decision_and_error, _thread_local, batch_policy_parity, policy_parity, @@ -85,6 +89,53 @@ def test_initialize_policy_log_file_skips_parallel_worker(self) -> None: mocked_open.assert_not_called() + def test_initialize_policy_log_file_skips_when_truncate_disabled(self) -> None: + """Do not touch parity log file when truncation is disabled.""" + mocked_open = mock_open() + with ( + patch("mreg.api.treetop.POLICY_TRUNCATE_LOG_FILE", False), + patch("mreg.api.treetop.open", mocked_open), + patch.dict("mreg.api.treetop.os.environ", {}, clear=True), + ): + _initialize_policy_parity_log_file() + + mocked_open.assert_not_called() + + def test_batch_queue_initializes_when_missing(self) -> None: + """_batch_queue should create and store an empty queue on first access.""" + if hasattr(_thread_local, "batch_queue"): + delattr(_thread_local, "batch_queue") + + queue = _batch_queue() + + self.assertEqual(queue, []) + self.assertIs(queue, _thread_local.batch_queue) + + def test_is_parity_enabled_false_when_globally_disabled(self) -> None: + with patch("mreg.api.treetop.POLICY_PARITY_ENABLED", False): + self.assertFalse(_is_parity_enabled()) + + def test_fully_qualified_action_without_namespace(self) -> None: + action = SimpleNamespace(id=SimpleNamespace(namespace=[], id="host_read")) + self.assertEqual(_fully_qualified_action(action), "host_read") + + def test_result_to_decision_and_error_missing_index(self) -> None: + allowed, error = _result_to_decision_and_error([], 0) + self.assertIsNone(allowed) + self.assertEqual(error, "Missing policy result at index 0") + + def test_result_to_decision_and_error_failure_status(self) -> None: + failed_result = SimpleNamespace( + is_success=lambda: False, + status="denied", + error=None, + ) + + allowed, error = _result_to_decision_and_error([failed_result], 0) + + self.assertIsNone(allowed) + self.assertEqual(error, "Authorization failed with status=denied") + @patch("mreg.api.treetop.MregUser.from_request") @patch("mreg.api.treetop.log_policy_parity") def test_batch_policy_parity_uses_single_authorize_call( @@ -229,6 +280,96 @@ def mock_get_response(http_request: HttpRequest) -> HttpResponse: self.assertEqual(calls, [(2, "test-correlation-id")]) self.assertEqual(mock_log_policy_parity.call_count, 2) + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop.log_policy_parity") + @patch("mreg.api.treetop.logger.warning") + @patch("mreg.api.treetop.logger.error") + def test_flush_policy_parity_batch_handles_authorize_exception( + self, + mock_error, + _mock_warning, + mock_log_policy_parity, + mock_from_request, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + + request = self._request() + with ( + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), + patch( + "mreg.api.treetop.treetopclient.authorize", + side_effect=RuntimeError("policy service unavailable"), + ), + batch_policy_parity(), + ): + policy_parity( + True, + request=request, + action="host_read", + resource_kind="Host", + resource_id="host1.example.org", + resource_attrs={"kind": "host", "hostname": "host1.example.org"}, + ) + policy_parity( + False, + request=request, + action="host_read", + resource_kind="Host", + resource_id="host2.example.org", + resource_attrs={"kind": "host", "hostname": "host2.example.org"}, + ) + + self.assertEqual(mock_log_policy_parity.call_count, 2) + self.assertEqual(mock_error.call_count, 1) + self.assertEqual(mock_error.call_args.kwargs["extra"]["batch_size"], 2) + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop.log_policy_parity") + @patch("mreg.api.treetop.logger.warning") + @patch("mreg.api.treetop.logger.error") + def test_policy_parity_non_batch_authorize_exception_records_error_metrics( + self, + mock_error, + _mock_warning, + mock_log_policy_parity, + mock_from_request, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + + base_policy_error = _metric_by_label("mreg_policy_decisions_total", 'decision="error"') + base_parity_error = _metric_by_label("mreg_policy_parity_results_total", 'result="error"') + + request = self._request() + with ( + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", False), + patch( + "mreg.api.treetop.treetopclient.authorize", + side_effect=RuntimeError("policy service unavailable"), + ), + ): + decision = policy_parity( + True, + request=request, + action="host_read", + resource_kind="Host", + resource_id="host1.example.org", + resource_attrs={"kind": "host", "hostname": "host1.example.org"}, + ) + + self.assertTrue(decision) + self.assertEqual( + _metric_by_label("mreg_policy_decisions_total", 'decision="error"') - base_policy_error, + 1.0, + ) + self.assertEqual( + _metric_by_label("mreg_policy_parity_results_total", 'result="error"') - base_parity_error, + 1.0, + ) + self.assertEqual(mock_error.call_count, 1) + self.assertEqual(mock_log_policy_parity.call_count, 1) + @patch("mreg.api.treetop.MregUser.from_request") @patch("mreg.api.treetop.log_policy_parity") @patch("mreg.api.treetop.logger.warning") From 48b465663771cb2d76f89f675a52abda33615ead Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sun, 15 Feb 2026 02:36:31 +0100 Subject: [PATCH 14/34] Update pyproject.toml and uv.lock against master. --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7305a346..d84e7bed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "pyyaml", # For testing inside Docker image "unittest-parametrize", - "treetop-client>=0.0.1", + "treetop-client>=0.0.7", "prometheus-client>=0.24", ] dynamic = ["version"] diff --git a/uv.lock b/uv.lock index 2fefb3a6..ebbbb097 100644 --- a/uv.lock +++ b/uv.lock @@ -511,7 +511,7 @@ requires-dist = [ { name = "rich", specifier = ">=14" }, { name = "sentry-sdk", specifier = ">=2.48.0" }, { name = "structlog", specifier = ">=25" }, - { name = "treetop-client", specifier = ">=0.0.1" }, + { name = "treetop-client", specifier = ">=0.0.7" }, { name = "tzdata", specifier = ">=2025.3" }, { name = "unittest-parametrize" }, { name = "uritemplate" }, From 5fdec4cd1401568757544879b7c9f93ddfca552e Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sun, 15 Feb 2026 13:50:59 +0100 Subject: [PATCH 15/34] Refacgtor policy parity logging and batching functionality - Add detailed docstrings for clarity on function purposes and side effects. - Deduplicate policy-engine authorize + metrics/error handling --- mreg/api/treetop.py | 270 ++++++++++++++++++++++++++++++++------------ 1 file changed, 198 insertions(+), 72 deletions(-) diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index e7775544..73036112 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -36,10 +36,18 @@ def _initialize_policy_parity_log_file() -> None: - """Truncate the parity log once in the main process. - - Parallel test workers import this module too. Guarding on the main process - and an environment marker prevents workers from re-truncating the file. + """Initialize the parity log file exactly once per process tree. + + The module is imported by Django workers and by parallel test processes. + Truncating on every import would erase events from other workers, so this + helper truncates only when all of the following are true: + 1. file truncation is enabled by configuration + 2. the current process is the main process + 3. an environment marker is not already set + + Side effects: + Opens and truncates ``POLICY_EXTRA_LOG_FILE_NAME`` in write mode. + Sets ``MREG_POLICY_PARITY_LOG_INITIALIZED=1`` in ``os.environ``. """ if not POLICY_TRUNCATE_LOG_FILE: return @@ -110,10 +118,12 @@ class _ParityBatchItem: def _batch_depth() -> int: + """Return current nesting depth for ``batch_policy_parity`` in this thread.""" return int(getattr(_thread_local, "batch_depth", 0)) def _batch_queue() -> list[_ParityBatchItem]: + """Return the thread-local parity queue, creating it on first access.""" queue = getattr(_thread_local, "batch_queue", None) if queue is None: queue = [] @@ -122,20 +132,32 @@ def _batch_queue() -> list[_ParityBatchItem]: def _is_batching() -> bool: + """Return ``True`` when parity checks should be enqueued instead of sent.""" return _batch_depth() > 0 and POLICY_PARITY_BATCH_ENABLED def _request_authorize_calls() -> int: + """Return number of authorize calls made for the current HTTP request context.""" return int(getattr(_thread_local, "policy_authorize_calls", 0)) def _inc_request_authorize_calls() -> None: + """Increment the per-request authorize call counter in thread-local storage.""" _thread_local.policy_authorize_calls = _request_authorize_calls() + 1 @contextmanager def batch_policy_parity(): - """Batch parity checks in the current thread and flush on scope exit.""" + """Batch parity checks for one request scope and flush on exit. + + This context manager supports nesting. The outermost scope initializes + request-local counters/queues, and the final exit flushes queued parity + checks via ``flush_policy_parity_batch()`` (when batching is enabled). + + Side effects: + Mutates thread-local batch state and updates + ``mreg_policy_queries_per_request`` histogram on outermost exit. + """ current_depth = _batch_depth() if current_depth == 0: _thread_local.policy_authorize_calls = 0 @@ -160,17 +182,12 @@ def batch_policy_parity(): @contextmanager def disable_policy_parity(): - """Context manager to temporarily disable policy parity checking. - - Useful for tests that modify permissions/state mid-test, which would - cause the legacy and policy systems to be out of sync. - - Example: - def test_permission_changes(self): - with disable_policy_parity(): - # Modify permissions here - user.groups.add(some_group) - # Make API calls - parity checking will be skipped + """Temporarily disable parity checks in the current thread context. + + This is primarily intended for tests that intentionally mutate permission + state mid-test, where legacy and policy decisions are expected to diverge. + The previous value is restored when leaving the context, so nested usage is + safe. """ old_value = getattr(_thread_local, "skip_parity", False) _thread_local.skip_parity = True @@ -180,19 +197,32 @@ def test_permission_changes(self): _thread_local.skip_parity = old_value def _is_parity_enabled() -> bool: - """Check if parity checking should be performed in current context.""" + """Return whether parity checks should run in the current thread context. + + Parity is enabled only when both conditions are true: + 1. global parity is enabled in settings + 2. parity is not temporarily disabled via ``disable_policy_parity()`` + """ if not POLICY_PARITY_ENABLED: return False # Skip parity checking if we're in a disabled context return not getattr(_thread_local, "skip_parity", False) def _corr_id(request: Request) -> Optional[str]: - """Return request correlation ID from standard header variants.""" + """Return request correlation ID from accepted inbound header names. + + Resolution order: + 1. ``X-Correlation-ID`` in ``request.headers`` + 2. ``HTTP_X_CORRELATION_ID`` in ``request.META`` + """ return request.headers.get("X-Correlation-ID") or request.META.get("HTTP_X_CORRELATION_ID") def _model_name_from_view(view) -> str: # type: ignore - """Best-effort view model name, preferring serializer Meta.model.""" - # Best effort: try serializer model, else view class name + """Best-effort model name for logging context. + + Attempts to resolve ``view.get_serializer_class().Meta.model.__name__``. + If serializer/model introspection fails, falls back to the view class name. + """ try: sc = view.get_serializer_class() return sc.Meta.model.__name__ @@ -201,6 +231,18 @@ def _model_name_from_view(view) -> str: # type: ignore def _build_resource_attrs(resource_attrs: Mapping[str, str]) -> dict[str, ResourceAttribute]: + """Convert plain resource attributes to typed TreeTop attributes. + + Each attribute value is parsed as an IP address when possible and tagged as + ``ResourceAttributeType.IP``. Non-IP values are stored as + ``ResourceAttributeType.STRING``. + + Args: + resource_attrs: Plain key/value attributes from parity call sites. + + Returns: + Mapping compatible with ``Resource.new(..., attrs=...)``. + """ attrs: dict[str, ResourceAttribute] = {} for key, value in resource_attrs.items(): try: @@ -212,6 +254,7 @@ def _build_resource_attrs(resource_attrs: Mapping[str, str]) -> dict[str, Resour def _fully_qualified_action(action: Action) -> str: + """Return canonical action name as ``namespace::...::id`` for logs.""" if len(action.id.namespace) > 0: return "::".join(action.id.namespace) + f"::{action.id.id}" return f"{action.id.id}" @@ -224,6 +267,17 @@ def _compute_parity_payload( error: Optional[str], context: dict[str, Any], ) -> dict[str, Any]: + """Build normalized payload used for parity logging and metrics accounting. + + Args: + decision: Legacy authorization decision. + pol_allowed: Decision returned by policy engine, or ``None`` on error. + error: Error text when policy evaluation failed. + context: Request/action metadata to include in parity logs. + + Returns: + Serializable payload containing decisions, parity result, and context. + """ parity = False if bool(decision) and pol_allowed: parity = True @@ -240,6 +294,13 @@ def _compute_parity_payload( def _log_parity_payload(payload: dict[str, Any]) -> None: + """Emit parity metrics and structured logs for one parity payload. + + Side effects: + Increments legacy/policy/parity Prometheus counters, logs either + ``policy_parity_ok`` or ``policy_parity_mismatch``, and appends the + payload to the parity log file. + """ legacy_decision = payload.get("legacy_decision") if legacy_decision is True: POLICY_LEGACY_DECISIONS_TOTAL.labels(decision="allow").inc() @@ -272,6 +333,18 @@ def _result_to_decision_and_error( results: Sequence[Any], index: int, ) -> tuple[Optional[bool], Optional[str]]: + """Extract one policy decision/error pair from batched authorize results. + + Args: + results: Sequence of authorize result objects. + index: Index of the result corresponding to one queued parity check. + + Returns: + Tuple ``(allowed, error)`` where: + - ``allowed`` is ``True``/``False`` on successful evaluation + - ``allowed`` is ``None`` when result is missing or unsuccessful + - ``error`` contains descriptive text when unavailable/unsuccessful + """ if index >= len(results): return None, f"Missing policy result at index {index}" @@ -284,45 +357,93 @@ def _result_to_decision_and_error( return None, str(error) -def flush_policy_parity_batch() -> None: - """Flush queued parity checks as one authorize batch call.""" - queue = _batch_queue() - if not queue: - return +def _authorize_with_metrics( + *, + policy_requests: Sequence[TreeTopRequest], + correlation_id: Optional[str], + path: Optional[str], +) -> tuple[list[Any], Optional[str]]: + """Call TreeTop authorize once and record shared metrics and errors. + + Args: + policy_requests: One or more policy requests to evaluate. + correlation_id: Correlation ID propagated to TreeTop and logs. + path: Request path used in error logging context. + + Returns: + Tuple ``(results, error)`` where: + - ``results`` is the response ``results`` list on success + - ``error`` is ``None`` on success, otherwise ``repr(exception)`` + + Side effects: + Increments authorize call counters, observes latency/request-size + histograms, and emits structured error logs on failures. + """ + request_count = len(policy_requests) + if request_count == 0: + return [], None - correlation_id = queue[0].context.get("correlation_id") - POLICY_REQUESTS_PER_AUTHORIZE.observe(float(len(queue))) - _inc_request_authorize_calls() - pol_allowed_by_index: list[Optional[bool]] = [None] * len(queue) - error_by_index: list[Optional[str]] = [None] * len(queue) + request_payload: TreeTopRequest | list[TreeTopRequest] + if request_count == 1: + request_payload = policy_requests[0] + else: + request_payload = list(policy_requests) + POLICY_REQUESTS_PER_AUTHORIZE.observe(float(request_count)) + _inc_request_authorize_calls() call_started = monotonic() try: response = treetopclient.authorize( - [item.policy_request for item in queue], + request_payload, correlation_id=correlation_id, ) POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="success").inc() POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="success").observe(monotonic() - call_started) - results = list(getattr(response, "results", [])) - for idx in range(len(queue)): - pol_allowed_by_index[idx], error_by_index[idx] = _result_to_decision_and_error(results, idx) + return list(getattr(response, "results", [])), None except Exception as exc: POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="exception").inc() POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="exception").observe(monotonic() - call_started) - error = repr(exc) + extra = { + "error_type": type(exc).__name__, + "error_msg": str(exc), + "path": path, + "correlation_id": correlation_id, + } + if request_count > 1: + extra["batch_size"] = request_count logger.error( f"Policy server error: {type(exc).__name__}: {exc}", - extra={ - "error_type": type(exc).__name__, - "error_msg": str(exc), - "path": queue[0].context.get("path"), - "correlation_id": correlation_id, - "batch_size": len(queue), - }, + extra=extra, ) + return [], repr(exc) + + +def flush_policy_parity_batch() -> None: + """Flush queued parity checks through one batched authorize request. + + For each queued item, this function computes the final parity payload and + emits metrics/logging via ``_log_parity_payload``. If the authorize call + fails, every queued item is logged as an error outcome using the same error + string. + """ + queue = _batch_queue() + if not queue: + return + + correlation_id = queue[0].context.get("correlation_id") + pol_allowed_by_index: list[Optional[bool]] = [None] * len(queue) + error_by_index: list[Optional[str]] = [None] * len(queue) + results, authorize_error = _authorize_with_metrics( + policy_requests=[item.policy_request for item in queue], + correlation_id=correlation_id, + path=queue[0].context.get("path"), + ) + if authorize_error is not None: + for idx in range(len(queue)): + error_by_index[idx] = authorize_error + else: for idx in range(len(queue)): - error_by_index[idx] = error + pol_allowed_by_index[idx], error_by_index[idx] = _result_to_decision_and_error(results, idx) for idx, item in enumerate(queue): payload = _compute_parity_payload( @@ -345,8 +466,23 @@ def policy_parity( resource_attrs: Mapping[str, str], ) -> bool: """ - Log legacy-vs-policy parity and return `decision` unchanged. - Use this anywhere you currently 'return True/False'. + Compare legacy authorization with TreeTop policy and log parity metadata. + + This helper never changes request behavior: it always returns ``decision`` + unchanged. Its purpose is observability during parity rollout. + + Args: + decision: Legacy authorization decision to preserve. + request: Active DRF request. + view: Optional view instance for context logging. + permission_class: Optional explicit permission class name. + action: Policy action ID to evaluate. + resource_kind: Resource type identifier. + resource_id: Resource ID for policy evaluation. + resource_attrs: Additional resource attributes as strings. + + Returns: + The original ``decision`` argument. """ if not _is_parity_enabled(): return decision @@ -384,33 +520,18 @@ def policy_parity( ) return decision + results, authorize_error = _authorize_with_metrics( + policy_requests=[pol_request], + correlation_id=context["correlation_id"], + path=request.path, + ) pol_allowed, error = None, None - _inc_request_authorize_calls() - call_started = monotonic() - try: - resp = treetopclient.authorize(pol_request, correlation_id=context["correlation_id"]) - POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="success").inc() - POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="success").observe(monotonic() - call_started) - POLICY_REQUESTS_PER_AUTHORIZE.observe(1.0) - results = list(getattr(resp, "results", [])) + if authorize_error is None: pol_allowed, error = _result_to_decision_and_error(results, 0) - except Exception as exc: - POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="exception").inc() - POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="exception").observe(monotonic() - call_started) - POLICY_REQUESTS_PER_AUTHORIZE.observe(1.0) - error = repr(exc) - # Log policy server errors prominently - logger.error( - f"Policy server error: {type(exc).__name__}: {exc}", - extra={ - "error_type": type(exc).__name__, - "error_msg": str(exc), - "path": request.path, - "correlation_id": _corr_id(request), - }, - ) - # If policy server fails, we cannot determine parity. Return legacy decision - # but flag this in the payload for monitoring. + else: + error = authorize_error + # If policy server fails, we cannot determine parity. Return legacy decision + # but flag this in the payload for monitoring. payload = _compute_parity_payload( decision=decision, @@ -423,7 +544,12 @@ def policy_parity( return decision # Log data to a file in addition to normal logging -def log_policy_parity(payload: dict[str, Any]): - """Append one JSON parity event to the configured parity log file.""" +def log_policy_parity(payload: dict[str, Any]) -> None: + """Append one JSON parity event to the configured parity log file. + + Args: + payload: Serializable parity payload produced by + ``_compute_parity_payload``. + """ with open(POLICY_EXTRA_LOG_FILE_NAME, "a") as log_file: log_file.write(f"{json.dumps(payload)}\n") From b9b847d8e2eb94537ab8be0c5a1aa1f3f6e174dd Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Sun, 15 Feb 2026 14:49:20 +0100 Subject: [PATCH 16/34] Add utility methods for request normalization and middleware compatibility --- mreg/tests/test_treetop_batching.py | 192 +++++++++++----------------- 1 file changed, 72 insertions(+), 120 deletions(-) diff --git a/mreg/tests/test_treetop_batching.py b/mreg/tests/test_treetop_batching.py index 263ba8fc..ac2c7ded 100644 --- a/mreg/tests/test_treetop_batching.py +++ b/mreg/tests/test_treetop_batching.py @@ -48,6 +48,7 @@ def tearDown(self) -> None: @staticmethod def _request() -> HttpRequest: + """Build a baseline request object for parity test invocations.""" request = HttpRequest() request.method = "GET" request.path = "/api/v1/hosts/" @@ -55,6 +56,49 @@ def _request() -> HttpRequest: request.user = SimpleNamespace(is_authenticated=True) return request + @staticmethod + def _middleware_request() -> HttpRequest: + """Build a request object compatible with LoggingMiddleware tests.""" + request = TreeTopParityBatchingTests._request() + request.path_info = request.path + request._body = b"" + request.user = SimpleNamespace(username="tester") + return request + + @staticmethod + def _normalize_authorize_requests(requests): # type: ignore[no-untyped-def] + """Normalize authorize input to a list for call-count assertions.""" + return requests if isinstance(requests, list) else [requests] + + @staticmethod + def _host_resource_attrs(hostname: str) -> dict[str, str]: + """Return standard host resource attributes used by parity tests.""" + return {"kind": "host", "hostname": hostname} + + def _run_parity_check(self, request: HttpRequest, *, decision: bool, hostname: str) -> bool: + """Run one host_read parity check with canonical host test payload.""" + return policy_parity( + decision, + request=request, + action="host_read", + resource_kind="Host", + resource_id=hostname, + resource_attrs=self._host_resource_attrs(hostname), + ) + + def _middleware_response_with_checks(self, checks: list[tuple[bool, str]]): + """Create middleware callback that emits parity checks then returns 200.""" + def mock_get_response(http_request: HttpRequest) -> HttpResponse: + for decision, hostname in checks: + self._run_parity_check( + http_request, + decision=decision, + hostname=hostname, + ) + return HttpResponse(status=200) + + return mock_get_response + def test_initialize_policy_log_file_truncates_only_once(self) -> None: """Main process should truncate once and then mark initialization.""" mocked_open = mock_open() @@ -148,7 +192,7 @@ def test_batch_policy_parity_uses_single_authorize_call( calls: list[tuple[int, str | None]] = [] def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-def] - request_list = requests if isinstance(requests, list) else [requests] + request_list = self._normalize_authorize_requests(requests) calls.append((len(request_list), correlation_id)) # Keep policy results aligned with legacy decisions in this test. decisions = [True, False][: len(request_list)] @@ -161,26 +205,8 @@ def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-de patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), batch_policy_parity(), ): - self.assertTrue( - policy_parity( - True, - request=request, - action="host_read", - resource_kind="Host", - resource_id="host1.example.org", - resource_attrs={"kind": "host", "hostname": "host1.example.org"}, - ) - ) - self.assertFalse( - policy_parity( - False, - request=request, - action="host_read", - resource_kind="Host", - resource_id="host2.example.org", - resource_attrs={"kind": "host", "hostname": "host2.example.org"}, - ) - ) + self.assertTrue(self._run_parity_check(request, decision=True, hostname="host1.example.org")) + self.assertFalse(self._run_parity_check(request, decision=False, hostname="host2.example.org")) self.assertEqual(calls, [(2, "test-correlation-id")]) self.assertEqual(mock_log_policy_parity.call_count, 2) @@ -197,7 +223,7 @@ def test_policy_parity_without_batch_context_calls_authorize_per_check( calls: list[int] = [] def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-def] - request_list = requests if isinstance(requests, list) else [requests] + request_list = self._normalize_authorize_requests(requests) calls.append(len(request_list)) return _DummyAuthorizeResponse([True] * len(request_list)) @@ -207,22 +233,8 @@ def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-de patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), ): - policy_parity( - True, - request=request, - action="host_read", - resource_kind="Host", - resource_id="host1.example.org", - resource_attrs={"kind": "host", "hostname": "host1.example.org"}, - ) - policy_parity( - True, - request=request, - action="host_read", - resource_kind="Host", - resource_id="host2.example.org", - resource_attrs={"kind": "host", "hostname": "host2.example.org"}, - ) + self._run_parity_check(request, decision=True, hostname="host1.example.org") + self._run_parity_check(request, decision=True, hostname="host2.example.org") self.assertEqual(calls, [1, 1]) self.assertEqual(mock_log_policy_parity.call_count, 2) @@ -240,35 +252,19 @@ def test_single_http_request_flushes_one_authorize_batch( calls: list[tuple[int, str | None]] = [] def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-def] - request_list = requests if isinstance(requests, list) else [requests] + request_list = self._normalize_authorize_requests(requests) calls.append((len(request_list), correlation_id)) return _DummyAuthorizeResponse([True] * len(request_list)) - request = self._request() - request.path_info = request.path - request._body = b"" - request.user = SimpleNamespace(username="tester") - - def mock_get_response(http_request: HttpRequest) -> HttpResponse: - policy_parity( - True, - request=http_request, - action="host_read", - resource_kind="Host", - resource_id="host1.example.org", - resource_attrs={"kind": "host", "hostname": "host1.example.org"}, - ) - policy_parity( - True, - request=http_request, - action="host_read", - resource_kind="Host", - resource_id="host2.example.org", - resource_attrs={"kind": "host", "hostname": "host2.example.org"}, + request = self._middleware_request() + middleware = LoggingMiddleware( + self._middleware_response_with_checks( + [ + (True, "host1.example.org"), + (True, "host2.example.org"), + ] ) - return HttpResponse(status=200) - - middleware = LoggingMiddleware(mock_get_response) + ) with ( patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), @@ -303,22 +299,8 @@ def test_flush_policy_parity_batch_handles_authorize_exception( ), batch_policy_parity(), ): - policy_parity( - True, - request=request, - action="host_read", - resource_kind="Host", - resource_id="host1.example.org", - resource_attrs={"kind": "host", "hostname": "host1.example.org"}, - ) - policy_parity( - False, - request=request, - action="host_read", - resource_kind="Host", - resource_id="host2.example.org", - resource_attrs={"kind": "host", "hostname": "host2.example.org"}, - ) + self._run_parity_check(request, decision=True, hostname="host1.example.org") + self._run_parity_check(request, decision=False, hostname="host2.example.org") self.assertEqual(mock_log_policy_parity.call_count, 2) self.assertEqual(mock_error.call_count, 1) @@ -349,14 +331,7 @@ def test_policy_parity_non_batch_authorize_exception_records_error_metrics( side_effect=RuntimeError("policy service unavailable"), ), ): - decision = policy_parity( - True, - request=request, - action="host_read", - resource_kind="Host", - resource_id="host1.example.org", - resource_attrs={"kind": "host", "hostname": "host1.example.org"}, - ) + decision = self._run_parity_check(request, decision=True, hostname="host1.example.org") self.assertTrue(decision) self.assertEqual( @@ -395,43 +370,20 @@ def test_policy_metrics_are_recorded_for_batched_request( base_req_per_auth_sum = _metric_total("mreg_policy_requests_per_authorize_sum") def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-def] - request_list = requests if isinstance(requests, list) else [requests] + request_list = self._normalize_authorize_requests(requests) decisions = [True, False, True][: len(request_list)] return _DummyAuthorizeResponse(decisions) - request = self._request() - request.path_info = request.path - request._body = b"" - request.user = SimpleNamespace(username="tester") - - def mock_get_response(http_request: HttpRequest) -> HttpResponse: - policy_parity( - True, - request=http_request, - action="host_read", - resource_kind="Host", - resource_id="host1.example.org", - resource_attrs={"kind": "host", "hostname": "host1.example.org"}, + request = self._middleware_request() + middleware = LoggingMiddleware( + self._middleware_response_with_checks( + [ + (True, "host1.example.org"), + (True, "host2.example.org"), + (False, "host3.example.org"), + ] ) - policy_parity( - True, - request=http_request, - action="host_read", - resource_kind="Host", - resource_id="host2.example.org", - resource_attrs={"kind": "host", "hostname": "host2.example.org"}, - ) - policy_parity( - False, - request=http_request, - action="host_read", - resource_kind="Host", - resource_id="host3.example.org", - resource_attrs={"kind": "host", "hostname": "host3.example.org"}, - ) - return HttpResponse(status=200) - - middleware = LoggingMiddleware(mock_get_response) + ) with ( patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), From 31821914a7067782aef0bd1f120995a7079f2351 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Mon, 16 Feb 2026 07:40:33 +0100 Subject: [PATCH 17/34] Complete env support for treetop. --- docs/env.md | 34 +++++++++++++++++++++++++++++ mreg/api/treetop.py | 11 +++++++--- mreg/tests/test_treetop_batching.py | 16 ++++++++++++++ mregsite/settings.py | 8 +++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/docs/env.md b/docs/env.md index bdc145e8..ba8915c1 100644 --- a/docs/env.md +++ b/docs/env.md @@ -28,6 +28,40 @@ Must be one of the following: - `ERROR` - `CRITICAL` +## `MREG_POLICY_PARITY_ENABLED` + +Boolean flag controlling whether policy parity checks run. Default: `True` + +Parity checks run only when both `MREG_POLICY_PARITY_ENABLED` is true and +`MREG_POLICY_BASE_URL` is set to a non-empty value. + +## `MREG_POLICY_BASE_URL` + +Base URL for the TreeTop policy engine REST service. Default: empty (disabled) + +If unset or empty, the policy parity code is disabled and no requests are made +to the policy engine. + +Example: `http://localhost:9999` + +## `MREG_POLICY_NAMESPACE` + +Namespace used when constructing policy principal/action IDs. Default: `MREG` + +Use Cedar-style `::` separators (commas are also accepted). + +Example: `MREG` or `org::MREG` + +## `MREG_POLICY_EXTRA_LOG_FILE_NAME` + +File path for the parity JSONL log file (one JSON object per line). Default: +`policy_parity.log` + +## `MREG_POLICY_TRUNCATE_LOG_FILE` + +Boolean flag controlling whether `MREG_POLICY_EXTRA_LOG_FILE_NAME` is +truncated once at startup (main process only). Default: `True` + ## `MREG_POLICY_PARITY_BATCH_ENABLED` Boolean flag controlling request-scoped batching of parity authorize checks. diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index 73036112..362a1756 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -27,12 +27,13 @@ # Configure these in settings.py POLICY_PARITY_ENABLED = getattr(settings, "POLICY_PARITY_ENABLED", True) -POLICY_BASE_URL = getattr(settings, "POLICY_BASE_URL", "http://localhost:9999") +POLICY_BASE_URL = (getattr(settings, "POLICY_BASE_URL", "") or "").strip() POLICY_NAMESPACE = getattr(settings, "POLICY_NAMESPACE", ["MREG"]) POLICY_EXTRA_LOG_FILE_NAME = getattr(settings, "POLICY_EXTRA_LOG_FILE_NAME", "policy_parity.log") POLICY_TRUNCATE_LOG_FILE = getattr(settings, "POLICY_TRUNCATE_LOG_FILE", True) POLICY_PARITY_BATCH_ENABLED = getattr(settings, "POLICY_PARITY_BATCH_ENABLED", True) _POLICY_PARITY_LOG_INITIALIZED_ENV = "MREG_POLICY_PARITY_LOG_INITIALIZED" +_DEFAULT_POLICY_BASE_URL = "http://localhost:9999" def _initialize_policy_parity_log_file() -> None: @@ -49,6 +50,8 @@ def _initialize_policy_parity_log_file() -> None: Opens and truncates ``POLICY_EXTRA_LOG_FILE_NAME`` in write mode. Sets ``MREG_POLICY_PARITY_LOG_INITIALIZED=1`` in ``os.environ``. """ + if not POLICY_PARITY_ENABLED or not POLICY_BASE_URL: + return if not POLICY_TRUNCATE_LOG_FILE: return if multiprocessing.current_process().name != "MainProcess": @@ -62,7 +65,7 @@ def _initialize_policy_parity_log_file() -> None: _initialize_policy_parity_log_file() -treetopclient = TreeTopClient(base_url=POLICY_BASE_URL) +treetopclient = TreeTopClient(base_url=POLICY_BASE_URL or _DEFAULT_POLICY_BASE_URL) POLICY_DECISIONS_TOTAL = Counter( "mreg_policy_decisions_total", @@ -200,11 +203,13 @@ def _is_parity_enabled() -> bool: """Return whether parity checks should run in the current thread context. Parity is enabled only when both conditions are true: - 1. global parity is enabled in settings + 1. global parity is enabled in settings, and a policy base URL is configured 2. parity is not temporarily disabled via ``disable_policy_parity()`` """ if not POLICY_PARITY_ENABLED: return False + if not POLICY_BASE_URL: + return False # Skip parity checking if we're in a disabled context return not getattr(_thread_local, "skip_parity", False) diff --git a/mreg/tests/test_treetop_batching.py b/mreg/tests/test_treetop_batching.py index ac2c7ded..dc1f4a81 100644 --- a/mreg/tests/test_treetop_batching.py +++ b/mreg/tests/test_treetop_batching.py @@ -104,6 +104,7 @@ def test_initialize_policy_log_file_truncates_only_once(self) -> None: mocked_open = mock_open() with ( patch("mreg.api.treetop.POLICY_TRUNCATE_LOG_FILE", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://localhost:9999"), patch("mreg.api.treetop.POLICY_EXTRA_LOG_FILE_NAME", "policy_parity.log"), patch( "mreg.api.treetop.multiprocessing.current_process", @@ -122,6 +123,7 @@ def test_initialize_policy_log_file_skips_parallel_worker(self) -> None: mocked_open = mock_open() with ( patch("mreg.api.treetop.POLICY_TRUNCATE_LOG_FILE", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://localhost:9999"), patch( "mreg.api.treetop.multiprocessing.current_process", return_value=SimpleNamespace(name="ForkPoolWorker-1"), @@ -138,6 +140,7 @@ def test_initialize_policy_log_file_skips_when_truncate_disabled(self) -> None: mocked_open = mock_open() with ( patch("mreg.api.treetop.POLICY_TRUNCATE_LOG_FILE", False), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://localhost:9999"), patch("mreg.api.treetop.open", mocked_open), patch.dict("mreg.api.treetop.os.environ", {}, clear=True), ): @@ -159,6 +162,13 @@ def test_is_parity_enabled_false_when_globally_disabled(self) -> None: with patch("mreg.api.treetop.POLICY_PARITY_ENABLED", False): self.assertFalse(_is_parity_enabled()) + def test_is_parity_enabled_false_when_base_url_unset(self) -> None: + with ( + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", ""), + ): + self.assertFalse(_is_parity_enabled()) + def test_fully_qualified_action_without_namespace(self) -> None: action = SimpleNamespace(id=SimpleNamespace(namespace=[], id="host_read")) self.assertEqual(_fully_qualified_action(action), "host_read") @@ -201,6 +211,7 @@ def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-de request = self._request() with ( patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://localhost:9999"), patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), batch_policy_parity(), @@ -230,6 +241,7 @@ def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-de request = self._request() with ( patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://localhost:9999"), patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), ): @@ -268,6 +280,7 @@ def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-de with ( patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://localhost:9999"), patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), ): @@ -292,6 +305,7 @@ def test_flush_policy_parity_batch_handles_authorize_exception( request = self._request() with ( patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://localhost:9999"), patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), patch( "mreg.api.treetop.treetopclient.authorize", @@ -325,6 +339,7 @@ def test_policy_parity_non_batch_authorize_exception_records_error_metrics( request = self._request() with ( patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://localhost:9999"), patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", False), patch( "mreg.api.treetop.treetopclient.authorize", @@ -387,6 +402,7 @@ def fake_authorize(requests, correlation_id=None): # type: ignore[no-untyped-de with ( patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://localhost:9999"), patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), patch("mreg.api.treetop.treetopclient.authorize", side_effect=fake_authorize), ): diff --git a/mregsite/settings.py b/mregsite/settings.py index 691846cf..1350abc9 100644 --- a/mregsite/settings.py +++ b/mregsite/settings.py @@ -80,6 +80,14 @@ def parse_protected_attrs(raw: str) -> list[dict]: LOG_LEVEL = envvar("MREG_LOG_LEVEL", "CRITICAL").upper() POLICY_PARITY_LOG_LEVEL = envvar("MREG_POLICY_PARITY_LOG_LEVEL", "WARNING").upper() +POLICY_PARITY_ENABLED = envvar("MREG_POLICY_PARITY_ENABLED", True) +POLICY_BASE_URL = envvar("MREG_POLICY_BASE_URL", "").strip() +raw = (envvar("MREG_POLICY_NAMESPACE", "MREG") or "").strip() +# Accept both Cedar-style `org::MREG` and comma-separated `org,MREG`. +raw = raw.replace("::", ",") +POLICY_NAMESPACE = [ns.strip() for ns in raw.split(",") if ns.strip()] or ["MREG"] +POLICY_EXTRA_LOG_FILE_NAME = envvar("MREG_POLICY_EXTRA_LOG_FILE_NAME", "policy_parity.log") +POLICY_TRUNCATE_LOG_FILE = envvar("MREG_POLICY_TRUNCATE_LOG_FILE", True) POLICY_PARITY_BATCH_ENABLED = envvar("MREG_POLICY_PARITY_BATCH_ENABLED", True) REQUESTS_THRESHOLD_SLOW = envvar("MREG_REQUESTS_THRESHOLD_SLOW", 1000) From 151688faa8b3e69ca05e2077e16b131ebe0643e4 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Mon, 16 Feb 2026 07:41:55 +0100 Subject: [PATCH 18/34] Documentation update. --- docs/parity_testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/parity_testing.md b/docs/parity_testing.md index 6113ecba..d866cdb0 100644 --- a/docs/parity_testing.md +++ b/docs/parity_testing.md @@ -6,7 +6,7 @@ Related documentation: ## Problem -Tests that modify permissions or group memberships mid-test cause the legacy permission system and the TreeTop policy engine to be out of sync. Since TreeTop's policy content is immutable, these tests cannot maintain parity between the two systems. +Tests that modify permissions or group memberships mid-test cause the legacy permission system and the TreeTop policy engine to be out of sync. Since TreeTop's policy content is immutable (in this context), these tests cannot maintain parity between the two systems. ## Solutions From 7178ce2e799cc53c61723dadf89c4089bfd6a646 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Mon, 16 Feb 2026 07:44:11 +0100 Subject: [PATCH 19/34] More documentation fixes. --- docs/parity_testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/parity_testing.md b/docs/parity_testing.md index d866cdb0..df56e6a6 100644 --- a/docs/parity_testing.md +++ b/docs/parity_testing.md @@ -157,4 +157,4 @@ Use the payload fields `legacy_decision`, `policy_decision`, `context.action`, a - Unexpected `context.resource_kind` (for example view name fallback): - Fix serializer `Meta.model` usage or explicit resource kind dispatch in permission code. -When fixing mismatches, update code and Cedar together, then rerun the runbook until mismatch count is zero. +When fixing mismatches, update code and Cedar together, then rerun the tests until mismatch count is zero. From e4f4e5c957f03a8526ef5f4e3a39869727d28aeb Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 18 Aug 2026 12:08:11 +0200 Subject: [PATCH 20/34] Use TreeTop policy bundles --- docs/policies.md | 35 ++++++++++++++++--- treetop/data/global.cedar | 8 +++++ treetop/data/mreg-bundle.tar.gz | Bin 0 -> 2383 bytes treetop/data/mreg.cedar | 9 ----- treetop/data/treetop-bundle.toml | 14 ++++++++ treetop/data/treetop-global-module.toml | 4 +++ treetop/data/treetop-host-labels-module.toml | 4 +++ treetop/data/treetop-mreg-module.toml | 4 +++ treetop/docker-compose.yml | 11 +++--- 9 files changed, 72 insertions(+), 17 deletions(-) create mode 100644 treetop/data/global.cedar create mode 100644 treetop/data/mreg-bundle.tar.gz create mode 100644 treetop/data/treetop-bundle.toml create mode 100644 treetop/data/treetop-global-module.toml create mode 100644 treetop/data/treetop-host-labels-module.toml create mode 100644 treetop/data/treetop-mreg-module.toml diff --git a/docs/policies.md b/docs/policies.md index 1ea8c833..cc6e1d48 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -9,10 +9,36 @@ Related documentation: ## Source of Truth -- Policy definitions: `treetop/data/mreg.cedar` +- Bundle manifest: `treetop/data/treetop-bundle.toml` +- Policy module: `treetop/data/treetop-mreg-module.toml` +- Global policy module: `treetop/data/treetop-global-module.toml` +- Label module: `treetop/data/treetop-host-labels-module.toml` +- Policy definitions: `treetop/data/mreg.cedar` and `treetop/data/global.cedar` +- Derived labels: `treetop/data/labels.json` +- Generated bundle: `treetop/data/mreg-bundle.tar.gz` - Action generation in code: `mreg/api/permissions.py` (`ParityMixin._crud_action`) - Parity transport/logging: `mreg/api/treetop.py` +## Building the Bundle + +Install `treetop-bundle` 0.0.4 from the +[`treetop-bundle` releases](https://github.com/treetop-policy-engine/treetop-bundle/releases/tag/v0.0.4), +which matches the bundle format and Treetop Core version supported by the +pinned REST server. Then validate and build the bundle from the repository +root: + +```console +$ treetop-bundle check bundle treetop/data/treetop-bundle.toml +$ treetop-bundle build \ + --manifest treetop/data/treetop-bundle.toml \ + --output /tmp/mreg-bundle.tar.gz +$ mv /tmp/mreg-bundle.tar.gz treetop/data/mreg-bundle.tar.gz +``` + +Bundle output is deterministic. Commit the regenerated archive whenever a +module manifest, Cedar policy, schema, or label definition changes. The local +TreeTop stack loads the archive atomically through `TREETOP_BUNDLE_URL`. + ## Adding a New Protected Resource When introducing a new resource that should be parity-checked, use this checklist: @@ -25,9 +51,10 @@ When introducing a new resource that should be parity-checked, use this checklis 4. Verify resource ID resolution produces stable IDs for list/detail/custom views. 5. Add or update Cedar actions/rules in `treetop/data/mreg.cedar`. 6. If policy conditions depend on derived labels, update `treetop/data/labels.json`. -7. Add tests for create/read/update/delete behavior and group/admin overrides. -8. Run parity checks and confirm zero mismatches. -9. If tests mutate permissions mid-test, scope `disable_policy_parity()` as narrowly as possible. +7. Rebuild `treetop/data/mreg-bundle.tar.gz`. +8. Add tests for create/read/update/delete behavior and group/admin overrides. +9. Run parity checks and confirm zero mismatches. +10. If tests mutate permissions mid-test, scope `disable_policy_parity()` as narrowly as possible. ## Resource Kind and ID Resolution diff --git a/treetop/data/global.cedar b/treetop/data/global.cedar new file mode 100644 index 00000000..721e612e --- /dev/null +++ b/treetop/data/global.cedar @@ -0,0 +1,8 @@ +// Global policy kept in its own bundle module because it intentionally applies +// across every namespace and action. +@id("global.super_admin_allow_all_policy") +permit ( + principal == User::"super", + action, + resource +); diff --git a/treetop/data/mreg-bundle.tar.gz b/treetop/data/mreg-bundle.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..4982959d7afb3d6d05377570311f5d09ff90a215 GIT binary patch literal 2383 zcmV-V39$AbiwFP!00000|Lq%VZ{s#HpYtnN;m~vL99xo~aZ%tF?O|_my&~9l(GR;@ zgCZ%L7g-W0Io@2F|9(SKPg@SzVG4*^YM&~0UFOeAN%tWb;k3N!{$Cf z&Rl0qCLSHxw(N`lk68=H9lO(qenB$89t#pR4y*J=Kw~;`FrCin)qLViX7<(0W_UzK z%w<JLOepI1wD5EWhxLQ#2X3upV$9?D8Ygrwt(q@cWLOX4U8!wH5yhy`${=>ZHM?c`^!k=BHzpqE;wd({60E&H^~ZX-c?h{c`tQ1-PEY}GtE z4~^0^G!I5w{-g=@!ts6T=yXUo5;GePf)(__-{BwS>SQ017ZSAN=~_`A;+PLBGzanBVxd-rr6vNz|G{Z*&nsS>E1VSd z&JmPY@R0Br2YvN7V~>aGpDb@B`eN}yH7xXcx0JnE{Q?Fx!H_6SW%O(U2I7>08jOHM zu98O5JoFlFk}KC0BheDn zNV)_yyT@X^GP}qE)a)h;(2Ri=>tK;jwqFW0-gB&eZ9=<~R>(aCAP8vX&WKV3gt@=u zNi`xkM2zyZk2GcF(|*#c=>7tI{dFCI#{-_c3SUcY_jug1>KeB$F3unHSa?wm1W1Q?ybYx7ad-!I z%#GMb?3RHleFTO8PBB{Tz~0Cc5r%CmFmMhidz5C*Hgn1j;8XHS2k7TD`~)CK^pJ`t zp+CV8OkkfwE`}3i0dOifNg1_(3rNilCU3dx_H6SPNUU~Cog!{nm_8aHE(=w(#>n@( z+QR1=JOv@sVil!f;Jvzjq0OZH1DJ4#eg<4^9ncg}7KDvDis)9zc`tYK;Ja#=!wtUn zmXWw!YSE<@U24F}x5r52x4YJp_DhU)PeT0;~gmlYK?XR%qYAX5I4<#4gf%iESK zz%kyBtOSpO=VInu=jE$jO*kH3>_Zf>XCZjNZy18*^>2fGYd#Vyt3bLyzx{SZCtvmc zoLXrubY0|QSeYvKn|lXbJrdW;s~L74)K@jy3fIxNlhbkc^NFXS!{>11o{a9osoa8k z^TlE~9sdq&XXv=|r{KGQf0a|Op_c#juRPHjq92Q#2=b8H$(0JNWrEC9#=T~u$e{Gb zJ5s0Q2nR`|x)O{qe#>N+Du#Z|R_w!d(IbLDU&wRRN__svVgKo;0a<6=p+Fb8caeJ+ zxp$F!7rA$ld!=-ddl$Lyk?A7$&p&dnpOwIldVWpI;`>FHIT%pNcOp!Yb`4A^S z2Mj-z*Apgtb|GFJ+FAW0>{WLn64X;2UvAjOgE_Z`Wv$$tKxECx&BZa7_DTF#ee2?J z(c~d|esK8|>Gq#ooGzcjK0&%Yif!b(R-PgAEqlz@9)_IGec8o{B2?)bWcdEapU^Ma zgO@`g8lJ1meZ;K|R^Bq_UTFUvmZ!{Y&*9X^W#97OR``9JVf_U5NBsLZrJw9wQi*a&>VeOks5f4sign1dihXwkXwA zAtItAk#|hQ9V%XPsry7)#B7Od^YG`Wf Date: Tue, 18 Aug 2026 15:03:22 +0200 Subject: [PATCH 21/34] Preserve Gunicorn PID health check --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 08f58827..6e8fd2be 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -6,4 +6,4 @@ python manage.py create_citext_extension python manage.py migrate # Let gunicorn become PID 1 so container stop signals are delivered directly. -exec gunicorn --workers 3 --bind 0.0.0.0:8000 mregsite.wsgi +exec gunicorn --workers 3 --bind 0.0.0.0:8000 --pid /var/run/gunicorn.pid mregsite.wsgi From c3620916f99d129a05e3dedadce80159dd2ebe48 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 18 Aug 2026 15:14:09 +0200 Subject: [PATCH 22/34] Update TreeTop stack and compatible CLI tests --- .github/workflows/test.yml | 2 +- ci/MREG-CLI_COMMIT | 1 + docs/policies.md | 4 ++-- treetop/data/mreg-bundle.tar.gz | Bin 3477 -> 3477 bytes treetop/docker-compose.yml | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 ci/MREG-CLI_COMMIT diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8847e61f..ced2ad7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: name: TreeTop bundle runs-on: ubuntu-latest env: - TREETOP_BUNDLE_VERSION: 0.0.4 + TREETOP_BUNDLE_VERSION: 0.0.5 steps: - name: Checkout uses: actions/checkout@v6 diff --git a/ci/MREG-CLI_COMMIT b/ci/MREG-CLI_COMMIT new file mode 100644 index 00000000..9c6de612 --- /dev/null +++ b/ci/MREG-CLI_COMMIT @@ -0,0 +1 @@ +7ace29daec005c30f33fbb14bfb9254cad1d8bc8 diff --git a/docs/policies.md b/docs/policies.md index 07d4788b..622cafe1 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -21,8 +21,8 @@ Related documentation: ## Building the Bundle -Install `treetop-bundle` 0.0.4 from the -[`treetop-bundle` releases](https://github.com/treetop-policy-engine/treetop-bundle/releases/tag/v0.0.4), +Install `treetop-bundle` 0.0.5 from the +[`treetop-bundle` releases](https://github.com/treetop-policy-engine/treetop-bundle/releases/tag/v0.0.5), which matches the bundle format and Treetop Core version supported by the pinned REST server. Then validate and build the bundle from the repository root: diff --git a/treetop/data/mreg-bundle.tar.gz b/treetop/data/mreg-bundle.tar.gz index 306715a67101be9716b80329811007f704d4add4..02e941077691d9bf39373ad5da8683c664a64c15 100644 GIT binary patch literal 3477 zcmV;G4QlcqiwFP!00000|Lt8_bK5u)_OpKlhP5?i@5(YsiMpnyYO|TCY-*ECD#y-4 zl2Jh`r$)@pO@o1od4~-9vi=NZapRzP-|DHzS$#H@S zq1Ww-UkK~J9jjNq%b#0j*RoH{lSc@PG@}V2C*NqeH)xXiQ|e}^G5T zu4^%CI++d}&zwxDXAXH<{(qSdx4l3OlUpV|I7QOwaVZiOVdfH*qH& zI$@sK8w{<gy&>ysWOUn>-8LaYmN1q@@mS`A zXQFMko!SdGN{Yv}Wh`69Jn|L+ljh*h<0uiP0YwSvoJ+oZ^Yb%9-lZ{hYmbE0+(B7+ z1o5z)FzStI5Wrh;PrbPxj@@M8tvpG`GH&tkhD{ou=MVjO?Rf?iHoi)t#rlhoWj9gs zc`WrRB}CH2lU2Dz3Yc$yEzK#{(~#*#b;ZM3lx8w-8;^5LSDt&2&l^ART$*^BZ=j+f zr@UJ8cTjanBP zZmH7NP8RWLd3u83PZL%<`jexChyA~f+3EB*{6EJ=|L-xv*r~k$NCpzG0A3rN&-L4h!2tOLYLh5*-U7^q_KZl)7g&iMkB%c zW2pp`|K0b_)>!ZSRK&^1*(re#Cw}PqF%4ScHqV|PiaQ=Y@aUt_6Y*j+GB}}+d1^+h zlp#$rZWLzRdsYd9a?b-8l!7kr1(mA(a2lzEg2+{g%x_f^A&s*{Ed*JHegxc7l_*aw z0=dNZl%-75*)&!PCD$sE>{eNq6Y4_$2hN7FdcevTp`YDJ!B0}MumW)`gs9Ol7!>*xNzMV*z_msmVP!T}7*Uyz6=`KgR-~0Vk$kJnf)rEgmSSq9z~5?>Af#3Z zLP~YKFHoxELP~X9NU5&sSCmRCQKjliR85a%f2F3#QdCWorD&dk9s0p?1i5=xDB~TY z_uIv_+pC9M1Au@-E2aMlLIMtOc7H7f)5N7AVVYJN;3dnf6P|1;Jzz>Wl@KwF zK!mI5hkkfP3>cP6elc2vh8}p(j68qMb>tlU z-L2Ko#xQ^PN-CrNKh)Fd&5VVo_p359nu>aOA!c*P_up^6_T%-(?J7n)uetQBR&VDU z5&XFAr_aMTyr$0l_{>HcVGjB0kjCaW>5`N0+k4* zq~z)bY=umch{D7c7)K$~;1Yyjs7QoI3h5<`00m5a7|j6y z1_)<~?`F(1AQv#W+i3pPyxh?k090JvjO8G6&cf_L7js-fv2%=Kj%-)?0>OPE ziEl&vVWE8`YIP^Cis9BJc~y>#U^KF_0w!E*KE!5X%syjDIgXMO_(ZI)6B?wENGf42 z^RF3CQuxqs*cH2dU8YFD-B0*>Y|TG^U{k>Us|tByxVV6xJ9_TuxufTfo;!N(wSb;G zdhTm#&~yL#J@?fW6PQ;oR!hK9=2y8}yv^%w+90dn_x|z6eU=&R#mt5E8@DKYS}XmwUBSB-TBLBa@@-wGt6Fx|qQ9_3?LYv7!V)Wst67>C zK}Q9Y2R@Qq&dtx9)NpRvAilq0i5=H_j^-qJk3|L^=!_)`E5)vTcB2&H3*NcV))g(W zVp`hoD}LN1kmK%jeMM2{p1Mub%=fR0B{ZX9cf-cKKuXR;!WVNkpMdSpz!a;g0^-h? z`}uqu!8{6^tWG36Tr|EMUMwSiN$J)PO6*@g5qpo)t!dn)e6K}=%bV<4=2wn$?4SE8 zi9L%@*Q+SQPp{sRU-M%sn}ujN#Ik=eAd!nAk=@F$sdk4_ye|*pG?!!Jpw5?-q)KM@ zvaGA#UYdrz^zuYZv70_WA5-q9e~c?cU@*=qQG5%Fj*Z{g1?KH(~U=j?61=h1kbCBNiTk8t;v&j#|}*Cbo%KeiJR%s>+KAW-S#lLB=fXR#H zvb8x)hmh)nG@m6xNcBM~_S_<%`T)%*J`vJGAzhhvMmY7ri43Y0PGnDwU?Owsf^&VP z79rIKDKe;bNRdS~1By(l3)1?!8NzueoGa^#2-t`AG>iZr6xaGU^v)d6hWaL56XJrvn(pEN{dH6XkCT}gyg z1EispL0mQC+Vs^?1Xd5Q4PR$PXb*>W?K@8hsXj{ERimCA&G3M3CnQv`|1d)PEe6Qj{qtXsdh+_QMG|u`!^W~rye+wMYY0-{HYO4 zWKLah=KGj%JL6Gc#V$!Rpx7yC1~ksMGZ!LgvPVZ4ZTQ#-qpx*be9`?nHp16j2D$i= za7IRc9GsDt9|L7%=i}gvYKiJg!pFt2@fgZ~^#3z&c=3o}rb!*XBcY;?aY zN-i(Ue2UuwK6%4GjVv!EM)H#53Q49j8cVw`^b1Whc%3J2$Y0|vycZ=-*nbv&!hm|F zG)NgVpgW#~@w3S6FA2{~6dL>+SKfi~<63-QRQg`=Kl|Z0-!1*_jn|!886@b{A@JJ{iJA-#fvPL$-5oVMPw`qdDE^~G#q=; zocf`l!7B#DWw2X1j|2AcW8019e*xkwrLSq=d*d56$b>IsqKSpqeiDVEo2*62UyDeemYR literal 3477 zcmV;G4QlcqiwFP!00000|Lt8{bKAHT_OpKlmebj>TUjP4QkJx{GflHQX{JdwlS}s@ z$!Z`Hvbdp0mZW^C*Z;i-0B@2gLy92Jra5@9csalUJ{){FxY2VO1yh#g-QTk~Iyy`+ zA@m(b{6bj$?OFZmUGdy99m_s4j~*c`vYe)b9DSqV&Y)=?OsSV=#^}SHk z1214%*JD0SjgFDcsO|RQ&6L@G-|kx`>ksXo<1p${|H5*oJI)2}B{i!#&=vh-JHnWEPq3b!`z;@k%%`D4tok69~zOe5< zb&Sa(@6(_`ADBKQVmmgqr@rs{_H@$k zUoblGFNPMRJ&n^j&Bs?P%>u~y$m$rEEMh6mrAH&D@# zQ(mpbJE*!e7Ihz|eh|_0*5K`+t!^*FctXRK+N~Zg*ROos%@zqui*iFn$JhLCjX)ta zPub;Ih_2?TNd3Of1ea~MrlqoOG^I7`Xf>r)sFVQjSW^#JMXMp`GuPFIx3PsN?7F2T zZm81ERu=K;{`d&PpC&AK^hXB?5Bq;Tv)8*=^Z#5M{lCWu=jWtw1kPl?KO(%@|3J8x zG$Oo>OVc<8N&bdF+nxs*_dYVhZsp#`27S%R*9? z#f#KqXLB0SOXia&ub*dK9_wF>B_y8(83}_dCjmsdjXZK46}LuWU+9gPI* zkEIe&{`bH?Sz*1?V-Y7OC&vUrk_M3%BsA=Z+ah~GB<^_lz@v{wPsEGS$l!!N=BXK- zN`|z_cyW|-?^z`b%6%VTPzrjy7gVYaqG_xW3S&H#ZXL_vNl1wTp2(!P&Re`uOx!$sL- zklf;>if6oY1`R%&(nXk`iO6T7%Qkj0TMu%UGnL^k5`PORpM~r}soJiUYHLxY?pjot z0oh)Z%B7f6u@qA(mF-Urv=CDxEyR>cd3#o>@Ug_YS@Vnk&=mZX&#S&~-fMDndN3sOv}TZ*Zb0)MMjf{+Q8hhQ{gs*`D^WE~R-#1)w&(||5#;t=p^SHk z-ftV%Zl@k{1popLt(5*J2njgA+5NQ~OcRerg!w_lurV@0Mj|G!ymkLd@on@4sJv9VDxdyLF6iQFGZzqux%} zA_Pe{$eu@Ucuk!I$%(OCTgK_>@x8R8C$$m5fGS89A-5YE-GU922GAkwia|9@2~;AK zl9J17uoW^%V+s>nU>v0&Pm!BQF#%L30DpO2{VsVogG&&Cp&}6;DWsP&0u(R}U^E8+ z7$BUdftNGifLy@fZms#(^Ku7c08nvxJ(h#aIg9cKUCeO>#nv&(IkIi#O9Xd`P@ve1 zve9UtmwvKf7^ui|@(xmvbp&{Mq8?$2&nNyO<;=n70pYhy9`9C!-(7fUCzX==iih>it_&Zx>;X3AeM3+1pmg@-SaIMX6f2D5#1`TCF8og$6Xy&d|7n(d)~8mj&8&1-pHQ}Mx*n-vqx;}+_DE> z65odS!&3WF)cQ_dmBX!b@~Rpc!DwV@1x&cse2C4&n0?05Y8)je@QGMoCp64rkyOe& z7F;o&r0}8NuuFFHx=N9NyPxv)*oJ@pz@~uRR~3rHaB%@Wcl6xRb4Sk|J$Lln8v#9c z^xRj}py&Sed+y6CCNQsFu9kqK%&!W!c$3%Nwn0|E@BQPCyDT}VRs(qP-;2QqFLfM+ zw`I{`c}8rsOPB}iH(puztWo++yMnhbv`FD#<=ePUSGVlCMSo$7+JXQEr6txD*RV7{ zhK>p-4}2uKoLihZY2e(pL40?^5<9MU9L-7c9*Yb-&>2e>R*GE(>{=WZSyJ$37*S?pgIOK3*R?uNB_fsCApgwN(|J^|aGfhpEg1;m>% z@AK&5saH{kpI*HszZS<-)(g>ch-LR;Kq6;lBHNW=UF{a7cwZgFX)njxL7guvNuA90 zWm#3dxil?%>E(%-ayNZ(KBn4F|4iHyS>xn0OT_;7045US>xjq`dqgHhCd{MIrHaiQ zQ4YN%57$q{ z*-wz*v(@nXBIeOv-oT$%e8N@y&e_{y&!h1=Pk$+<9^vjSpAF={=@tBacf(Wv$CIbv z?>zsBCtgk{Raq$RUqZ=(h#xk6AzfFa@aXER*X60lrTZs2tkO&td^Tgli+|H_0h1T^ z_nq}=I)qdoq{S=|LaGl^vF8>6)dy%X@rjV`3+d9dGs39{PGnH6a3XtZ1QVH47o4jr zwFs#`NRdIcLy9b_8Bk z`+~W&g&g7311GxqTH!<&Un7|4-s^%>e$=lVHqE6{q|yW9i`;rxe35Jqi7v9QUVNLM zR7FtrfQlWMMl`Xz(upRvbA4E1SELcmmg58ntPWt4mO~DR?7qlu`lKNus{z^N?@A)1 z8X%3N4C1O8*S4>YBCvXZZTUJYLc2e-E8lrSNcBOAV^!^t;(%2%pg3yP1!*Ra)-E_s z24O;|?j-&aRab%Kigc>qw6NVP+XjH(UP#=prxIQ77ZEUFbwGkvVn2 zS$<9eH@4}5G{3>`@oEJVJ0gu>VmG7>O6-I*fq50MYy9U>TEGM}S(Jma0+zcvXQR9O zvgGoz%BQ$3;gdJ~)5!8tVk9p)u8?FlqlvWpQoqnNgV#m!hWs_&!h2ETl>KKBqztHM zO2dpn1G?o&7(a{5{*v;{#F4?japmtBKWW4dM5XT(|8o%SnS=c?;4y`b2wb82E5Ag! zn&ie5HtBlHa{fA@`D~vo-dAtlFfUGh@ubz*E5j*UqIgg@#?9vRwHrb!MlTAMyO*RR zE`WE$Z-a7*drn32vLK4)O~DoGk7~Mj;GJX)A<$$u@qiPXybw$$E;-GKi}%I1=jLoa zVd*c^ntugna1OdZKEm*42%=BTy1!>}babd-Lg+h=_=T|gYdUs)EuUMa+3O#fM~@H| zSx!?zj=s_G;m()_ECi}D3XeyUn(^|@&(B7q!cU49d9uh0nWEbfT|~~(h&Sz;MWeAF z&uI_|8oXpcTn4+L^CV;+KX$!%{udz5O8Sb1fj_=xlj1CQk%cniZ}8^q<46B*%{cOq zUACC4>G@5(0yRx>sNPHdFPd0%6{K+_y2(bA;HZVa5(W$yzM1epq22y-0MGyc D#O>OA diff --git a/treetop/docker-compose.yml b/treetop/docker-compose.yml index 7ecd38bc..f0ac2caa 100644 --- a/treetop/docker-compose.yml +++ b/treetop/docker-compose.yml @@ -9,7 +9,7 @@ services: command: ["/data", "--port", "8080"] treetop-server: - image: ghcr.io/treetop-policy-engine/treetop-rest:v0.0.13 + image: ghcr.io/treetop-policy-engine/treetop-rest:v0.0.14 pull_policy: "always" depends_on: - bundle-server From e2d411ceb8c579d340af2b62d0877181aecec431 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 18 Aug 2026 16:34:31 +0200 Subject: [PATCH 23/34] fix: harden TreeTop policy rollout --- .github/workflows/container-image.yml | 16 +- .github/workflows/test.yml | 8 + Dockerfile | 34 +- README.md | 16 +- docs/env.md | 52 ++- docs/metrics.md | 44 +- docs/parity_testing.md | 20 +- docs/policies.md | 37 +- docs/testing.md | 11 + entrypoint.sh | 14 +- monitoring/grafana/treetop-parity.json | 51 +++ monitoring/treetop-alerts.yml | 53 +++ mreg/api/permissions.py | 145 ++----- mreg/api/tests/test_metrics.py | 15 + mreg/api/treetop.py | 383 +++++++++++++++--- mreg/api/views.py | 19 +- .../commands/check_policy_rollout.py | 45 ++ mreg/middleware/metrics.py | 7 +- mreg/migrations/0017_policyparityoutbox.py | 35 ++ mreg/models/__init__.py | 1 + mreg/models/policy.py | 26 ++ mreg/policy/__init__.py | 1 + mreg/policy/contracts.py | 174 ++++++++ mreg/policy/resources.py | 161 ++++++++ mreg/policy/rollout.py | 97 +++++ mreg/tests/test_gunicorn_conf.py | 34 ++ mreg/tests/test_policy_contracts.py | 13 +- mreg/tests/test_policy_rollout.py | 38 ++ mreg/tests/test_treetop_batching.py | 103 ++++- mregsite/gunicorn_conf.py | 40 ++ mregsite/settings.py | 14 +- pyproject.toml | 3 +- scripts/generate-treetop-schema.py | 35 ++ tox.ini | 2 +- treetop/data/mreg.cedarschema | 30 +- uv.lock | 6 +- 36 files changed, 1540 insertions(+), 243 deletions(-) create mode 100644 monitoring/grafana/treetop-parity.json create mode 100644 monitoring/treetop-alerts.yml create mode 100644 mreg/management/commands/check_policy_rollout.py create mode 100644 mreg/migrations/0017_policyparityoutbox.py create mode 100644 mreg/models/policy.py create mode 100644 mreg/policy/__init__.py create mode 100644 mreg/policy/contracts.py create mode 100644 mreg/policy/resources.py create mode 100644 mreg/policy/rollout.py create mode 100644 mreg/tests/test_gunicorn_conf.py create mode 100644 mreg/tests/test_policy_rollout.py create mode 100644 mregsite/gunicorn_conf.py create mode 100644 scripts/generate-treetop-schema.py diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 9443f935..ae433699 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -1,6 +1,8 @@ name: Container image on: push: + branches: [master] + tags: ['v*'] paths-ignore: - 'ci/**' - 'README.md' @@ -8,6 +10,10 @@ on: types: [opened, reopened, synchronize] workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + defaults: run: shell: bash @@ -20,9 +26,11 @@ jobs: - name: Checkout uses: actions/checkout@v6 - name: Docker build - run: docker build -t mreg . + run: | + docker build --target runtime -t mreg . + docker build --target test -t mreg-test . - name: Save image - run: docker save mreg | gzip > mreg.tgz + run: docker save mreg mreg-test | gzip > mreg.tgz - name: Upload artifact uses: actions/upload-artifact@v7 with: @@ -57,9 +65,9 @@ jobs: run: docker load --input mreg.tgz - name: Run tests run: | - docker run --rm -t --network host --entrypoint /app/entrypoint-test.sh \ + docker run --rm -t --network host \ -e MREG_DB_HOST=localhost -e MREG_DB_PASSWORD=mreg -e MREG_DB_USER=mreg \ - mreg + mreg-test mreg-cli: name: Test with mreg-cli diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ced2ad7f..b8eca0ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,7 @@ on: push: + branches: [master] + tags: ['v*'] paths-ignore: - 'ci/**' - 'README.md' @@ -8,6 +10,10 @@ on: types: [opened, reopened, synchronize] workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + #env: # UV_FROZEN: 1 @@ -36,6 +42,8 @@ jobs: run: scripts/check-treetop-bundle.sh env: TREETOP_BUNDLE_BIN: ./treetop-bundle + - name: Check generated Cedar contracts + run: python scripts/generate-treetop-schema.py --check test: name: Test diff --git a/Dockerfile b/Dockerfile index 829556a2..42dfec60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# build stage +# Runtime dependency build stage. FROM python:3.12-alpine AS builder WORKDIR /app ENV PYTHONDONTWRITEBYTECODE=1 @@ -20,8 +20,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \ ENTRYPOINT [ "/bin/sh" ] -# final stage -FROM python:3.12-alpine +# Test dependencies are isolated from the production environment. +FROM builder AS test-builder +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked --no-editable --group dev + +# Production runtime stage. +FROM python:3.12-alpine AS runtime EXPOSE 8000 WORKDIR /app @@ -36,8 +41,13 @@ ENV PATH="/app/.venv/bin:$PATH" COPY --from=builder /app/.venv /app/.venv # Copy over application files -COPY entrypoint* manage.py /app/ -COPY mreg /app/mreg/ +COPY entrypoint.sh manage.py /app/ +COPY \ + --exclude=tests \ + --exclude=api/tests \ + --exclude=api/v1/tests \ + --exclude=**/__pycache__ \ + mreg /app/mreg/ COPY mregsite /app/mregsite/ COPY hostpolicy /app/hostpolicy/ COPY --from=ghcr.io/astral-sh/uv:0.12.0 /uv /uvx /bin/ @@ -48,3 +58,17 @@ RUN apk update && apk upgrade \ && chmod a+x /app/entrypoint* CMD ["/app/entrypoint.sh"] + +# Dedicated test image. Production tests and their dependencies exist only here. +FROM runtime AS test +COPY --from=test-builder /app/.venv /app/.venv +COPY --from=test-builder /app/mreg/tests /app/mreg/tests +COPY --from=test-builder /app/mreg/api/tests /app/mreg/api/tests +COPY --from=test-builder /app/mreg/api/v1/tests /app/mreg/api/v1/tests +COPY entrypoint-test.sh /app/entrypoint-test.sh +RUN chmod a+x /app/entrypoint-test.sh +ENTRYPOINT ["/app/entrypoint-test.sh"] +CMD [] + +# Keep an unqualified `docker build .` production-safe. +FROM runtime AS final diff --git a/README.md b/README.md index 318d61cf..e9a1e27c 100644 --- a/README.md +++ b/README.md @@ -185,11 +185,23 @@ mreg supports configuration via environment variables with the `MREG_` prefix. T | `MREG_POLICY_PARITY_ENABLED` | `True` | Enable parity checks when a policy base URL is configured | | `MREG_POLICY_BASE_URL` | `""` | TreeTop REST base URL; an empty value disables calls | | `MREG_POLICY_NAMESPACE` | `MREG` | Cedar namespace used for principals, actions, and resources | -| `MREG_POLICY_PARITY_BATCH_ENABLED` | `True` | Submit one background parity batch per HTTP request | -| `MREG_POLICY_PARITY_QUEUE_SIZE` | `100` | Maximum queued parity batches per process | +| `MREG_POLICY_PARITY_BATCH_ENABLED` | `True` | Persist one durable parity batch per HTTP request | | `MREG_POLICY_TIMEOUT_SECONDS` | `5.0` | TreeTop client timeout in seconds | +| `MREG_POLICY_PARITY_MAX_ATTEMPTS` | `8` | Delivery attempts before retaining a dead letter | +| `MREG_POLICY_PARITY_RETRY_BASE_SECONDS` | `2.0` | Initial durable-outbox retry delay | +| `MREG_POLICY_PARITY_RETRY_MAX_SECONDS` | `300.0` | Maximum durable-outbox retry delay | +| `MREG_POLICY_PARITY_LEASE_SECONDS` | `60.0` | Time before another worker may reclaim an abandoned row | +| `MREG_POLICY_PARITY_POLL_SECONDS` | `1.0` | Durable-outbox polling interval | +| `MREG_POLICY_PARITY_CIRCUIT_FAILURES` | `5` | Consecutive failures that open the delivery circuit | +| `MREG_POLICY_PARITY_CIRCUIT_RESET_SECONDS` | `30.0` | Open-circuit cooldown | | `MREG_POLICY_PARITY_LOG_LEVEL` | `WARNING` | Dedicated parity logger level | | `MREG_POLICY_PARITY_LOG_DETAILS` | `False` | Include sensitive principal/resource details in parity logs | +| `MREG_POLICY_ROLLOUT_MIN_COMPARISONS` | `10000` | Minimum observations required by the enforcement gate | +| `MREG_POLICY_ROLLOUT_MAX_MISMATCH_RATE` | `0.001` | Maximum accepted mismatch ratio | +| `MREG_POLICY_ROLLOUT_MAX_ERROR_RATE` | `0.001` | Maximum accepted policy error ratio | +| `MREG_POLICY_ROLLOUT_MAX_PERSIST_FAILURES` | `0` | Maximum accepted outbox persistence failures | +| `MREG_POLICY_ROLLOUT_MAX_DEAD_LETTERS` | `0` | Maximum accepted dead letters | +| `MREG_POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS` | `300.0` | Maximum age of the oldest pending batch | ### Network Policy Configuration diff --git a/docs/env.md b/docs/env.md index 7a619940..28deb936 100644 --- a/docs/env.md +++ b/docs/env.md @@ -61,14 +61,6 @@ Keep this disabled unless detailed parity investigation is necessary. These fields may contain operationally sensitive data. Parity events use the normal console and rotating `MREG_LOG_FILE_NAME` handlers. -## `MREG_POLICY_PARITY_QUEUE_SIZE` - -Maximum number of parity batches waiting for the process-local background -worker. Default: `100` - -When the queue is full, the batch is dropped, the legacy decision is preserved, -and a metric/log event is emitted. - ## `MREG_POLICY_TIMEOUT_SECONDS` Timeout in seconds for calls from the background parity worker to TreeTop. @@ -79,9 +71,47 @@ Default: `5.0` Boolean flag controlling request-scoped batching of parity authorize checks. Default: `True` -When enabled, parity checks are collected during request handling and submitted -to a bounded background worker as one batch. Requests never wait for TreeTop. -When disabled, each check is submitted as its own background batch. +When enabled, parity checks are collected during request handling and persisted +to the PostgreSQL outbox as one batch. Requests never wait for TreeTop. When +disabled, each check is persisted as its own durable batch. + +## Durable parity delivery + +The following settings control the shared PostgreSQL outbox: + +- `MREG_POLICY_PARITY_MAX_ATTEMPTS` (`8`): delivery attempts before a row is + retained as a dead letter. +- `MREG_POLICY_PARITY_RETRY_BASE_SECONDS` (`2.0`): initial exponential-backoff + delay. +- `MREG_POLICY_PARITY_RETRY_MAX_SECONDS` (`300.0`): retry delay cap. +- `MREG_POLICY_PARITY_LEASE_SECONDS` (`60.0`): time before an abandoned claim + can be reclaimed by another worker. +- `MREG_POLICY_PARITY_POLL_SECONDS` (`1.0`): worker polling interval. +- `MREG_POLICY_PARITY_CIRCUIT_FAILURES` (`5`): consecutive delivery failures + that open a worker's circuit breaker. +- `MREG_POLICY_PARITY_CIRCUIT_RESET_SECONDS` (`30.0`): circuit cooldown. + +Successful rows are deleted. Exhausted rows remain in +`mreg_policyparityoutbox` with `failed_at` and `last_error` populated. The +outbox necessarily contains the principal, groups, resource identifier, and +resource attributes required for a later authorization call. Protect database +access accordingly and establish an operational dead-letter retention policy. + +The container sets `PROMETHEUS_MULTIPROC_DIR` to an isolated directory so +metrics from every Gunicorn worker are aggregated. Custom Gunicorn deployments +must set this variable to a clean, writable directory before starting Python. + +## TreeTop enforcement rollout gates + +`manage.py check_policy_rollout` evaluates Prometheus telemetry before an +operator enables policy enforcement. Defaults can be tuned with: + +- `MREG_POLICY_ROLLOUT_MIN_COMPARISONS` (`10000`) +- `MREG_POLICY_ROLLOUT_MAX_MISMATCH_RATE` (`0.001`) +- `MREG_POLICY_ROLLOUT_MAX_ERROR_RATE` (`0.001`) +- `MREG_POLICY_ROLLOUT_MAX_PERSIST_FAILURES` (`0`) +- `MREG_POLICY_ROLLOUT_MAX_DEAD_LETTERS` (`0`) +- `MREG_POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS` (`300.0`) ## `MREG_LOG_FILE_SIZE` diff --git a/docs/metrics.md b/docs/metrics.md index ba8e5458..35de62bf 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -135,15 +135,31 @@ Metrics are exposed at the following endpoint: `/api/meta/metrics`. - Type: Counter - Labels: status - Unit: batches - - Description: Batches submitted to or dropped by the bounded background worker. - - Label values: `submitted`, `dropped` + - Description: Durable outbox lifecycle events. + - Label values: `persisted`, `processed`, `retried`, `dead_letter`, `persist_failed` + +- Name: mreg_policy_parity_outbox_entries + - Type: Gauge + - Labels: status + - Description: Current shared outbox rows by `pending` or `dead_letter` status. + +- Name: mreg_policy_parity_outbox_oldest_seconds + - Type: Gauge + - Labels: none + - Unit: seconds + - Description: Age of the oldest pending durable batch. + +- Name: mreg_policy_parity_circuit_open + - Type: Gauge + - Labels: none + - Description: `1` while a worker's TreeTop delivery circuit is open, otherwise `0`. - Name: mreg_policy_parity_failures_total - Type: Counter - Labels: stage - Unit: failures - Description: Fail-open parity instrumentation failures by processing stage. - - Typical label values: `build`, `submit`, `request_exit`, `worker`, `result_logging` + - Typical label values: `build`, `persist`, `request_exit`, `worker`, `result_logging` - Name: mreg_policy_authorize_duration_seconds - Type: Histogram @@ -170,9 +186,19 @@ Metrics are exposed at the following endpoint: `/api/meta/metrics`. - Prometheus boundaries: [0, 1, 2, 3, 5, 8, +Inf] When request batching is enabled (default), `mreg_policy_queries_per_request` -should usually be `0` (no parity checks queued) or `1` (one batch submitted). -The background worker performs the corresponding authorize call after request -handling. +should usually be `0` (no parity checks produced) or `1` (one batch persisted). +The durable outbox worker performs the corresponding authorize call after +request handling. + +## TreeTop rollout dashboard and alerts + +- Grafana dashboard: `monitoring/grafana/treetop-parity.json` +- Prometheus alerts: `monitoring/treetop-alerts.yml` +- Executable gate: `python manage.py check_policy_rollout --prometheus-url URL` + +The default gate requires at least 10,000 comparisons over the selected window, +at most 0.1% mismatches, at most 0.1% errors, zero persistence failures, zero +dead letters, and a pending backlog younger than five minutes. ## Labeling Strategy @@ -186,7 +212,11 @@ handling. - Timing uses monotonic clocks to avoid wall-clock skew. - The metrics endpoint (/api/meta/metrics) is not instrumented and is tolerant to a trailing slash. - Gauges are carefully paired to prevent underflow. -- For multi-process deployments, ensure Prometheus client multiprocess mode is configured or scrape per-worker and aggregate in Prometheus. +- The container configures Prometheus client multiprocess mode and cleans its + per-process files before Gunicorn starts. Custom process managers must set + `PROMETHEUS_MULTIPROC_DIR` to a clean, writable directory before Python + starts and call `prometheus_client.multiprocess.mark_process_dead` when a + worker exits. - Avoid building dashboards/alerts on high-cardinality labels; stick to method/path/status/exception. ## Alerting Examples diff --git a/docs/parity_testing.md b/docs/parity_testing.md index 801add4e..a2ce0482 100644 --- a/docs/parity_testing.md +++ b/docs/parity_testing.md @@ -88,9 +88,11 @@ Keep parity disable scope as narrow as possible: The `disable_policy_parity()` context manager uses `ContextVar` state. Nested contexts and concurrently handled requests are isolated from one another. -Parity HTTP calls run on a bounded process-local background worker. Client, -serialization, queue, logging, and TreeTop failures are fail-open: they are -recorded, but never replace the legacy permission decision. +Parity batches are persisted in a shared PostgreSQL outbox. Post-fork workers +claim rows with database locks, retry with exponential backoff, and retain dead +letters after the configured attempt limit. A circuit breaker protects an +unavailable TreeTop service. Client, serialization, persistence, logging, and +TreeTop failures remain fail-open and never replace the legacy decision. ## Parity Runbook @@ -125,6 +127,18 @@ Set `MREG_POLICY_PARITY_LOG_DETAILS=True` temporarily in a suitably protected environment only when principal, group, resource ID, or attribute details are required for triage. +5. Run the enforcement readiness gate against the production Prometheus: + +```bash +python manage.py check_policy_rollout \ + --prometheus-url https://prometheus.example.org \ + --window 24h +``` + +Do not enable enforcement until this command passes. Import +`monitoring/grafana/treetop-parity.json` and load +`monitoring/treetop-alerts.yml` before the observation window begins. + ## Mismatch Triage Guide Use `legacy_decision`, `policy_decision`, and `context.action`. Detailed resource diff --git a/docs/policies.md b/docs/policies.md index 622cafe1..30647343 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -13,10 +13,11 @@ Related documentation: - Policy module: `treetop/data/treetop-mreg-module.toml` - Global policy module: `treetop/data/treetop-global-module.toml` - Policy definitions: `treetop/data/mreg.cedar` and `treetop/data/global.cedar` -- Cedar schema: `treetop/data/mreg.cedarschema` +- Python resource/action contracts: `mreg/policy/contracts.py` +- Typed resource adapters: `mreg/policy/resources.py` +- Generated Cedar schema: `treetop/data/mreg.cedarschema` - Derived labels: `treetop/data/labels.json` - Generated bundle: `treetop/data/mreg-bundle.tar.gz` -- Action generation in code: `mreg/api/permissions.py` (`ParityMixin._crud_action`) - Parity transport/logging: `mreg/api/treetop.py` ## Building the Bundle @@ -28,14 +29,18 @@ pinned REST server. Then validate and build the bundle from the repository root: ```console +$ python scripts/generate-treetop-schema.py --check $ treetop-bundle build \ --manifest treetop/data/treetop-bundle.toml \ --output treetop/data/mreg-bundle.tar.gz $ TREETOP_BUNDLE_BIN=treetop-bundle scripts/check-treetop-bundle.sh ``` -Bundle output is deterministic. Commit the regenerated archive whenever a -module manifest, Cedar policy, schema, or label definition changes. The local +The schema's entities and action declarations are generated from the Python +contracts. Add a `ResourceContract` and its adapter before changing policies; +CI rejects a stale generated schema. Bundle output is deterministic. Commit the +regenerated archive whenever a module manifest, Cedar policy, contract/schema, +or label definition changes. The local TreeTop stack loads the archive atomically through `TREETOP_BUNDLE_URL`. MREG currently uses unsigned bundles, verified with the explicit `allow-unsigned` signature policy. @@ -45,23 +50,25 @@ signature policy. When introducing a new resource that should be parity-checked, use this checklist: 1. Ensure the permission path reaches `ParityMixin.pp()` or `pp_generic_action()`. -2. Confirm CRUD action dispatch is used (`_`). -3. Define the resource kind contract for the endpoint: +2. Add a `ResourceContract` and registered adapter in `mreg/policy/`. +3. Confirm CRUD action dispatch is used (`_`). +4. Define the resource kind contract for the endpoint: - Use serializer `Meta.model` for model-backed views. - Set `policy_resource_kind` explicitly on non-model views. - Set a `policy_actions` operation mapping when an endpoint action is not the model's conventional CRUD action. -4. Verify resource ID resolution produces stable IDs for list/detail/custom views. -5. Add or update Cedar actions/rules in `treetop/data/mreg.cedar`. -6. If policy conditions depend on derived labels, update `treetop/data/labels.json`. -7. Rebuild `treetop/data/mreg-bundle.tar.gz`. -8. Add tests for create/read/update/delete behavior and group/admin overrides. -9. Run parity checks and confirm zero mismatches. -10. If tests mutate permissions mid-test, scope `disable_policy_parity()` as narrowly as possible. +5. Verify resource ID resolution produces stable IDs for list/detail/custom views. +6. Regenerate `treetop/data/mreg.cedarschema` and update Cedar rules. +7. If policy conditions depend on derived labels, update `treetop/data/labels.json`. +8. Rebuild `treetop/data/mreg-bundle.tar.gz`. +9. Add tests for create/read/update/delete behavior and group/admin overrides. +10. Run parity checks and confirm zero mismatches. +11. If tests mutate permissions mid-test, scope `disable_policy_parity()` as narrowly as possible. ## Resource Kind and ID Resolution -`ParityMixin` resolves resource kind and ID using deterministic contracts. +`ParityMixin` delegates resource kind, ID, and attributes to the typed adapters +in `mreg/policy/resources.py`. Resource kind fallback order (`_resource_kind_from_view`): @@ -138,7 +145,7 @@ Where operation is mapped from HTTP method: ## Attribute Contract for Policy Checks -All resource attributes are normalized through `ParityMixin._normalize_resource_attrs`: +All resource attributes are normalized through the registered resource adapter: - `kind` is always added using snake_case resource kind. - Attribute values are stringified. diff --git a/docs/testing.md b/docs/testing.md index 8354b900..c1229a84 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -177,6 +177,17 @@ If a test fails only when run in parallel: ## CI/CD Integration +Container tests use the dedicated `test` target, which contains test modules +and development-only dependencies such as `unittest-parametrize`: + +```bash +docker build --target test -t mreg-test . +docker run --rm mreg-test +``` + +The default/final image is the `runtime` target and excludes test packages and +test-only dependencies. + The parallel flag is already enabled in `tox.ini` for all test environments: ```ini diff --git a/entrypoint.sh b/entrypoint.sh index 6e8fd2be..ab7991e8 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,5 +5,17 @@ cd /app python manage.py create_citext_extension python manage.py migrate +# Configure multiprocess metrics only for the new Gunicorn process. Doing this +# after one-shot management commands prevents their metric files becoming stale. +PROMETHEUS_MULTIPROC_DIR="${PROMETHEUS_MULTIPROC_DIR:-/tmp/mreg-prometheus-multiproc}" +export PROMETHEUS_MULTIPROC_DIR +mkdir -p "$PROMETHEUS_MULTIPROC_DIR" +find "$PROMETHEUS_MULTIPROC_DIR" -maxdepth 1 -type f -name '*.db' -delete + # Let gunicorn become PID 1 so container stop signals are delivered directly. -exec gunicorn --workers 3 --bind 0.0.0.0:8000 --pid /var/run/gunicorn.pid mregsite.wsgi +exec gunicorn \ + --config /app/mregsite/gunicorn_conf.py \ + --workers 3 \ + --bind 0.0.0.0:8000 \ + --pid /var/run/gunicorn.pid \ + mregsite.wsgi diff --git a/monitoring/grafana/treetop-parity.json b/monitoring/grafana/treetop-parity.json new file mode 100644 index 00000000..f700013a --- /dev/null +++ b/monitoring/grafana/treetop-parity.json @@ -0,0 +1,51 @@ +{ + "annotations": {"list": []}, + "editable": true, + "panels": [ + { + "id": 1, + "title": "Parity mismatch rate", + "type": "timeseries", + "targets": [{"expr": "sum(rate(mreg_policy_parity_results_total{result=\"mismatch\"}[5m])) / clamp_min(sum(rate(mreg_policy_parity_results_total{result=~\"match|mismatch\"}[5m])), 1)", "legendFormat": "mismatch"}], + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"steps": [{"color": "green", "value": null}, {"color": "red", "value": 0.001}]}}, "overrides": []}, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0} + }, + { + "id": 2, + "title": "Parity error rate", + "type": "timeseries", + "targets": [{"expr": "sum(rate(mreg_policy_parity_results_total{result=\"error\"}[5m])) / clamp_min(sum(rate(mreg_policy_parity_results_total[5m])), 1)", "legendFormat": "errors"}], + "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"steps": [{"color": "green", "value": null}, {"color": "red", "value": 0.001}]}}, "overrides": []}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0} + }, + { + "id": 3, + "title": "Outbox entries", + "type": "timeseries", + "targets": [{"expr": "max by (status) (mreg_policy_parity_outbox_entries)", "legendFormat": "{{status}}"}], + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8} + }, + { + "id": 4, + "title": "Oldest pending batch", + "type": "stat", + "targets": [{"expr": "max(mreg_policy_parity_outbox_oldest_seconds)"}], + "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"steps": [{"color": "green", "value": null}, {"color": "red", "value": 300}]}}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 8} + }, + { + "id": 5, + "title": "Delivery lifecycle", + "type": "timeseries", + "targets": [{"expr": "sum by (status) (rate(mreg_policy_parity_batches_total[5m]))", "legendFormat": "{{status}}"}], + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8} + } + ], + "schemaVersion": 41, + "tags": ["mreg", "treetop", "rollout"], + "templating": {"list": []}, + "time": {"from": "now-24h", "to": "now"}, + "title": "MREG TreeTop parity rollout", + "uid": "mreg-treetop-parity", + "version": 1 +} diff --git a/monitoring/treetop-alerts.yml b/monitoring/treetop-alerts.yml new file mode 100644 index 00000000..99eeef28 --- /dev/null +++ b/monitoring/treetop-alerts.yml @@ -0,0 +1,53 @@ +groups: + - name: mreg-treetop-rollout + rules: + - alert: MregTreeTopParityMismatchRateHigh + expr: | + sum(rate(mreg_policy_parity_results_total{result="mismatch"}[30m])) + / + clamp_min(sum(rate(mreg_policy_parity_results_total{result=~"match|mismatch"}[30m])), 1) + > 0.001 + for: 30m + labels: + severity: page + annotations: + summary: TreeTop parity mismatch rate exceeds 0.1% + - alert: MregTreeTopParityErrorRateHigh + expr: | + sum(rate(mreg_policy_parity_results_total{result="error"}[30m])) + / + clamp_min(sum(rate(mreg_policy_parity_results_total[30m])), 1) + > 0.001 + for: 15m + labels: + severity: page + annotations: + summary: TreeTop parity error rate exceeds 0.1% + - alert: MregTreeTopParityPersistenceFailure + expr: increase(mreg_policy_parity_batches_total{status="persist_failed"}[5m]) > 0 + for: 0m + labels: + severity: page + annotations: + summary: A policy parity batch could not be persisted + - alert: MregTreeTopParityDeadLetter + expr: max(mreg_policy_parity_outbox_entries{status="dead_letter"}) > 0 + for: 0m + labels: + severity: page + annotations: + summary: The policy parity outbox contains dead letters + - alert: MregTreeTopParityBacklogStale + expr: max(mreg_policy_parity_outbox_oldest_seconds) > 300 + for: 10m + labels: + severity: page + annotations: + summary: The oldest policy parity batch is over five minutes old + - alert: MregTreeTopParityCircuitOpen + expr: max(mreg_policy_parity_circuit_open) > 0 + for: 2m + labels: + severity: ticket + annotations: + summary: A policy parity delivery circuit breaker is open diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index 674c2d77..ba0f11d6 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -1,9 +1,7 @@ from __future__ import annotations import ipaddress -import re from collections.abc import Mapping -from django.db import models from typing import TYPE_CHECKING, Any from rest_framework import exceptions from rest_framework.permissions import IsAuthenticated as DRFIsAuthenticated, SAFE_METHODS @@ -18,6 +16,15 @@ from mreg.models.auth import User, MregAdminGroup from mreg.api.treetop import PolicyCheck, PolicyResource, policy_parity +from mreg.policy.contracts import MEMBERSHIP_ACTIONS, snake_case +from mreg.policy.resources import ( + adapter_for_kind, + crud_operation_from_method, + policy_action_from_view, + resource_id_from_view, + resource_kind_from_view, + stringify_attribute, +) # NOTE: We _must_ import `rest_framework.generics` in an `if TYPE_CHECKING:` # block because DRF does some dynamic import shenanigans on runtime using @@ -36,48 +43,25 @@ class ParityMixin: """Translate legacy permission results into explicit policy contracts.""" - _CRUD_METHOD_TO_OPERATION = { - "GET": "read", - "HEAD": "read", - "OPTIONS": "read", - "POST": "create", - "PUT": "update", - "PATCH": "update", - "DELETE": "delete", - } - _IDENTIFIER_FIELDS = ("pk", "id", "name") - _VIEW_IDENTIFIER_FIELDS = ("pk", "id", "name", "cpk", "hostpk", "network") _MEMBERSHIP_ACTIONS = { - MregAdminGroup.SUPERUSER: "superuser_access", - MregAdminGroup.ADMINUSER: "admin_access", - MregAdminGroup.GROUP_ADMIN: "hostgroup_admin_access", - MregAdminGroup.NETWORK_ADMIN: "network_admin_access", - MregAdminGroup.DNS_WILDCARD: "dns_wildcard_admin_access", - MregAdminGroup.DNS_UNDERSCORE: "dns_underscore_admin_access", - MregAdminGroup.HOSTPOLICY_ADMIN: "hostpolicy_admin_access", + MregAdminGroup.SUPERUSER: MEMBERSHIP_ACTIONS["superuser"], + MregAdminGroup.ADMINUSER: MEMBERSHIP_ACTIONS["admin"], + MregAdminGroup.GROUP_ADMIN: MEMBERSHIP_ACTIONS["group_admin"], + MregAdminGroup.NETWORK_ADMIN: MEMBERSHIP_ACTIONS["network_admin"], + MregAdminGroup.DNS_WILDCARD: MEMBERSHIP_ACTIONS["dns_wildcard"], + MregAdminGroup.DNS_UNDERSCORE: MEMBERSHIP_ACTIONS["dns_underscore"], + MregAdminGroup.HOSTPOLICY_ADMIN: MEMBERSHIP_ACTIONS["hostpolicy_admin"], } @staticmethod def _stringify_attr_value(value: Any) -> str: """Convert attribute values to strings for TreeTop resource attributes.""" - return "" if value is None else str(value) + return stringify_attribute(value) @staticmethod def _snake_case(value: str) -> str: """Normalize model/resource names to snake_case action/resource tokens.""" - if value.startswith("BACnet"): - value = f"Bacnet{value[len('BACnet') :]}" - value = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", value) - value = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value) - value = value.replace("-", "_") - value = re.sub(r"[^a-zA-Z0-9_]+", "_", value).strip("_").lower() - return value or "generic" - - @staticmethod - def _resource_name_from_model(model: Any) -> str | None: - """Return a model class name if available, otherwise None.""" - name = getattr(model, "__name__", None) - return str(name) if name else None + return snake_case(value) def _resource_kind_from_view( self, @@ -86,46 +70,7 @@ def _resource_kind_from_view( validated_serializer: "Serializer | None" = None, obj: Any = None, ) -> str: - """Resolve a resource kind from a concrete object or serializer model. - - View-name guessing is deliberately rejected: renaming a view must not - silently alter authorization action names. - """ - if obj is not None: - return obj.__class__.__name__ - - if validated_serializer is not None: - serializer_model = self._resource_name_from_model(getattr(getattr(validated_serializer, "Meta", None), "model", None)) - if serializer_model: - return serializer_model - - if validated_serializer is not None: - instance = getattr(validated_serializer, "instance", None) - if instance is not None: - return instance.__class__.__name__ - - explicit_kind = getattr(view, "policy_resource_kind", None) - if isinstance(explicit_kind, str) and explicit_kind.strip(): - return explicit_kind - - try: - serializer_class = view.get_serializer_class() - except (AttributeError, TypeError) as exc: - raise ValueError(f"{view.__class__.__name__} must declare an explicit policy resource kind") from exc - view_model = self._resource_name_from_model(getattr(getattr(serializer_class, "Meta", None), "model", None)) - if view_model: - return view_model - raise ValueError(f"{view.__class__.__name__} serializer must declare Meta.model for policy parity") - - @classmethod - def _identifier_from(cls, source: Any, fields: tuple[str, ...]) -> str | None: - if source is None: - return None - for field_name in fields: - value = source.get(field_name) if isinstance(source, Mapping) else getattr(source, field_name, None) - if value is not None: - return str(value) - return None + return resource_kind_from_view(view=view, validated_serializer=validated_serializer, obj=obj) def _resource_id_from_view( self, @@ -136,25 +81,26 @@ def _resource_id_from_view( data: Mapping[str, Any] | None = None, default: str = "any", ) -> str: - """Resolve a stable resource identifier for parity logging/evaluation.""" - serializer_instance = getattr(validated_serializer, "instance", None) - candidates = ( - self._identifier_from(obj, self._IDENTIFIER_FIELDS), - self._identifier_from(data, self._IDENTIFIER_FIELDS), - self._identifier_from(serializer_instance, self._IDENTIFIER_FIELDS), - self._identifier_from(getattr(view, "kwargs", None), self._VIEW_IDENTIFIER_FIELDS), + """Resolve a stable resource identifier through its registered adapter.""" + kind = self._resource_kind_from_view(view=view, validated_serializer=validated_serializer, obj=obj) + return resource_id_from_view( + view=view, + kind=kind, + validated_serializer=validated_serializer, + obj=obj, + data=data, + default=default, ) - return next((value for value in candidates if value is not None), default) def _crud_operation_from_method(self, method: str) -> str: """Map an HTTP method to a CRUD operation token.""" - try: - return self._CRUD_METHOD_TO_OPERATION[method.upper()] - except KeyError as exc: - raise ValueError(f"Unsupported HTTP method for policy parity: {method}") from exc + return crud_operation_from_method(method) def _crud_action(self, resource_kind: str, operation: str) -> str: """Build a policy action name like `_`.""" + contract = adapter_for_kind(resource_kind).contract + if operation not in contract.operations: + raise ValueError(f"{resource_kind} does not declare the {operation} policy operation") return f"{self._snake_case(resource_kind)}_{operation}" def _policy_action_from_view( @@ -165,12 +111,7 @@ def _policy_action_from_view( operation: str, ) -> str: """Resolve an explicit custom action or the model-backed CRUD action.""" - explicit_actions = getattr(view, "policy_actions", None) - if isinstance(explicit_actions, Mapping): - explicit_action = explicit_actions.get(operation) - if isinstance(explicit_action, str) and explicit_action.strip(): - return explicit_action - return self._crud_action(resource_kind, operation) + return policy_action_from_view(view=view, resource_kind=resource_kind, operation=operation) def _normalize_resource_attrs( self, @@ -179,10 +120,7 @@ def _normalize_resource_attrs( attrs: Mapping[str, Any] | None, ) -> dict[str, str]: """Normalize resource attributes to string values with a canonical kind.""" - normalized = {str(key): self._stringify_attr_value(value) for key, value in (attrs or {}).items()} - # Callers cannot override the resource kind through request data. - normalized["kind"] = self._snake_case(resource_kind) - return normalized + return adapter_for_kind(resource_kind).attributes(attrs) def pp( self, @@ -577,16 +515,9 @@ def has_obj_perm( resource_id=resource_id, ) - def _flatten_policy_attrs(self, data: Mapping[str, Any]) -> dict[str, str]: - """Flatten one model level into scalar attributes for policy parity.""" - attrs: dict[str, str] = {} - for key, value in data.items(): - if isinstance(value, models.Model): - for field in value._meta.fields: - attrs[f"{key}_{field.name}"] = self._stringify_attr_value(getattr(value, field.name, "")) - else: - attrs[key] = self._stringify_attr_value(value) - return attrs + def _flatten_policy_attrs(self, data: Mapping[str, Any], *, resource_kind: str) -> dict[str, str]: + """Adapt serializer data through the resource's registered adapter.""" + return adapter_for_kind(resource_kind).attributes(data) def _has_create_target_permission( self, @@ -701,7 +632,7 @@ def has_create_permission(self, request, view, validated_serializer): validated_serializer=validated_serializer, data=data, ) - attrs = self._flatten_policy_attrs(data) + attrs = self._flatten_policy_attrs(data, resource_kind=resource_kind) ip_value = data.get("ipaddress") # First check if we are asking for a restricted name. diff --git a/mreg/api/tests/test_metrics.py b/mreg/api/tests/test_metrics.py index 45b763bd..d5912b27 100644 --- a/mreg/api/tests/test_metrics.py +++ b/mreg/api/tests/test_metrics.py @@ -1,4 +1,6 @@ import ldap +import os +from tempfile import TemporaryDirectory from unittest.mock import Mock, patch from rest_framework.test import APIClient @@ -12,6 +14,19 @@ class MetricsTests(TestCase): + @patch("mreg.api.views.multiprocess.MultiProcessCollector") + @patch("mreg.api.views.generate_latest", return_value=b"# multiprocess metrics\n") + def test_metrics_endpoint_aggregates_gunicorn_workers(self, generate_latest_mock, collector_mock) -> None: + with TemporaryDirectory() as metrics_dir: + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": metrics_dir}): + response: Any = APIClient().get("/api/meta/metrics") + + assert response.status_code == 200 + assert response.content == b"# multiprocess metrics\n" + registry = collector_mock.call_args.args[0] + collector_mock.assert_called_once_with(registry, path=metrics_dir) + generate_latest_mock.assert_called_once_with(registry) + def test_metrics_endpoint_exposes_prometheus_metrics(self) -> None: """Test that metrics endpoint returns Prometheus-formatted output.""" client = APIClient() diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index e98abb46..5b2cfa34 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -5,18 +5,20 @@ import ipaddress import logging import os -import queue import threading from collections.abc import Mapping, Sequence from contextlib import contextmanager, suppress from contextvars import ContextVar from dataclasses import dataclass, field +from datetime import timedelta from time import monotonic -from typing import Final from django.conf import settings +from django.db import close_old_connections, transaction +from django.db.models import Q +from django.utils import timezone from django.views import View -from prometheus_client import Counter, Histogram +from prometheus_client import Counter, Gauge, Histogram from rest_framework.request import Request import structlog from treetop_client.client import TreeTopClient @@ -31,6 +33,7 @@ ) from mreg.models.auth import User as MregUser +from mreg.models.policy import PolicyParityOutbox logger = structlog.get_logger("mreg.policy.parity") @@ -39,8 +42,14 @@ POLICY_NAMESPACE = getattr(settings, "POLICY_NAMESPACE", ["MREG"]) POLICY_PARITY_BATCH_ENABLED = getattr(settings, "POLICY_PARITY_BATCH_ENABLED", True) POLICY_PARITY_LOG_DETAILS = getattr(settings, "POLICY_PARITY_LOG_DETAILS", False) -POLICY_PARITY_QUEUE_SIZE = getattr(settings, "POLICY_PARITY_QUEUE_SIZE", 100) POLICY_TIMEOUT_SECONDS = getattr(settings, "POLICY_TIMEOUT_SECONDS", 5.0) +POLICY_PARITY_MAX_ATTEMPTS = getattr(settings, "POLICY_PARITY_MAX_ATTEMPTS", 8) +POLICY_PARITY_RETRY_BASE_SECONDS = getattr(settings, "POLICY_PARITY_RETRY_BASE_SECONDS", 2.0) +POLICY_PARITY_RETRY_MAX_SECONDS = getattr(settings, "POLICY_PARITY_RETRY_MAX_SECONDS", 300.0) +POLICY_PARITY_LEASE_SECONDS = getattr(settings, "POLICY_PARITY_LEASE_SECONDS", 60.0) +POLICY_PARITY_POLL_SECONDS = getattr(settings, "POLICY_PARITY_POLL_SECONDS", 1.0) +POLICY_PARITY_CIRCUIT_FAILURES = getattr(settings, "POLICY_PARITY_CIRCUIT_FAILURES", 5) +POLICY_PARITY_CIRCUIT_RESET_SECONDS = getattr(settings, "POLICY_PARITY_CIRCUIT_RESET_SECONDS", 30.0) POLICY_DECISIONS_TOTAL = Counter( @@ -69,7 +78,7 @@ POLICY_PARITY_BATCHES_TOTAL = Counter( "mreg_policy_parity_batches_total", - "Policy parity batches submitted to or dropped by the background worker.", + "Durable policy parity batch lifecycle events.", ["status"], ) @@ -98,6 +107,25 @@ buckets=[0, 1, 2, 3, 5, 8], ) +POLICY_PARITY_OUTBOX_ENTRIES = Gauge( + "mreg_policy_parity_outbox_entries", + "Current durable policy parity outbox entries.", + ["status"], + multiprocess_mode="livemax", +) + +POLICY_PARITY_OUTBOX_OLDEST_SECONDS = Gauge( + "mreg_policy_parity_outbox_oldest_seconds", + "Age of the oldest pending durable policy parity batch.", + multiprocess_mode="livemax", +) + +POLICY_PARITY_CIRCUIT_OPEN = Gauge( + "mreg_policy_parity_circuit_open", + "Whether this worker's TreeTop delivery circuit breaker is open.", + multiprocess_mode="livemax", +) + @dataclass(frozen=True, slots=True) class PolicyResource: @@ -135,6 +163,86 @@ class _ParityBatchItem: context: dict[str, object] +def _serialize_policy_batch(items: Sequence[_ParityBatchItem]) -> dict[str, object]: + """Convert a batch into the versioned JSON outbox representation.""" + return { + "version": 1, + "items": [ + { + "decision": item.decision, + "policy_request": item.policy_request.to_api(), + "context": item.context, + } + for item in items + ], + } + + +def _deserialize_policy_request(payload: Mapping[str, object]) -> TreeTopRequest: + principal_payload = payload["principal"] + if not isinstance(principal_payload, Mapping): + raise ValueError("Invalid durable policy principal") + user_payload = principal_payload["User"] + if not isinstance(user_payload, Mapping): + raise ValueError("Invalid durable policy user") + namespace = [str(part) for part in user_payload.get("namespace", [])] + groups_payload = user_payload.get("groups", []) + groups = [str(group["id"]) for group in groups_payload if isinstance(group, Mapping)] + + action_payload = payload["action"] + resource_payload = payload["resource"] + if not isinstance(action_payload, Mapping) or not isinstance(resource_payload, Mapping): + raise ValueError("Invalid durable policy action or resource") + action = Action.new( + str(action_payload["id"]), + [str(part) for part in action_payload.get("namespace", [])], + ) + attrs_payload = resource_payload.get("attrs", {}) + if not isinstance(attrs_payload, Mapping): + raise ValueError("Invalid durable policy resource attributes") + attrs: dict[str, ResourceAttribute] = {} + for key, raw_attribute in attrs_payload.items(): + if not isinstance(raw_attribute, Mapping): + raise ValueError("Invalid durable policy resource attribute") + attrs[str(key)] = ResourceAttribute.new( + str(raw_attribute["value"]), + ResourceAttributeType(str(raw_attribute["type"])), + ) + return TreeTopRequest( + principal=TreeTopUser.new(str(user_payload["id"]), namespace, groups=groups), + action=action, + resource=TreeTopResource.new( + kind=str(resource_payload["kind"]), + id=str(resource_payload["id"]), + attrs=attrs, + ), + ) + + +def _deserialize_policy_batch(payload: Mapping[str, object]) -> list[_ParityBatchItem]: + if payload.get("version") != 1: + raise ValueError(f"Unsupported policy outbox payload version: {payload.get('version')}") + raw_items = payload.get("items") + if not isinstance(raw_items, list): + raise ValueError("Invalid durable policy batch") + items: list[_ParityBatchItem] = [] + for raw_item in raw_items: + if not isinstance(raw_item, Mapping): + raise ValueError("Invalid durable policy batch item") + request_payload = raw_item.get("policy_request") + context = raw_item.get("context") + if not isinstance(request_payload, Mapping) or not isinstance(context, Mapping): + raise ValueError("Invalid durable policy batch request or context") + items.append( + _ParityBatchItem( + decision=bool(raw_item.get("decision")), + policy_request=_deserialize_policy_request(request_payload), + context={str(key): value for key, value in context.items()}, + ) + ) + return items + + @dataclass(slots=True) class _RequestParityState: items: list[_ParityBatchItem] = field(default_factory=list) @@ -188,67 +296,225 @@ def _close_treetop_client() -> None: client.close() -class _ParityDispatcher: - """Bounded, process-local worker that keeps parity I/O off request threads.""" +@dataclass(frozen=True, slots=True) +class _ClaimedPolicyBatch: + id: int + attempts: int + payload: Mapping[str, object] + + +class _CircuitBreaker: + """Small process-local breaker protecting the shared TreeTop service.""" + + def __init__(self, failure_threshold: int, reset_seconds: float) -> None: + self.failure_threshold = max(1, failure_threshold) + self.reset_seconds = max(0.1, reset_seconds) + self.consecutive_failures = 0 + self.open_until = 0.0 + + def wait_seconds(self) -> float: + remaining = self.open_until - monotonic() + if remaining <= 0: + POLICY_PARITY_CIRCUIT_OPEN.set(0) + return 0.0 + POLICY_PARITY_CIRCUIT_OPEN.set(1) + return remaining + + def success(self) -> None: + self.consecutive_failures = 0 + self.open_until = 0.0 + POLICY_PARITY_CIRCUIT_OPEN.set(0) + + def failure(self) -> None: + self.consecutive_failures += 1 + if self.consecutive_failures >= self.failure_threshold: + self.open_until = monotonic() + self.reset_seconds + POLICY_PARITY_CIRCUIT_OPEN.set(1) + _safe_log( + logging.ERROR, + "policy_parity_circuit_open", + reset_seconds=self.reset_seconds, + consecutive_failures=self.consecutive_failures, + ) + + +def _refresh_outbox_metrics() -> None: + """Refresh low-cardinality gauges from the durable shared queue.""" + try: + now = timezone.now() + pending = PolicyParityOutbox.objects.filter(failed_at__isnull=True) + POLICY_PARITY_OUTBOX_ENTRIES.labels(status="pending").set(pending.count()) + POLICY_PARITY_OUTBOX_ENTRIES.labels(status="dead_letter").set( + PolicyParityOutbox.objects.filter(failed_at__isnull=False).count() + ) + oldest = pending.order_by("created_at").values_list("created_at", flat=True).first() + POLICY_PARITY_OUTBOX_OLDEST_SECONDS.set(max(0.0, (now - oldest).total_seconds()) if oldest else 0.0) + except Exception as exc: + _record_instrumentation_failure(exc, stage="outbox_metrics") - _STOP: Final = object() - def __init__(self, max_queue_size: int) -> None: - self._queue: queue.Queue[list[_ParityBatchItem] | object] = queue.Queue(maxsize=max(1, max_queue_size)) +class _ParityDispatcher: + """Database-outbox worker shared safely by all application processes.""" + + def __init__(self) -> None: self._thread: threading.Thread | None = None self._start_lock = threading.Lock() + self._wake = threading.Event() + self._stop = threading.Event() + self._circuit = _CircuitBreaker( + int(POLICY_PARITY_CIRCUIT_FAILURES), + float(POLICY_PARITY_CIRCUIT_RESET_SECONDS), + ) def submit(self, items: Sequence[_ParityBatchItem]) -> bool: + """Persist a batch before waking a worker; no policy I/O occurs here.""" if not items: return False self._ensure_started() - try: - self._queue.put_nowait(list(items)) - except queue.Full: - with suppress(Exception): - POLICY_PARITY_BATCHES_TOTAL.labels(status="dropped").inc() - _safe_log( - logging.ERROR, - "policy_parity_queue_full", - queue_size=self._queue.maxsize, - batch_size=len(items), - ) - return False - with suppress(Exception): - POLICY_PARITY_BATCHES_TOTAL.labels(status="submitted").inc() + PolicyParityOutbox.objects.create(payload=_serialize_policy_batch(items)) + POLICY_PARITY_BATCHES_TOTAL.labels(status="persisted").inc() + transaction.on_commit(self.wake) + _refresh_outbox_metrics() return True + def wake(self) -> None: + self._wake.set() + def _ensure_started(self) -> None: if self._thread is not None and self._thread.is_alive(): return with self._start_lock: if self._thread is None or not self._thread.is_alive(): + self._stop.clear() self._thread = threading.Thread( target=self._run, - name="mreg-policy-parity", + name="mreg-policy-parity-outbox", daemon=True, ) self._thread.start() + def _claim(self) -> _ClaimedPolicyBatch | None: + now = timezone.now() + stale_before = now - timedelta(seconds=float(POLICY_PARITY_LEASE_SECONDS)) + with transaction.atomic(): + row = ( + PolicyParityOutbox.objects.select_for_update(skip_locked=True) + .filter(failed_at__isnull=True, available_at__lte=now) + .filter(Q(locked_at__isnull=True) | Q(locked_at__lt=stale_before)) + .order_by("available_at", "id") + .first() + ) + if row is None: + return None + row.attempts += 1 + row.locked_at = now + row.save(update_fields=("attempts", "locked_at")) + return _ClaimedPolicyBatch(id=row.id, attempts=row.attempts, payload=row.payload) + + def _complete(self, claimed: _ClaimedPolicyBatch) -> None: + PolicyParityOutbox.objects.filter(id=claimed.id).delete() + POLICY_PARITY_BATCHES_TOTAL.labels(status="processed").inc() + self._circuit.success() + + def _fail( + self, + claimed: _ClaimedPolicyBatch, + exc: Exception, + items: Sequence[_ParityBatchItem], + ) -> None: + error = f"{type(exc).__name__}: {exc}" + self._circuit.failure() + now = timezone.now() + if claimed.attempts >= int(POLICY_PARITY_MAX_ATTEMPTS): + PolicyParityOutbox.objects.filter(id=claimed.id).update( + locked_at=None, + failed_at=now, + last_error=error, + ) + POLICY_PARITY_BATCHES_TOTAL.labels(status="dead_letter").inc() + for item in items: + _log_parity_payload( + _compute_parity_payload( + decision=item.decision, + policy_allowed=None, + error=error, + context=item.context, + ) + ) + _safe_log( + logging.ERROR, + "policy_parity_dead_letter", + outbox_id=claimed.id, + attempts=claimed.attempts, + error_type=type(exc).__name__, + ) + return + delay = min( + float(POLICY_PARITY_RETRY_MAX_SECONDS), + float(POLICY_PARITY_RETRY_BASE_SECONDS) * (2 ** (claimed.attempts - 1)), + ) + PolicyParityOutbox.objects.filter(id=claimed.id).update( + locked_at=None, + available_at=now + timedelta(seconds=delay), + last_error=error, + ) + POLICY_PARITY_BATCHES_TOTAL.labels(status="retried").inc() + _safe_log( + logging.WARNING, + "policy_parity_retry_scheduled", + outbox_id=claimed.id, + attempts=claimed.attempts, + delay_seconds=delay, + error_type=type(exc).__name__, + ) + def _run(self) -> None: - while True: - items = self._queue.get() - try: - if items is self._STOP: - return - _process_policy_parity_batch(items) - except Exception as exc: - _record_instrumentation_failure(exc, stage="worker") - finally: - self._queue.task_done() + close_old_connections() + try: + while not self._stop.is_set(): + circuit_wait = self._circuit.wait_seconds() + if circuit_wait > 0: + self._wake.wait(timeout=min(circuit_wait, float(POLICY_PARITY_POLL_SECONDS))) + self._wake.clear() + continue + close_old_connections() + try: + claimed = self._claim() + except Exception as exc: + _record_instrumentation_failure(exc, stage="outbox_claim") + close_old_connections() + self._wake.wait(timeout=float(POLICY_PARITY_POLL_SECONDS)) + self._wake.clear() + continue + if claimed is None: + _refresh_outbox_metrics() + self._wake.wait(timeout=float(POLICY_PARITY_POLL_SECONDS)) + self._wake.clear() + continue + items: list[_ParityBatchItem] = [] + try: + items = _deserialize_policy_batch(claimed.payload) + _process_policy_parity_batch(items) + self._complete(claimed) + except Exception as exc: + _record_instrumentation_failure(exc, stage="worker") + try: + self._fail(claimed, exc, items) + except Exception as fail_exc: + _record_instrumentation_failure(fail_exc, stage="outbox_retry") + close_old_connections() + finally: + _refresh_outbox_metrics() + finally: + close_old_connections() def shutdown(self) -> None: thread = self._thread if thread is None or not thread.is_alive(): return - with suppress(queue.Full): - self._queue.put_nowait(self._STOP) - thread.join(timeout=1.0) + self._stop.set() + self._wake.set() + thread.join(timeout=max(1.0, float(POLICY_TIMEOUT_SECONDS) + 1.0)) _dispatcher: _ParityDispatcher | None = None @@ -262,14 +528,25 @@ def _get_dispatcher() -> _ParityDispatcher: pid = os.getpid() with _dispatcher_lock: if _dispatcher is None or _dispatcher_pid != pid: - _dispatcher = _ParityDispatcher(int(POLICY_PARITY_QUEUE_SIZE)) + _dispatcher = _ParityDispatcher() _dispatcher_pid = pid return _dispatcher -def _shutdown_policy_runtime() -> None: +def start_policy_parity_dispatcher() -> None: + """Start a post-fork outbox worker so persisted work survives restarts.""" + if POLICY_PARITY_ENABLED and POLICY_BASE_URL: + _get_dispatcher()._ensure_started() + + +def stop_policy_parity_dispatcher() -> None: + """Stop this process's outbox worker without affecting persisted work.""" if _dispatcher is not None and _dispatcher_pid == os.getpid(): _dispatcher.shutdown() + + +def _shutdown_policy_runtime() -> None: + stop_policy_parity_dispatcher() _close_treetop_client() @@ -297,13 +574,15 @@ def _submit_policy_batch(items: Sequence[_ParityBatchItem]) -> bool: try: return _get_dispatcher().submit(items) except Exception as exc: - _record_instrumentation_failure(exc, stage="submit") + with suppress(Exception): + POLICY_PARITY_BATCHES_TOTAL.labels(status="persist_failed").inc() + _record_instrumentation_failure(exc, stage="persist") return False @contextmanager def batch_policy_parity(): - """Collect one request's parity checks and enqueue them as one batch.""" + """Collect one request's parity checks and persist them as one batch.""" if _request_state.get() is not None: yield return @@ -490,7 +769,12 @@ def _authorize_with_metrics( def _process_policy_parity_batch(items: Sequence[_ParityBatchItem]) -> None: - """Evaluate and record one batch. This function runs outside request threads.""" + """Evaluate and record one durable batch outside request threads. + + Delivery-level errors raise so the outbox can retry them. Successful + responses are recorded only after a successful authorize call and before + deleting the durable row. Delivery is at-least-once across process crashes. + """ if not items: return @@ -501,11 +785,14 @@ def _process_policy_parity_batch(items: Sequence[_ParityBatchItem]) -> None: correlation_id=correlation_id if isinstance(correlation_id, str) else None, path=path if isinstance(path, str) else None, ) + if authorize_error is not None: + raise RuntimeError(authorize_error) + parsed_results = [_result_to_decision_and_error(results, index) for index in range(len(items))] + result_error = next((error for _, error in parsed_results if error is not None), None) + if result_error is not None: + raise RuntimeError(result_error) for index, item in enumerate(items): - if authorize_error is None: - policy_allowed, error = _result_to_decision_and_error(results, index) - else: - policy_allowed, error = None, authorize_error + policy_allowed, error = parsed_results[index] _log_parity_payload( _compute_parity_payload( decision=item.decision, @@ -517,7 +804,7 @@ def _process_policy_parity_batch(items: Sequence[_ParityBatchItem]) -> None: def flush_policy_parity_batch() -> bool: - """Enqueue and clear the current request batch, if one exists.""" + """Persist and clear the current request batch, if one exists.""" state = _request_state.get() if state is None or not state.items: return False diff --git a/mreg/api/views.py b/mreg/api/views.py index 75bd4718..7b3fb3de 100644 --- a/mreg/api/views.py +++ b/mreg/api/views.py @@ -1,3 +1,4 @@ +import os import platform import time from time import monotonic @@ -21,7 +22,14 @@ from rest_framework.views import APIView from django.http import HttpResponse from drf_spectacular.utils import OpenApiParameter, OpenApiTypes, extend_schema -from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest +from prometheus_client import ( + CONTENT_TYPE_LATEST, + CollectorRegistry, + Counter, + Histogram, + generate_latest, + multiprocess, +) from mreg.__about__ import __version__ as mreg_version from mreg.api.permissions import IsSuperOrNetworkAdminMember @@ -350,4 +358,11 @@ class MetricsView(APIView): responses={(status.HTTP_200_OK, "text/plain"): PROMETHEUS_METRICS_TEXT_SCHEMA}, ) def get(self, request: Request): - return HttpResponse(generate_latest(), content_type=CONTENT_TYPE_LATEST) + multiprocess_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR") + if multiprocess_dir: + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry, path=multiprocess_dir) + metrics = generate_latest(registry) + else: + metrics = generate_latest() + return HttpResponse(metrics, content_type=CONTENT_TYPE_LATEST) diff --git a/mreg/management/commands/check_policy_rollout.py b/mreg/management/commands/check_policy_rollout.py new file mode 100644 index 00000000..d3c77946 --- /dev/null +++ b/mreg/management/commands/check_policy_rollout.py @@ -0,0 +1,45 @@ +"""Fail unless TreeTop parity telemetry is ready for enforcement.""" + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError, CommandParser + +from mreg.policy.rollout import RolloutThresholds, evaluate_rollout, fetch_rollout_snapshot + + +class Command(BaseCommand): + help = "Check Prometheus parity signals against the TreeTop enforcement rollout gates" + + def add_arguments(self, parser: CommandParser) -> None: + parser.add_argument("--prometheus-url", required=True) + parser.add_argument("--window", default="24h") + parser.add_argument("--timeout", type=float, default=10.0) + + def handle(self, *args, **options): # type: ignore[no-untyped-def] + thresholds = RolloutThresholds( + min_comparisons=settings.POLICY_ROLLOUT_MIN_COMPARISONS, + max_mismatch_rate=settings.POLICY_ROLLOUT_MAX_MISMATCH_RATE, + max_error_rate=settings.POLICY_ROLLOUT_MAX_ERROR_RATE, + max_persist_failures=settings.POLICY_ROLLOUT_MAX_PERSIST_FAILURES, + max_dead_letters=settings.POLICY_ROLLOUT_MAX_DEAD_LETTERS, + max_backlog_age_seconds=settings.POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS, + ) + try: + snapshot = fetch_rollout_snapshot( + options["prometheus_url"], + window=options["window"], + timeout=options["timeout"], + ) + except Exception as exc: + raise CommandError(f"Unable to query Prometheus: {exc}") from exc + evaluation = evaluate_rollout(snapshot, thresholds) + summary = ( + f"comparisons={snapshot.comparisons:g} " + f"mismatch_rate={snapshot.mismatch_rate:.6f} " + f"error_rate={snapshot.error_rate:.6f} " + f"persist_failures={snapshot.persist_failures:g} " + f"dead_letters={snapshot.dead_letters:g} " + f"backlog_age_seconds={snapshot.backlog_age_seconds:g}" + ) + if not evaluation.ready: + raise CommandError(f"TreeTop rollout gate failed: {'; '.join(evaluation.reasons)} ({summary})") + self.stdout.write(self.style.SUCCESS(f"TreeTop rollout gate passed: {summary}")) diff --git a/mreg/middleware/metrics.py b/mreg/middleware/metrics.py index 55745baa..260ac21f 100644 --- a/mreg/middleware/metrics.py +++ b/mreg/middleware/metrics.py @@ -29,7 +29,12 @@ buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], ) -INPROGRESS = Gauge("mreg_http_inprogress_requests", "Inprogress requests", ["method", "path"]) +INPROGRESS = Gauge( + "mreg_http_inprogress_requests", + "Inprogress requests", + ["method", "path"], + multiprocess_mode="livesum", +) # Request/response sizes (bytes) REQUEST_SIZE = Histogram( diff --git a/mreg/migrations/0017_policyparityoutbox.py b/mreg/migrations/0017_policyparityoutbox.py new file mode 100644 index 00000000..2878427f --- /dev/null +++ b/mreg/migrations/0017_policyparityoutbox.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2 for MREG's durable policy parity outbox. + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("mreg", "0016_host_contacts"), + ] + + operations = [ + migrations.CreateModel( + name="PolicyParityOutbox", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("payload", models.JSONField()), + ("attempts", models.PositiveIntegerField(default=0)), + ("available_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ("locked_at", models.DateTimeField(blank=True, db_index=True, null=True)), + ("failed_at", models.DateTimeField(blank=True, db_index=True, null=True)), + ("last_error", models.TextField(blank=True, default="")), + ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)), + ], + options={ + "verbose_name_plural": "policy parity outbox entries", + "indexes": [ + models.Index( + fields=["failed_at", "available_at", "id"], + name="policy_outbox_ready_idx", + ) + ], + }, + ), + ] diff --git a/mreg/models/__init__.py b/mreg/models/__init__.py index b099b24b..3f132229 100644 --- a/mreg/models/__init__.py +++ b/mreg/models/__init__.py @@ -1,3 +1,4 @@ """Models for mreg.""" from .auth import User # noqa: F401, needed by mreg.settings for now +from .policy import PolicyParityOutbox # noqa: F401 diff --git a/mreg/models/policy.py b/mreg/models/policy.py new file mode 100644 index 00000000..db6c7525 --- /dev/null +++ b/mreg/models/policy.py @@ -0,0 +1,26 @@ +"""Durable policy parity delivery models.""" + +from django.db import models +from django.utils import timezone + + +class PolicyParityOutbox(models.Model): + """One durable, shared TreeTop parity batch. + + Successful rows are deleted. Rows that exhaust their retries remain as + dead letters so operators can inspect and explicitly resolve them. + """ + + payload = models.JSONField() + attempts = models.PositiveIntegerField(default=0) + available_at = models.DateTimeField(default=timezone.now, db_index=True) + locked_at = models.DateTimeField(null=True, blank=True, db_index=True) + failed_at = models.DateTimeField(null=True, blank=True, db_index=True) + last_error = models.TextField(blank=True, default="") + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + + class Meta: + indexes = [ + models.Index(fields=("failed_at", "available_at", "id"), name="policy_outbox_ready_idx"), + ] + verbose_name_plural = "policy parity outbox entries" diff --git a/mreg/policy/__init__.py b/mreg/policy/__init__.py new file mode 100644 index 00000000..173517a3 --- /dev/null +++ b/mreg/policy/__init__.py @@ -0,0 +1 @@ +"""Policy contracts, resource adapters, and rollout tooling.""" diff --git a/mreg/policy/contracts.py b/mreg/policy/contracts.py new file mode 100644 index 00000000..57624509 --- /dev/null +++ b/mreg/policy/contracts.py @@ -0,0 +1,174 @@ +"""Authoritative MREG policy resource and action contracts. + +This module deliberately has no Django dependencies. Runtime resource adapters +and the Cedar schema generator both consume these declarations so their view of +resource kinds, identifiers, attributes, and actions cannot drift. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +CRUD_OPERATIONS = ("create", "delete", "read", "update") + + +@dataclass(frozen=True) +class ResourceAttributeContract: + """One optional Cedar resource attribute.""" + + name: str + cedar_type: str = "String" + + +@dataclass(frozen=True) +class ResourceContract: + """Policy-facing resource metadata shared by Python and Cedar.""" + + kind: str + operations: tuple[str, ...] = () + attributes: tuple[ResourceAttributeContract, ...] = () + identifier_fields: tuple[str, ...] = ("pk", "id", "name", "cpk", "hostpk", "network") + + @property + def actions(self) -> tuple[str, ...]: + token = snake_case(self.kind) + return tuple(f"{token}_{operation}" for operation in self.operations) + + +def snake_case(value: str) -> str: + """Return the stable action token for a Python/Cedar resource name.""" + import re + + if value.startswith("BACnet"): + value = f"Bacnet{value[len('BACnet') :]}" + value = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", value) + value = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value) + value = value.replace("-", "_") + return re.sub(r"[^a-zA-Z0-9_]+", "_", value).strip("_").lower() or "generic" + + +HOST_ATTRIBUTES = tuple( + ResourceAttributeContract(name, cedar_type) + for name, cedar_type in ( + ("kind", "String"), + ("id", "String"), + ("name", "String"), + ("path", "String"), + ("hostname", "String"), + ("ip", "ipaddr"), + ("nameLabels", "Set"), + ) +) + + +RESOURCE_CONTRACTS = ( + ResourceContract("Generic"), + ResourceContract("Host", CRUD_OPERATIONS, HOST_ATTRIBUTES), + ResourceContract("HostContact", identifier_fields=("pk", "id", "email")), + ResourceContract( + "Ipaddress", + CRUD_OPERATIONS, + ( + ResourceAttributeContract("kind"), + ResourceAttributeContract("id"), + ResourceAttributeContract("ip", "ipaddr"), + ), + identifier_fields=("pk", "id", "ipaddress"), + ), + *(ResourceContract(kind, CRUD_OPERATIONS) for kind in ( + "Cname", + "Hinfo", + "Loc", + "Mx", + "Naptr", + "NameServer", + "PtrOverride", + "Sshfp", + "Srv", + "Txt", + "BACnetID", + "Community", + "HostCommunityMapping", + "Label", + "Network", + "NetworkPolicy", + "NetworkPolicyAttribute", + "NetworkPolicyAttributeValue", + )), +) + + +RESOURCE_CONTRACT_BY_KIND = {contract.kind: contract for contract in RESOURCE_CONTRACTS} + +MEMBERSHIP_ACTIONS = { + "superuser": "superuser_access", + "admin": "admin_access", + "group_admin": "hostgroup_admin_access", + "network_admin": "network_admin_access", + "dns_wildcard": "dns_wildcard_admin_access", + "dns_underscore": "dns_underscore_admin_access", + "hostpolicy_admin": "hostpolicy_admin_access", +} + +CUSTOM_ACTIONS = frozenset( + { + *MEMBERSHIP_ACTIONS.values(), + "create_label", + "delete_label", + "edit_label", + "host_contacts_read", + "ip_broadcast_management", + "ip_gw_management", + "ip_network_management", + "ip_reserved_management", + "ip_restricted_management", + "is_superuser", + "view_label", + } +) + +POLICY_ACTIONS = tuple( + sorted( + { + *CUSTOM_ACTIONS, + *(action for contract in RESOURCE_CONTRACTS for action in contract.actions), + } + ) +) + + +def render_cedar_schema() -> str: + """Render the deterministic human-readable Cedar schema.""" + lines = [ + "namespace MREG {", + " entity Group;", + " entity User in [Group];", + "", + ] + for contract in RESOURCE_CONTRACTS: + if contract.attributes: + lines.append(f" entity {contract.kind} = {{") + lines.extend( + f" {attribute.name}?: {attribute.cedar_type}," + for attribute in contract.attributes + ) + lines.append(" };") + else: + lines.append(f" entity {contract.kind};") + lines.extend(("", " action")) + for index, action in enumerate(POLICY_ACTIONS): + suffix = "," if index < len(POLICY_ACTIONS) - 1 else "" + lines.append(f' "{action}"{suffix}') + lines.extend( + ( + " appliesTo {", + " principal: User,", + " resource: [", + ) + ) + for index, contract in enumerate(RESOURCE_CONTRACTS): + suffix = "," if index < len(RESOURCE_CONTRACTS) - 1 else "" + lines.append(f" {contract.kind}{suffix}") + lines.extend((" ]", " };", "}", "")) + return "\n".join(lines) diff --git a/mreg/policy/resources.py b/mreg/policy/resources.py new file mode 100644 index 00000000..fa61f59a --- /dev/null +++ b/mreg/policy/resources.py @@ -0,0 +1,161 @@ +"""Typed adapters from Django/DRF objects to policy resource contracts.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, Protocol, TypeVar + +from mreg.policy.contracts import RESOURCE_CONTRACT_BY_KIND, ResourceContract, snake_case + + +SourceT = TypeVar("SourceT") + + +class ResourceAdapter(Protocol[SourceT]): + """Contract implemented by policy resource adapters.""" + + contract: ResourceContract + + def identifier(self, *sources: SourceT | Mapping[str, Any] | None, default: str = "any") -> str: ... + + def attributes(self, data: Mapping[str, Any] | None) -> dict[str, str]: ... + + +def stringify_attribute(value: Any) -> str: + """Convert an attribute value to the client wire representation.""" + return "" if value is None else str(value) + + +@dataclass(frozen=True, slots=True) +class ModelResourceAdapter: + """Default adapter for one explicitly registered resource kind.""" + + contract: ResourceContract + + def identifier(self, *sources: Any, default: str = "any") -> str: + for source in sources: + if source is None: + continue + for field_name in self.contract.identifier_fields: + value = source.get(field_name) if isinstance(source, Mapping) else getattr(source, field_name, None) + if value is not None: + return str(value) + return default + + def attributes(self, data: Mapping[str, Any] | None) -> dict[str, str]: + attrs: dict[str, str] = {} + for key, value in (data or {}).items(): + model_fields = getattr(getattr(value, "_meta", None), "fields", None) + if model_fields is not None: + for field in model_fields: + attrs[f"{key}_{field.name}"] = stringify_attribute(getattr(value, field.name, "")) + else: + attrs[str(key)] = stringify_attribute(value) + # Request data cannot spoof the policy resource kind. + attrs["kind"] = snake_case(self.contract.kind) + return attrs + + +@dataclass(frozen=True, slots=True) +class HostResourceAdapter(ModelResourceAdapter): + """Host adapter, named explicitly because host policy attributes are typed.""" + + +@dataclass(frozen=True, slots=True) +class IpaddressResourceAdapter(ModelResourceAdapter): + """IP-address adapter with its IP-oriented identifier precedence.""" + + +def _build_registry() -> dict[str, ModelResourceAdapter]: + registry: dict[str, ModelResourceAdapter] = {} + for kind, contract in RESOURCE_CONTRACT_BY_KIND.items(): + adapter_type: type[ModelResourceAdapter] + if kind == "Host": + adapter_type = HostResourceAdapter + elif kind == "Ipaddress": + adapter_type = IpaddressResourceAdapter + else: + adapter_type = ModelResourceAdapter + registry[kind] = adapter_type(contract) + return registry + + +RESOURCE_ADAPTERS = _build_registry() + + +def adapter_for_kind(kind: str) -> ModelResourceAdapter: + """Return the registered adapter; unknown resources must be explicit.""" + try: + return RESOURCE_ADAPTERS[kind] + except KeyError as exc: + raise ValueError(f"No policy resource adapter registered for {kind}") from exc + + +def resource_kind_from_view(*, view: Any, validated_serializer: Any = None, obj: Any = None) -> str: + """Resolve a registered kind without relying on a view class name.""" + candidates = ( + obj.__class__.__name__ if obj is not None else None, + getattr(getattr(getattr(validated_serializer, "Meta", None), "model", None), "__name__", None), + getattr(getattr(validated_serializer, "instance", None), "__class__", type(None)).__name__ + if getattr(validated_serializer, "instance", None) is not None + else None, + getattr(view, "policy_resource_kind", None), + ) + kind = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate.strip()), None) + if kind is None: + try: + serializer_class = view.get_serializer_class() + except (AttributeError, TypeError) as exc: + raise ValueError(f"{view.__class__.__name__} must declare an explicit policy resource kind") from exc + kind = getattr(getattr(getattr(serializer_class, "Meta", None), "model", None), "__name__", None) + if not kind: + raise ValueError(f"{view.__class__.__name__} serializer must declare Meta.model for policy parity") + adapter_for_kind(kind) + return kind + + +def resource_id_from_view( + *, + view: Any, + kind: str, + validated_serializer: Any = None, + obj: Any = None, + data: Mapping[str, Any] | None = None, + default: str = "any", +) -> str: + """Resolve a stable identifier with adapter-defined precedence.""" + adapter = adapter_for_kind(kind) + serializer_instance = getattr(validated_serializer, "instance", None) + return adapter.identifier(obj, data, serializer_instance, getattr(view, "kwargs", None), default=default) + + +CRUD_METHOD_TO_OPERATION = { + "GET": "read", + "HEAD": "read", + "OPTIONS": "read", + "POST": "create", + "PUT": "update", + "PATCH": "update", + "DELETE": "delete", +} + + +def crud_operation_from_method(method: str) -> str: + try: + return CRUD_METHOD_TO_OPERATION[method.upper()] + except KeyError as exc: + raise ValueError(f"Unsupported HTTP method for policy parity: {method}") from exc + + +def policy_action_from_view(*, view: Any, resource_kind: str, operation: str) -> str: + """Resolve an explicit custom action or a registered CRUD action.""" + explicit_actions = getattr(view, "policy_actions", None) + if isinstance(explicit_actions, Mapping): + explicit_action = explicit_actions.get(operation) + if isinstance(explicit_action, str) and explicit_action.strip(): + return explicit_action + contract = adapter_for_kind(resource_kind).contract + if operation not in contract.operations: + raise ValueError(f"{resource_kind} does not declare the {operation} policy operation") + return f"{snake_case(resource_kind)}_{operation}" diff --git a/mreg/policy/rollout.py b/mreg/policy/rollout.py new file mode 100644 index 00000000..00995c9c --- /dev/null +++ b/mreg/policy/rollout.py @@ -0,0 +1,97 @@ +"""Prometheus-backed TreeTop rollout readiness evaluation.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from urllib.parse import urlencode +from urllib.request import urlopen + + +@dataclass(frozen=True, slots=True) +class RolloutThresholds: + min_comparisons: int = 10_000 + max_mismatch_rate: float = 0.001 + max_error_rate: float = 0.001 + max_persist_failures: int = 0 + max_dead_letters: int = 0 + max_backlog_age_seconds: float = 300.0 + + +@dataclass(frozen=True, slots=True) +class RolloutSnapshot: + comparisons: float + mismatches: float + errors: float + persist_failures: float + dead_letters: float + backlog_age_seconds: float + + @property + def mismatch_rate(self) -> float: + return self.mismatches / self.comparisons if self.comparisons else 0.0 + + @property + def error_rate(self) -> float: + total = self.comparisons + self.errors + return self.errors / total if total else 0.0 + + +@dataclass(frozen=True, slots=True) +class RolloutEvaluation: + ready: bool + reasons: tuple[str, ...] + + +def evaluate_rollout(snapshot: RolloutSnapshot, thresholds: RolloutThresholds) -> RolloutEvaluation: + """Evaluate every rollout gate and return all failures at once.""" + reasons: list[str] = [] + if snapshot.comparisons < thresholds.min_comparisons: + reasons.append(f"comparisons {snapshot.comparisons:g} < {thresholds.min_comparisons}") + if snapshot.mismatch_rate > thresholds.max_mismatch_rate: + reasons.append(f"mismatch rate {snapshot.mismatch_rate:.6f} > {thresholds.max_mismatch_rate:.6f}") + if snapshot.error_rate > thresholds.max_error_rate: + reasons.append(f"error rate {snapshot.error_rate:.6f} > {thresholds.max_error_rate:.6f}") + if snapshot.persist_failures > thresholds.max_persist_failures: + reasons.append(f"persist failures {snapshot.persist_failures:g} > {thresholds.max_persist_failures}") + if snapshot.dead_letters > thresholds.max_dead_letters: + reasons.append(f"dead letters {snapshot.dead_letters:g} > {thresholds.max_dead_letters}") + if snapshot.backlog_age_seconds > thresholds.max_backlog_age_seconds: + reasons.append( + f"oldest backlog age {snapshot.backlog_age_seconds:g}s > {thresholds.max_backlog_age_seconds:g}s" + ) + return RolloutEvaluation(ready=not reasons, reasons=tuple(reasons)) + + +def _prometheus_value(base_url: str, query: str, timeout: float) -> float: + endpoint = f"{base_url.rstrip('/')}/api/v1/query?{urlencode({'query': query})}" + with urlopen(endpoint, timeout=timeout) as response: # noqa: S310 - operator-provided Prometheus URL + payload = json.load(response) + if payload.get("status") != "success": + raise RuntimeError(f"Prometheus query failed: {payload}") + results = payload.get("data", {}).get("result", []) + if not results: + return 0.0 + return float(results[0]["value"][1]) + + +def fetch_rollout_snapshot( + prometheus_url: str, + *, + window: str = "24h", + timeout: float = 10.0, +) -> RolloutSnapshot: + """Read the six low-cardinality signals required by the rollout gate.""" + queries = { + "comparisons": f'sum(increase(mreg_policy_parity_results_total{{result=~"match|mismatch"}}[{window}]))', + "mismatches": f'sum(increase(mreg_policy_parity_results_total{{result="mismatch"}}[{window}]))', + "errors": f'sum(increase(mreg_policy_parity_results_total{{result="error"}}[{window}]))', + "persist_failures": f'sum(increase(mreg_policy_parity_batches_total{{status="persist_failed"}}[{window}]))', + "dead_letters": 'max(mreg_policy_parity_outbox_entries{status="dead_letter"})', + "backlog_age_seconds": "max(mreg_policy_parity_outbox_oldest_seconds)", + } + values = { + name: _prometheus_value(prometheus_url, query, timeout) + for name, query in queries.items() + } + return RolloutSnapshot(**values) diff --git a/mreg/tests/test_gunicorn_conf.py b/mreg/tests/test_gunicorn_conf.py new file mode 100644 index 00000000..4a66ebdc --- /dev/null +++ b/mreg/tests/test_gunicorn_conf.py @@ -0,0 +1,34 @@ +"""Tests for worker-scoped Gunicorn lifecycle hooks.""" + +import os +from types import SimpleNamespace +from unittest.mock import patch + +from django.test import SimpleTestCase + +from mregsite import gunicorn_conf + + +class GunicornLifecycleHookTests(SimpleTestCase): + """Ensure parity dispatchers follow each Gunicorn worker lifecycle.""" + + @patch("mreg.api.treetop.start_policy_parity_dispatcher") + def test_post_fork_starts_dispatcher(self, start_dispatcher): + gunicorn_conf.post_fork(None, None) + + start_dispatcher.assert_called_once_with() + + @patch("mreg.api.treetop.stop_policy_parity_dispatcher") + def test_worker_exit_stops_dispatcher(self, stop_dispatcher): + gunicorn_conf.worker_exit(None, None) + + stop_dispatcher.assert_called_once_with() + + @patch("prometheus_client.multiprocess.mark_process_dead") + @patch("mreg.api.treetop.stop_policy_parity_dispatcher") + def test_worker_exit_marks_prometheus_process_dead(self, stop_dispatcher, mark_process_dead): + with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": "/tmp/prometheus"}): + gunicorn_conf.worker_exit(None, SimpleNamespace(pid=42)) + + stop_dispatcher.assert_called_once_with() + mark_process_dead.assert_called_once_with(42) diff --git a/mreg/tests/test_policy_contracts.py b/mreg/tests/test_policy_contracts.py index 724d29ed..cfd7d73b 100644 --- a/mreg/tests/test_policy_contracts.py +++ b/mreg/tests/test_policy_contracts.py @@ -6,13 +6,13 @@ from mreg.api.treetop import PolicyCheck, PolicyResource -class _ExampleModel: +class Host: pass class _ModelSerializer: class Meta: - model = _ExampleModel + model = Host class _ModelView: @@ -30,16 +30,16 @@ def setUp(self): def test_resource_kind_uses_serializer_model(self): self.assertEqual( self.mixin._resource_kind_from_view(view=_ModelView()), - "_ExampleModel", + "Host", ) def test_resource_kind_supports_explicit_non_model_contract(self): - view = mock.Mock(policy_resource_kind="HealthCheck") + view = mock.Mock(policy_resource_kind="Generic") view.get_serializer_class.side_effect = AttributeError self.assertEqual( self.mixin._resource_kind_from_view(view=view), - "HealthCheck", + "Generic", ) def test_resource_kind_does_not_guess_from_view_name(self): @@ -52,7 +52,8 @@ def get_serializer_class(): self.mixin._resource_kind_from_view(view=ReportList()) def test_resource_id_has_stable_precedence(self): - obj = mock.Mock(pk=7) + obj = Host() + obj.pk = 7 self.assertEqual( self.mixin._resource_id_from_view( diff --git a/mreg/tests/test_policy_rollout.py b/mreg/tests/test_policy_rollout.py new file mode 100644 index 00000000..bb7eeb1a --- /dev/null +++ b/mreg/tests/test_policy_rollout.py @@ -0,0 +1,38 @@ +from django.test import SimpleTestCase + +from mreg.policy.rollout import RolloutSnapshot, RolloutThresholds, evaluate_rollout + + +class PolicyRolloutTests(SimpleTestCase): + def test_ready_snapshot_passes_every_gate(self) -> None: + result = evaluate_rollout( + RolloutSnapshot( + comparisons=20_000, + mismatches=1, + errors=1, + persist_failures=0, + dead_letters=0, + backlog_age_seconds=10, + ), + RolloutThresholds(), + ) + + self.assertTrue(result.ready) + self.assertEqual(result.reasons, ()) + + def test_failed_snapshot_reports_every_broken_gate(self) -> None: + result = evaluate_rollout( + RolloutSnapshot( + comparisons=100, + mismatches=5, + errors=5, + persist_failures=2, + dead_letters=3, + backlog_age_seconds=600, + ), + RolloutThresholds(), + ) + + self.assertFalse(result.ready) + self.assertEqual(len(result.reasons), 6) + self.assertIn("comparisons", result.reasons[0]) diff --git a/mreg/tests/test_treetop_batching.py b/mreg/tests/test_treetop_batching.py index daed95be..36cbbabf 100644 --- a/mreg/tests/test_treetop_batching.py +++ b/mreg/tests/test_treetop_batching.py @@ -5,16 +5,18 @@ from unittest.mock import Mock, patch from django.http import HttpRequest, HttpResponse -from django.test import SimpleTestCase +from django.test import TestCase from mreg.api.treetop import ( PolicyCheck, PolicyResource, + _CircuitBreaker, _ParityBatchItem, _ParityDispatcher, _build_policy_request, _build_resource_attrs, _compute_parity_payload, + _deserialize_policy_batch, _fully_qualified_action, _is_parity_enabled, _process_policy_parity_batch, @@ -22,6 +24,7 @@ _request_state, _result_to_decision_and_error, _safe_log, + _serialize_policy_batch, batch_policy_parity, disable_policy_parity, flush_policy_parity_batch, @@ -54,7 +57,7 @@ def __init__(self, decisions: list[bool]) -> None: self.results = [_DummyAuthorizeResult(decision) for decision in decisions] -class TreeTopParityBatchingTests(SimpleTestCase): +class TreeTopParityBatchingTests(TestCase): @staticmethod def _request() -> HttpRequest: request = HttpRequest() @@ -370,17 +373,99 @@ def test_worker_records_authorize_exceptions_for_every_item( _ParityBatchItem(False, {"request": "two"}, {}), ] - _process_policy_parity_batch(items) + with self.assertRaisesRegex(RuntimeError, "offline"): + _process_policy_parity_batch(items) - self.assertEqual(log_payload.call_count, 2) - self.assertIn("offline", log_payload.call_args_list[0].args[0]["error"]) + log_payload.assert_not_called() - def test_bounded_dispatcher_drops_when_queue_is_full(self) -> None: - dispatcher = _ParityDispatcher(max_queue_size=1) - item = _ParityBatchItem(True, {}, {}) + def test_dispatcher_persists_batches_in_the_shared_outbox(self) -> None: + dispatcher = _ParityDispatcher() + request = _build_policy_request( + SimpleNamespace(username="tester", group_list=[]), + self._check(), + ) + item = _ParityBatchItem(True, request, {"path": "/one"}) with patch.object(dispatcher, "_ensure_started"): self.assertTrue(dispatcher.submit([item])) - self.assertFalse(dispatcher.submit([item])) + + from mreg.models.policy import PolicyParityOutbox + + row = PolicyParityOutbox.objects.get() + self.assertEqual(row.payload["version"], 1) + self.assertEqual(row.payload["items"][0]["context"]["path"], "/one") + + def test_durable_batch_round_trip_preserves_typed_requests(self) -> None: + request = _build_policy_request( + SimpleNamespace(username="tester", group_list=["admins"]), + self._check(), + ) + items = [_ParityBatchItem(False, request, {"correlation_id": "cid"})] + + restored = _deserialize_policy_batch(_serialize_policy_batch(items)) + + self.assertEqual(len(restored), 1) + self.assertFalse(restored[0].decision) + self.assertEqual(restored[0].context, {"correlation_id": "cid"}) + self.assertEqual(restored[0].policy_request.to_api(), request.to_api()) + + def test_dispatcher_claims_and_completes_a_durable_batch(self) -> None: + from mreg.models.policy import PolicyParityOutbox + + request = _build_policy_request( + SimpleNamespace(username="tester", group_list=[]), + self._check(), + ) + row = PolicyParityOutbox.objects.create( + payload=_serialize_policy_batch([_ParityBatchItem(True, request, {})]), + ) + dispatcher = _ParityDispatcher() + + claimed = dispatcher._claim() + + self.assertIsNotNone(claimed) + assert claimed is not None + self.assertEqual(claimed.id, row.id) + self.assertEqual(claimed.attempts, 1) + dispatcher._complete(claimed) + self.assertFalse(PolicyParityOutbox.objects.filter(id=row.id).exists()) + + def test_dispatcher_retries_then_retains_a_dead_letter(self) -> None: + from mreg.models.policy import PolicyParityOutbox + + request = _build_policy_request( + SimpleNamespace(username="tester", group_list=[]), + self._check(), + ) + item = _ParityBatchItem(True, request, {}) + row = PolicyParityOutbox.objects.create(payload=_serialize_policy_batch([item])) + dispatcher = _ParityDispatcher() + claimed = dispatcher._claim() + assert claimed is not None + + with patch("mreg.api.treetop.POLICY_PARITY_MAX_ATTEMPTS", 2): + dispatcher._fail(claimed, RuntimeError("offline"), [item]) + row.refresh_from_db() + self.assertIsNone(row.failed_at) + self.assertIsNone(row.locked_at) + + row.available_at = row.created_at + row.save(update_fields=("available_at",)) + claimed = dispatcher._claim() + assert claimed is not None + dispatcher._fail(claimed, RuntimeError("still offline"), [item]) + + row.refresh_from_db() + self.assertIsNotNone(row.failed_at) + self.assertIn("still offline", row.last_error) + + def test_circuit_breaker_opens_and_recovers(self) -> None: + circuit = _CircuitBreaker(failure_threshold=2, reset_seconds=30) + circuit.failure() + self.assertEqual(circuit.wait_seconds(), 0) + circuit.failure() + self.assertGreater(circuit.wait_seconds(), 0) + circuit.success() + self.assertEqual(circuit.wait_seconds(), 0) @patch("mreg.api.treetop.MregUser.from_request") def test_logging_middleware_only_enqueues_policy_work(self, mock_from_request: Mock) -> None: diff --git a/mregsite/gunicorn_conf.py b/mregsite/gunicorn_conf.py new file mode 100644 index 00000000..13089a00 --- /dev/null +++ b/mregsite/gunicorn_conf.py @@ -0,0 +1,40 @@ +"""Gunicorn lifecycle hooks for process-local runtime resources.""" + +import os + + +def _setup_django() -> None: + """Initialize Django before Gunicorn loads worker-scoped integrations.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mregsite.settings") + + import django + from django.apps import apps + + if not apps.ready: + django.setup() + + +def post_fork(server, worker): # noqa: ARG001 + """Start the durable parity outbox consumer only after worker fork.""" + _setup_django() + + from mreg.api.treetop import start_policy_parity_dispatcher + + start_policy_parity_dispatcher() + + +def worker_exit(server, worker): # noqa: ARG001 + """Stop the worker thread; unprocessed rows remain durable in PostgreSQL.""" + from django.apps import apps + + if not apps.ready: + return + + from mreg.api.treetop import stop_policy_parity_dispatcher + + stop_policy_parity_dispatcher() + + if os.environ.get("PROMETHEUS_MULTIPROC_DIR"): + from prometheus_client import multiprocess + + multiprocess.mark_process_dead(worker.pid) diff --git a/mregsite/settings.py b/mregsite/settings.py index 67952711..b686c41b 100644 --- a/mregsite/settings.py +++ b/mregsite/settings.py @@ -90,8 +90,20 @@ def parse_protected_attrs(raw: str) -> list[dict[str, str]]: POLICY_NAMESPACE = [ns.strip() for ns in raw.split(",") if ns.strip()] or ["MREG"] POLICY_PARITY_BATCH_ENABLED = envvar("MREG_POLICY_PARITY_BATCH_ENABLED", True) POLICY_PARITY_LOG_DETAILS = envvar("MREG_POLICY_PARITY_LOG_DETAILS", False) -POLICY_PARITY_QUEUE_SIZE = envvar("MREG_POLICY_PARITY_QUEUE_SIZE", 100) POLICY_TIMEOUT_SECONDS = envvar("MREG_POLICY_TIMEOUT_SECONDS", 5.0) +POLICY_PARITY_MAX_ATTEMPTS = envvar("MREG_POLICY_PARITY_MAX_ATTEMPTS", 8) +POLICY_PARITY_RETRY_BASE_SECONDS = envvar("MREG_POLICY_PARITY_RETRY_BASE_SECONDS", 2.0) +POLICY_PARITY_RETRY_MAX_SECONDS = envvar("MREG_POLICY_PARITY_RETRY_MAX_SECONDS", 300.0) +POLICY_PARITY_LEASE_SECONDS = envvar("MREG_POLICY_PARITY_LEASE_SECONDS", 60.0) +POLICY_PARITY_POLL_SECONDS = envvar("MREG_POLICY_PARITY_POLL_SECONDS", 1.0) +POLICY_PARITY_CIRCUIT_FAILURES = envvar("MREG_POLICY_PARITY_CIRCUIT_FAILURES", 5) +POLICY_PARITY_CIRCUIT_RESET_SECONDS = envvar("MREG_POLICY_PARITY_CIRCUIT_RESET_SECONDS", 30.0) +POLICY_ROLLOUT_MIN_COMPARISONS = envvar("MREG_POLICY_ROLLOUT_MIN_COMPARISONS", 10_000) +POLICY_ROLLOUT_MAX_MISMATCH_RATE = envvar("MREG_POLICY_ROLLOUT_MAX_MISMATCH_RATE", 0.001) +POLICY_ROLLOUT_MAX_ERROR_RATE = envvar("MREG_POLICY_ROLLOUT_MAX_ERROR_RATE", 0.001) +POLICY_ROLLOUT_MAX_PERSIST_FAILURES = envvar("MREG_POLICY_ROLLOUT_MAX_PERSIST_FAILURES", 0) +POLICY_ROLLOUT_MAX_DEAD_LETTERS = envvar("MREG_POLICY_ROLLOUT_MAX_DEAD_LETTERS", 0) +POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS = envvar("MREG_POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS", 300.0) REQUESTS_THRESHOLD_SLOW = envvar("MREG_REQUESTS_THRESHOLD_SLOW", 1000) REQUESTS_LOG_LEVEL_SLOW = envvar("MREG_REQUESTS_LOG_LEVEL_SLOW", "WARNING") diff --git a/pyproject.toml b/pyproject.toml index b20d37a0..994c41f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,8 +24,6 @@ dependencies = [ # For OpenAPI schema generation # Pinned to prevent breaking changes: https://drf-spectacular.readthedocs.io/en/latest/readme.html#release-management "drf-spectacular[sidecar]==0.29.0", - # For testing inside Docker image - "unittest-parametrize", "treetop-client>=0.0.12", "prometheus-client>=0.24", ] @@ -36,6 +34,7 @@ dynamic = ["version"] dev = [ "tox-uv>=1.29", "coverage[toml]", + "unittest-parametrize", "uv>=0.10", "tblib>=3", {include-group = "profile"}, diff --git a/scripts/generate-treetop-schema.py b/scripts/generate-treetop-schema.py new file mode 100644 index 00000000..5bdb5dda --- /dev/null +++ b/scripts/generate-treetop-schema.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Generate or verify the Cedar schema from MREG's policy contracts.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from mreg.policy.contracts import render_cedar_schema + + +SCHEMA_PATH = Path("treetop/data/mreg.cedarschema") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true", help="fail if the committed schema is stale") + args = parser.parse_args() + rendered = render_cedar_schema() + if args.check: + if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered: + print(f"{SCHEMA_PATH} is stale; run {sys.argv[0]}", file=sys.stderr) + return 1 + print(f"{SCHEMA_PATH} matches the Python policy contracts") + return 0 + SCHEMA_PATH.write_text(rendered) + print(f"wrote {SCHEMA_PATH}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tox.ini b/tox.ini index 5c4045d2..701091a9 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ basepython = python312: python3.12 python313: python3.13 python314: python3.14 - python3 + python3.12 commands = python --version django52: python -c "import django; assert django.VERSION[:2] == (5, 2), django.get_version(); print(django.get_version())" diff --git a/treetop/data/mreg.cedarschema b/treetop/data/mreg.cedarschema index 32d984d1..09d6b9ad 100644 --- a/treetop/data/mreg.cedarschema +++ b/treetop/data/mreg.cedarschema @@ -3,7 +3,21 @@ namespace MREG { entity User in [Group]; entity Generic; + entity Host = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + }; entity HostContact; + entity Ipaddress = { + kind?: String, + id?: String, + ip?: ipaddr, + }; entity Cname; entity Hinfo; entity Loc; @@ -23,22 +37,6 @@ namespace MREG { entity NetworkPolicyAttribute; entity NetworkPolicyAttributeValue; - entity Host = { - kind?: String, - id?: String, - name?: String, - path?: String, - hostname?: String, - ip?: ipaddr, - nameLabels?: Set, - }; - - entity Ipaddress = { - kind?: String, - id?: String, - ip?: ipaddr, - }; - action "admin_access", "bacnet_id_create", diff --git a/uv.lock b/uv.lock index 5cf7e36f..c0518869 100644 --- a/uv.lock +++ b/uv.lock @@ -689,7 +689,6 @@ dependencies = [ { name = "structlog" }, { name = "treetop-client" }, { name = "tzdata" }, - { name = "unittest-parametrize" }, ] [package.dev-dependencies] @@ -700,6 +699,7 @@ ci = [ { name = "tblib" }, { name = "tox-gh-actions" }, { name = "tox-uv" }, + { name = "unittest-parametrize" }, { name = "uv" }, ] dev = [ @@ -707,6 +707,7 @@ dev = [ { name = "django-silk", extra = ["formatting"] }, { name = "tblib" }, { name = "tox-uv" }, + { name = "unittest-parametrize" }, { name = "uv" }, ] django52 = [ @@ -739,7 +740,6 @@ requires-dist = [ { name = "structlog", specifier = ">=25" }, { name = "treetop-client", specifier = ">=0.0.12" }, { name = "tzdata", specifier = ">=2025.3" }, - { name = "unittest-parametrize" }, ] [package.metadata.requires-dev] @@ -750,6 +750,7 @@ ci = [ { name = "tblib", specifier = ">=3" }, { name = "tox-gh-actions" }, { name = "tox-uv", specifier = ">=1.29" }, + { name = "unittest-parametrize" }, { name = "uv", specifier = ">=0.10" }, ] dev = [ @@ -757,6 +758,7 @@ dev = [ { name = "django-silk", extras = ["formatting"], specifier = ">=5.5.0" }, { name = "tblib", specifier = ">=3" }, { name = "tox-uv", specifier = ">=1.29" }, + { name = "unittest-parametrize" }, { name = "uv", specifier = ">=0.10" }, ] django52 = [{ name = "django", specifier = ">=5.2,<5.3" }] From 6665ac259be94919f21d97c33ff51ed5f0f337eb Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 18 Aug 2026 16:36:30 +0200 Subject: [PATCH 24/34] fix: support CI Dockerfile parser --- Dockerfile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 42dfec60..8706daa2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,16 @@ FROM builder AS test-builder RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --locked --no-editable --group dev +# Prepare application sources for production without relying on the newer +# Dockerfile COPY --exclude flag used only by recent BuildKit releases. +FROM builder AS runtime-builder +RUN rm -rf \ + /app/mreg/tests \ + /app/mreg/api/tests \ + /app/mreg/api/v1/tests \ + && find /app/mreg -type f -name '*.pyc' -delete \ + && find /app/mreg -depth -type d -name __pycache__ -empty -delete + # Production runtime stage. FROM python:3.12-alpine AS runtime EXPOSE 8000 @@ -42,12 +52,7 @@ COPY --from=builder /app/.venv /app/.venv # Copy over application files COPY entrypoint.sh manage.py /app/ -COPY \ - --exclude=tests \ - --exclude=api/tests \ - --exclude=api/v1/tests \ - --exclude=**/__pycache__ \ - mreg /app/mreg/ +COPY --from=runtime-builder /app/mreg /app/mreg/ COPY mregsite /app/mregsite/ COPY hostpolicy /app/hostpolicy/ COPY --from=ghcr.io/astral-sh/uv:0.12.0 /uv /uvx /bin/ From bb381bf6aa14c78cc848a8f87409d40111893962 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 18 Aug 2026 16:38:34 +0200 Subject: [PATCH 25/34] fix: run schema check from clean checkout --- scripts/generate-treetop-schema.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/generate-treetop-schema.py b/scripts/generate-treetop-schema.py index 5bdb5dda..62b5a77c 100644 --- a/scripts/generate-treetop-schema.py +++ b/scripts/generate-treetop-schema.py @@ -4,22 +4,33 @@ from __future__ import annotations import argparse +import importlib.util from pathlib import Path import sys -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from mreg.policy.contracts import render_cedar_schema +ROOT = Path(__file__).resolve().parents[1] +SCHEMA_PATH = Path("treetop/data/mreg.cedarschema") +CONTRACTS_PATH = ROOT / "mreg/policy/contracts.py" -SCHEMA_PATH = Path("treetop/data/mreg.cedarschema") +def _render_cedar_schema() -> str: + """Load the dependency-free contracts without importing the MREG package.""" + spec = importlib.util.spec_from_file_location("_mreg_policy_contracts", CONTRACTS_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load policy contracts from {CONTRACTS_PATH}") + module = importlib.util.module_from_spec(spec) + # dataclasses resolves annotations through the defining module while the + # class decorators execute, so the standalone module must be registered. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module.render_cedar_schema() def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--check", action="store_true", help="fail if the committed schema is stale") args = parser.parse_args() - rendered = render_cedar_schema() + rendered = _render_cedar_schema() if args.check: if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered: print(f"{SCHEMA_PATH} is stale; run {sys.argv[0]}", file=sys.stderr) From f46c238cba39d45d6e0f64b9ecb1318288f38e2d Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 18 Aug 2026 16:48:37 +0200 Subject: [PATCH 26/34] test: restore API regression discovery --- mreg/api/tests/__init__.py | 1 + mreg/api/tests/test_metrics.py | 16 ++++-- mreg/tests/test_gunicorn_conf.py | 18 ++++++ mreg/tests/test_policy_contracts.py | 7 +++ mreg/tests/test_policy_rollout.py | 43 +++++++++++++- mreg/tests/test_treetop_batching.py | 89 +++++++++++++++++++++++++++++ 6 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 mreg/api/tests/__init__.py diff --git a/mreg/api/tests/__init__.py b/mreg/api/tests/__init__.py new file mode 100644 index 00000000..42d30c0e --- /dev/null +++ b/mreg/api/tests/__init__.py @@ -0,0 +1 @@ +"""API-level regression tests.""" diff --git a/mreg/api/tests/test_metrics.py b/mreg/api/tests/test_metrics.py index d5912b27..d9d205a9 100644 --- a/mreg/api/tests/test_metrics.py +++ b/mreg/api/tests/test_metrics.py @@ -5,7 +5,8 @@ from rest_framework.test import APIClient from django.contrib.auth import get_user_model -from django.test import TestCase +from django.http import HttpResponse +from django.test import RequestFactory, TestCase from typing import Any from mreg.models.host import Host, Ipaddress @@ -71,7 +72,6 @@ def test_db_metrics_recorded_with_values(self) -> None: host = Host.objects.create( name="db_metric_test.example.com", - contact="test@example.com", ttl=3600, comment="test", ) @@ -99,7 +99,6 @@ def test_db_query_count_metrics(self) -> None: host = Host.objects.create( name="db_count_test.example.com", - contact="test@example.com", ttl=3600, comment="test", ) @@ -146,10 +145,17 @@ def test_request_and_response_size_histograms(self) -> None: user = User.objects.create_user(username="size_metrics_user", password="x") client.force_authenticate(user=user) - # Simple GET with no body (request size ~0), small response - r: Any = client.get("/api/meta/health/heartbeat") + # An explicit content length is observed without forcing request-body access. + r: Any = client.get("/api/meta/health/heartbeat", CONTENT_LENGTH="0") assert r.status_code == 200 + # Response sizes are recorded only when an upstream view/middleware sets + # Content-Length; exercise that explicit contract directly. + middleware = PrometheusRequestMiddleware( + lambda _request: HttpResponse(b"ok", headers={"Content-Length": "2"}) + ) + middleware(RequestFactory().get("/api/meta/health/heartbeat")) + metrics_resp: Any = client.get("/api/meta/metrics") raw = metrics_resp.content.decode("utf-8") diff --git a/mreg/tests/test_gunicorn_conf.py b/mreg/tests/test_gunicorn_conf.py index 4a66ebdc..e44a1f3a 100644 --- a/mreg/tests/test_gunicorn_conf.py +++ b/mreg/tests/test_gunicorn_conf.py @@ -12,6 +12,15 @@ class GunicornLifecycleHookTests(SimpleTestCase): """Ensure parity dispatchers follow each Gunicorn worker lifecycle.""" + @patch("django.setup") + @patch("django.apps.apps") + def test_setup_initializes_django_when_apps_are_not_ready(self, apps, django_setup): + apps.ready = False + + gunicorn_conf._setup_django() + + django_setup.assert_called_once_with() + @patch("mreg.api.treetop.start_policy_parity_dispatcher") def test_post_fork_starts_dispatcher(self, start_dispatcher): gunicorn_conf.post_fork(None, None) @@ -24,6 +33,15 @@ def test_worker_exit_stops_dispatcher(self, stop_dispatcher): stop_dispatcher.assert_called_once_with() + @patch("mreg.api.treetop.stop_policy_parity_dispatcher") + @patch("django.apps.apps") + def test_worker_exit_is_safe_before_django_setup(self, apps, stop_dispatcher): + apps.ready = False + + gunicorn_conf.worker_exit(None, SimpleNamespace(pid=42)) + + stop_dispatcher.assert_not_called() + @patch("prometheus_client.multiprocess.mark_process_dead") @patch("mreg.api.treetop.stop_policy_parity_dispatcher") def test_worker_exit_marks_prometheus_process_dead(self, stop_dispatcher, mark_process_dead): diff --git a/mreg/tests/test_policy_contracts.py b/mreg/tests/test_policy_contracts.py index cfd7d73b..61b1f346 100644 --- a/mreg/tests/test_policy_contracts.py +++ b/mreg/tests/test_policy_contracts.py @@ -1,9 +1,11 @@ +from pathlib import Path from unittest import mock from django.test import SimpleTestCase from mreg.api.permissions import ParityMixin from mreg.api.treetop import PolicyCheck, PolicyResource +from mreg.policy.contracts import render_cedar_schema class Host: @@ -33,6 +35,11 @@ def test_resource_kind_uses_serializer_model(self): "Host", ) + def test_rendered_schema_matches_committed_contract(self): + schema_path = Path(__file__).resolve().parents[2] / "treetop/data/mreg.cedarschema" + + self.assertEqual(render_cedar_schema(), schema_path.read_text()) + def test_resource_kind_supports_explicit_non_model_contract(self): view = mock.Mock(policy_resource_kind="Generic") view.get_serializer_class.side_effect = AttributeError diff --git a/mreg/tests/test_policy_rollout.py b/mreg/tests/test_policy_rollout.py index bb7eeb1a..02917df1 100644 --- a/mreg/tests/test_policy_rollout.py +++ b/mreg/tests/test_policy_rollout.py @@ -1,9 +1,50 @@ +from io import BytesIO +from unittest.mock import patch + from django.test import SimpleTestCase -from mreg.policy.rollout import RolloutSnapshot, RolloutThresholds, evaluate_rollout +from mreg.policy.rollout import ( + RolloutSnapshot, + RolloutThresholds, + _prometheus_value, + evaluate_rollout, + fetch_rollout_snapshot, +) class PolicyRolloutTests(SimpleTestCase): + def test_prometheus_value_handles_value_empty_and_error_responses(self) -> None: + with patch( + "mreg.policy.rollout.urlopen", + return_value=BytesIO(b'{"status":"success","data":{"result":[{"value":[1,"42.5"]}]}}'), + ): + self.assertEqual(_prometheus_value("http://prometheus/", "up == 1", 2), 42.5) + + with patch( + "mreg.policy.rollout.urlopen", + return_value=BytesIO(b'{"status":"success","data":{"result":[]}}'), + ): + self.assertEqual(_prometheus_value("http://prometheus", "absent(up)", 2), 0) + + with ( + patch( + "mreg.policy.rollout.urlopen", + return_value=BytesIO(b'{"status":"error","error":"bad query"}'), + ), + self.assertRaisesRegex(RuntimeError, "Prometheus query failed"), + ): + _prometheus_value("http://prometheus", "invalid", 2) + + @patch("mreg.policy.rollout._prometheus_value", side_effect=[100, 1, 2, 0, 0, 3]) + def test_fetch_rollout_snapshot_queries_every_gate(self, prometheus_value) -> None: + snapshot = fetch_rollout_snapshot("http://prometheus", window="6h", timeout=4) + + self.assertEqual(snapshot, RolloutSnapshot(100, 1, 2, 0, 0, 3)) + self.assertEqual(prometheus_value.call_count, 6) + self.assertTrue(all(call.args[0] == "http://prometheus" for call in prometheus_value.call_args_list)) + self.assertTrue(all(call.args[2] == 4 for call in prometheus_value.call_args_list)) + self.assertIn("[6h]", prometheus_value.call_args_list[0].args[1]) + def test_ready_snapshot_passes_every_gate(self) -> None: result = evaluate_rollout( RolloutSnapshot( diff --git a/mreg/tests/test_treetop_batching.py b/mreg/tests/test_treetop_batching.py index 36cbbabf..99706687 100644 --- a/mreg/tests/test_treetop_batching.py +++ b/mreg/tests/test_treetop_batching.py @@ -1,5 +1,6 @@ from __future__ import annotations +from copy import deepcopy import logging from types import SimpleNamespace from unittest.mock import Mock, patch @@ -18,6 +19,7 @@ _compute_parity_payload, _deserialize_policy_batch, _fully_qualified_action, + _get_treetop_client, _is_parity_enabled, _process_policy_parity_batch, _qualified_resource_kind, @@ -25,10 +27,13 @@ _result_to_decision_and_error, _safe_log, _serialize_policy_batch, + _close_treetop_client, batch_policy_parity, disable_policy_parity, flush_policy_parity_batch, policy_parity, + start_policy_parity_dispatcher, + stop_policy_parity_dispatcher, ) from mreg.middleware.logging_http import LoggingMiddleware @@ -408,6 +413,90 @@ def test_durable_batch_round_trip_preserves_typed_requests(self) -> None: self.assertEqual(restored[0].context, {"correlation_id": "cid"}) self.assertEqual(restored[0].policy_request.to_api(), request.to_api()) + def test_durable_batch_rejects_malformed_payloads(self) -> None: + request = _build_policy_request( + SimpleNamespace(username="tester", group_list=[]), + self._check(), + ).to_api() + + def batch(policy_request): # type: ignore[no-untyped-def] + return { + "version": 1, + "items": [{"decision": True, "policy_request": policy_request, "context": {}}], + } + + invalid_payloads: list[dict[str, object]] = [ + {"version": 2, "items": []}, + {"version": 1, "items": {}}, + {"version": 1, "items": ["invalid"]}, + {"version": 1, "items": [{"policy_request": request, "context": []}]}, + batch({"principal": []}), + batch({"principal": {"User": []}}), + ] + + invalid_action = deepcopy(request) + invalid_action["action"] = [] + invalid_payloads.append(batch(invalid_action)) + invalid_attrs = deepcopy(request) + invalid_attrs["resource"]["attrs"] = [] + invalid_payloads.append(batch(invalid_attrs)) + invalid_attribute = deepcopy(request) + invalid_attribute["resource"]["attrs"]["kind"] = [] + invalid_payloads.append(batch(invalid_attribute)) + + for payload in invalid_payloads: + with self.subTest(payload=payload), self.assertRaises((ValueError, KeyError)): + _deserialize_policy_batch(payload) + + def test_treetop_client_is_process_local_and_closed_safely(self) -> None: + old_client = Mock() + new_client = Mock() + with ( + patch("mreg.api.treetop._client", old_client), + patch("mreg.api.treetop._client_pid", -1), + patch("mreg.api.treetop.os.getpid", return_value=42), + patch("mreg.api.treetop.TreeTopClient", return_value=new_client) as client_class, + ): + self.assertIs(_get_treetop_client(), new_client) + + old_client.close.assert_called_once_with() + client_class.assert_called_once() + + with ( + patch("mreg.api.treetop._client", new_client), + patch("mreg.api.treetop._client_pid", 42), + patch("mreg.api.treetop.asyncio.run") as async_run, + ): + _close_treetop_client() + async_run.assert_called_once_with(new_client.aclose.return_value) + + fallback_client = Mock() + with ( + patch("mreg.api.treetop._client", fallback_client), + patch("mreg.api.treetop._client_pid", 42), + patch("mreg.api.treetop.asyncio.run", side_effect=RuntimeError("no event loop")), + ): + _close_treetop_client() + fallback_client.close.assert_called_once_with() + + def test_dispatcher_lifecycle_respects_configuration_and_process(self) -> None: + dispatcher = Mock() + with ( + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch("mreg.api.treetop._get_dispatcher", return_value=dispatcher), + ): + start_policy_parity_dispatcher() + dispatcher._ensure_started.assert_called_once_with() + + with ( + patch("mreg.api.treetop._dispatcher", dispatcher), + patch("mreg.api.treetop._dispatcher_pid", 42), + patch("mreg.api.treetop.os.getpid", return_value=42), + ): + stop_policy_parity_dispatcher() + dispatcher.shutdown.assert_called_once_with() + def test_dispatcher_claims_and_completes_a_durable_batch(self) -> None: from mreg.models.policy import PolicyParityOutbox From 9b754892ce65aa8fcc4fb7e5da372e1b6b57081e Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 18 Aug 2026 16:56:16 +0200 Subject: [PATCH 27/34] test: keep contract checks image-independent --- mreg/tests/test_policy_contracts.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mreg/tests/test_policy_contracts.py b/mreg/tests/test_policy_contracts.py index 61b1f346..dd91db3c 100644 --- a/mreg/tests/test_policy_contracts.py +++ b/mreg/tests/test_policy_contracts.py @@ -1,11 +1,10 @@ -from pathlib import Path from unittest import mock from django.test import SimpleTestCase from mreg.api.permissions import ParityMixin from mreg.api.treetop import PolicyCheck, PolicyResource -from mreg.policy.contracts import render_cedar_schema +from mreg.policy.contracts import POLICY_ACTIONS, RESOURCE_CONTRACTS, render_cedar_schema class Host: @@ -35,10 +34,14 @@ def test_resource_kind_uses_serializer_model(self): "Host", ) - def test_rendered_schema_matches_committed_contract(self): - schema_path = Path(__file__).resolve().parents[2] / "treetop/data/mreg.cedarschema" + def test_rendered_schema_contains_every_declared_contract(self): + schema = render_cedar_schema() - self.assertEqual(render_cedar_schema(), schema_path.read_text()) + self.assertTrue(schema.startswith("namespace MREG {")) + for contract in RESOURCE_CONTRACTS: + self.assertIn(f"entity {contract.kind}", schema) + for action in POLICY_ACTIONS: + self.assertIn(f'"{action}"', schema) def test_resource_kind_supports_explicit_non_model_contract(self): view = mock.Mock(policy_resource_kind="Generic") From 69190e1475bf596238166c27a54c73ebe94c175f Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 18 Aug 2026 16:58:48 +0200 Subject: [PATCH 28/34] ci: fetch pinned CLI revision explicitly --- .github/workflows/container-image.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index ae433699..c0d08823 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -97,7 +97,10 @@ jobs: cp -r mreg-cli/ci /tmp C=$(cat ci/MREG-CLI_COMMIT) cd mreg-cli - git -c advice.detachedHead=false checkout $C + # A default shallow clone can contain the commit object without all of + # its trees. Fetch the pinned revision explicitly before checkout. + git fetch --depth=1 origin "$C" + git -c advice.detachedHead=false checkout FETCH_HEAD cp --no-clobber /tmp/ci/* ci/ - name: Run the tests run: mreg-cli/ci/run_testsuite_and_record_V2.sh From 8d339f828831e9488a3a81bf37a58774ab1d69ab Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Tue, 18 Aug 2026 22:56:05 +0200 Subject: [PATCH 29/34] Add authoritative TreeTop policy mode --- README.md | 5 +- docs/env.md | 55 +++- docs/metrics.md | 35 ++- docs/parity_testing.md | 23 +- docs/policies.md | 55 ++++ monitoring/grafana/treetop-parity.json | 14 + monitoring/treetop-alerts.yml | 7 + mreg/api/treetop.py | 260 ++++++++++++++++-- .../commands/check_policy_rollout.py | 2 + mreg/policy/config.py | 48 ++++ mreg/policy/rollout.py | 7 + mreg/tests/test_policy_config.py | 51 ++++ mreg/tests/test_policy_rollout.py | 10 +- mreg/tests/test_treetop_batching.py | 228 +++++++++++++++ mregsite/gunicorn_conf.py | 2 +- mregsite/settings.py | 52 +++- 16 files changed, 793 insertions(+), 61 deletions(-) create mode 100644 mreg/policy/config.py create mode 100644 mreg/tests/test_policy_config.py diff --git a/README.md b/README.md index e9a1e27c..0a30888b 100644 --- a/README.md +++ b/README.md @@ -182,11 +182,13 @@ mreg supports configuration via environment variables with the `MREG_` prefix. T | Variable | Default | Description | | -------- | ------- | ----------- | -| `MREG_POLICY_PARITY_ENABLED` | `True` | Enable parity checks when a policy base URL is configured | +| `MREG_POLICY_MODE` | `shadow` | `off`, asynchronous `shadow`, or synchronous authoritative `enforce` | +| `MREG_POLICY_PARITY_ENABLED` | `True` | Deprecated compatibility flag used only when `MREG_POLICY_MODE` is unset | | `MREG_POLICY_BASE_URL` | `""` | TreeTop REST base URL; an empty value disables calls | | `MREG_POLICY_NAMESPACE` | `MREG` | Cedar namespace used for principals, actions, and resources | | `MREG_POLICY_PARITY_BATCH_ENABLED` | `True` | Persist one durable parity batch per HTTP request | | `MREG_POLICY_TIMEOUT_SECONDS` | `5.0` | TreeTop client timeout in seconds | +| `MREG_POLICY_ENFORCEMENT_FAILURE_MODE` | `deny` | Deny on enforcement error, or use the transitional `legacy` fallback | | `MREG_POLICY_PARITY_MAX_ATTEMPTS` | `8` | Delivery attempts before retaining a dead letter | | `MREG_POLICY_PARITY_RETRY_BASE_SECONDS` | `2.0` | Initial durable-outbox retry delay | | `MREG_POLICY_PARITY_RETRY_MAX_SECONDS` | `300.0` | Maximum durable-outbox retry delay | @@ -201,6 +203,7 @@ mreg supports configuration via environment variables with the `MREG_` prefix. T | `MREG_POLICY_ROLLOUT_MAX_ERROR_RATE` | `0.001` | Maximum accepted policy error ratio | | `MREG_POLICY_ROLLOUT_MAX_PERSIST_FAILURES` | `0` | Maximum accepted outbox persistence failures | | `MREG_POLICY_ROLLOUT_MAX_DEAD_LETTERS` | `0` | Maximum accepted dead letters | +| `MREG_POLICY_ROLLOUT_MAX_PENDING_BATCHES` | `0` | Maximum pending shadow batches before enforcement | | `MREG_POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS` | `300.0` | Maximum age of the oldest pending batch | ### Network Policy Configuration diff --git a/docs/env.md b/docs/env.md index 28deb936..6e7c1143 100644 --- a/docs/env.md +++ b/docs/env.md @@ -28,19 +28,33 @@ Must be one of the following: - `ERROR` - `CRITICAL` +## `MREG_POLICY_MODE` + +Controls how MREG uses TreeTop. Default: `shadow` + +- `off`: use legacy permissions and make no TreeTop calls. +- `shadow`: keep legacy permissions authoritative and submit comparisons + asynchronously through the durable PostgreSQL outbox. +- `enforce`: call TreeTop synchronously at each mapped authorization checkpoint + and use its result. The shadow outbox and dispatcher are not used. + +`enforce` requires a non-empty `MREG_POLICY_BASE_URL`; invalid values or a +missing enforcement URL stop Django during configuration rather than silently +falling back. + ## `MREG_POLICY_PARITY_ENABLED` -Boolean flag controlling whether policy parity checks run. Default: `True` +Deprecated compatibility flag. Default: `True` -Parity checks run only when both `MREG_POLICY_PARITY_ENABLED` is true and -`MREG_POLICY_BASE_URL` is set to a non-empty value. +When `MREG_POLICY_MODE` is unset, true maps to `shadow` and false maps to `off`. +An explicit mode always takes precedence. ## `MREG_POLICY_BASE_URL` Base URL for the TreeTop policy engine REST service. Default: empty (disabled) -If unset or empty, the policy parity code is disabled and no requests are made -to the policy engine. +If unset or empty, no policy requests are made in `off`/`shadow` operation. It +is a configuration error in `enforce` mode. Example: `http://localhost:9999` @@ -63,8 +77,23 @@ console and rotating `MREG_LOG_FILE_NAME` handlers. ## `MREG_POLICY_TIMEOUT_SECONDS` -Timeout in seconds for calls from the background parity worker to TreeTop. -Default: `5.0` +Timeout in seconds for calls to TreeTop. Default: `5.0` + +These calls run in the background in `shadow` and synchronously on the request +path in `enforce`. + +## `MREG_POLICY_ENFORCEMENT_FAILURE_MODE` + +Decision used when a synchronous authoritative TreeTop call cannot return a +valid result. Default: `deny` + +- `deny`: fail closed. This is the production enforcement default. +- `legacy`: return the already-computed legacy decision. This is a transitional + rollout fallback and is not fully authoritative. + +Explicit TreeTop allow/deny responses are always authoritative in `enforce`; +this setting applies only to transport, serialization, configuration, or +invalid-result failures. ## `MREG_POLICY_PARITY_BATCH_ENABLED` @@ -73,11 +102,13 @@ Default: `True` When enabled, parity checks are collected during request handling and persisted to the PostgreSQL outbox as one batch. Requests never wait for TreeTop. When -disabled, each check is persisted as its own durable batch. +disabled, each check is persisted as its own durable batch. This setting applies +only to `shadow`; authoritative checks are necessarily synchronous and are not +queued. ## Durable parity delivery -The following settings control the shared PostgreSQL outbox: +The following settings control the shared PostgreSQL outbox in `shadow` mode: - `MREG_POLICY_PARITY_MAX_ATTEMPTS` (`8`): delivery attempts before a row is retained as a dead letter. @@ -97,6 +128,11 @@ outbox necessarily contains the principal, groups, resource identifier, and resource attributes required for a later authorization call. Protect database access accordingly and establish an operational dead-letter retention policy. +Before switching from `shadow` to `enforce`, drain pending rows and resolve dead +letters. Enforcement does not start the dispatcher or consume old shadow rows; +re-evaluating them against a later bundle would not represent the decision that +was available when the original request ran. + The container sets `PROMETHEUS_MULTIPROC_DIR` to an isolated directory so metrics from every Gunicorn worker are aggregated. Custom Gunicorn deployments must set this variable to a clean, writable directory before starting Python. @@ -111,6 +147,7 @@ operator enables policy enforcement. Defaults can be tuned with: - `MREG_POLICY_ROLLOUT_MAX_ERROR_RATE` (`0.001`) - `MREG_POLICY_ROLLOUT_MAX_PERSIST_FAILURES` (`0`) - `MREG_POLICY_ROLLOUT_MAX_DEAD_LETTERS` (`0`) +- `MREG_POLICY_ROLLOUT_MAX_PENDING_BATCHES` (`0`) - `MREG_POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS` (`300.0`) ## `MREG_LOG_FILE_SIZE` diff --git a/docs/metrics.md b/docs/metrics.md index 35de62bf..f39e97f9 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -158,8 +158,21 @@ Metrics are exposed at the following endpoint: `/api/meta/metrics`. - Type: Counter - Labels: stage - Unit: failures - - Description: Fail-open parity instrumentation failures by processing stage. - - Typical label values: `build`, `persist`, `request_exit`, `worker`, `result_logging` + - Description: Policy integration failures by processing stage. Shadow failures are fail-open; enforcement failures follow the configured failure mode. + - Typical label values: `shadow_build`, `persist`, `request_exit`, `worker`, `result_logging`, `enforce_build`, `enforce_authorize`, `enforce_result` + +- Name: mreg_policy_enforcement_results_total + - Type: Counter + - Labels: result + - Unit: decisions + - Description: Synchronous authoritative outcomes in `enforce` mode. + - Label values: `allow`, `deny`, `error_deny`, `error_legacy` + +- Name: mreg_policy_mode_info + - Type: Gauge + - Labels: mode + - Description: Configured policy mode for the worker. + - Label values: `off`, `shadow`, `enforce` - Name: mreg_policy_authorize_duration_seconds - Type: Histogram @@ -181,14 +194,15 @@ Metrics are exposed at the following endpoint: `/api/meta/metrics`. - Type: Histogram - Labels: none - Unit: submitted batches - - Description: Number of policy batches submitted by each HTTP request. + - Description: Number of shadow batches submitted or synchronous enforcement calls made by each HTTP request. - Buckets/ranges: `0`, `1`, `2`, `3`, `4-5`, `6-8`, `9+` - Prometheus boundaries: [0, 1, 2, 3, 5, 8, +Inf] -When request batching is enabled (default), `mreg_policy_queries_per_request` -should usually be `0` (no parity checks produced) or `1` (one batch persisted). -The durable outbox worker performs the corresponding authorize call after -request handling. +In `shadow`, request batching normally produces `0` (no checks) or `1` (one +durable batch) and the worker authorizes it later. In `enforce`, the value is the +number of synchronous mapped authorization checkpoints reached by the request; +these cannot be delayed or combined after the request because their result +controls permission flow. ## TreeTop rollout dashboard and alerts @@ -198,7 +212,12 @@ request handling. The default gate requires at least 10,000 comparisons over the selected window, at most 0.1% mismatches, at most 0.1% errors, zero persistence failures, zero -dead letters, and a pending backlog younger than five minutes. +dead letters, zero pending shadow batches, and a pending backlog younger than +five minutes. + +`MregTreeTopEnforcementFailure` pages on any `error_deny` or `error_legacy` +outcome. `error_legacy` means TreeTop was not authoritative for that request and +should only exist during a deliberate transitional rollout. ## Labeling Strategy diff --git a/docs/parity_testing.md b/docs/parity_testing.md index a2ce0482..34fb02e8 100644 --- a/docs/parity_testing.md +++ b/docs/parity_testing.md @@ -86,13 +86,22 @@ Keep parity disable scope as narrow as possible: ## Implementation Details The `disable_policy_parity()` context manager uses `ContextVar` state. Nested -contexts and concurrently handled requests are isolated from one another. - -Parity batches are persisted in a shared PostgreSQL outbox. Post-fork workers -claim rows with database locks, retry with exponential backoff, and retain dead -letters after the configured attempt limit. A circuit breaker protects an -unavailable TreeTop service. Client, serialization, persistence, logging, and -TreeTop failures remain fail-open and never replace the legacy decision. +contexts and concurrently handled requests are isolated from one another. It +disables only `shadow` checks; it is deliberately ignored in `enforce` so test +or application code cannot bypass an authoritative decision accidentally. + +In `shadow`, parity batches are persisted in a shared PostgreSQL outbox. +Post-fork workers claim rows with database locks, retry with exponential +backoff, and retain dead letters after the configured attempt limit. A circuit +breaker protects an unavailable TreeTop service. Client, serialization, +persistence, logging, and TreeTop failures remain fail-open and never replace +the legacy decision. + +The outbox and its worker exist only in `shadow`. In `enforce`, each mapped +permission checkpoint calls TreeTop synchronously, records the comparison +immediately, and returns the policy decision. No enforcement request is queued +for later re-authorization. Enforcement failures deny by default; the explicit +`legacy` failure mode is available only as a transitional fallback. ## Parity Runbook diff --git a/docs/policies.md b/docs/policies.md index 30647343..06350b37 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -65,6 +65,61 @@ When introducing a new resource that should be parity-checked, use this checklis 10. Run parity checks and confirm zero mismatches. 11. If tests mutate permissions mid-test, scope `disable_policy_parity()` as narrowly as possible. +## Enforcement Mapping Boundary + +`MREG_POLICY_MODE=enforce` makes TreeTop synchronous and authoritative wherever +the permission path reaches `ParityMixin.pp()` or `pp_generic_action()`. The +return value at that checkpoint becomes the TreeTop decision. Authentication, +serializer validation, object lookup, and business invariants remain MREG +responsibilities. + +The mapping unit is a semantic permission checkpoint, not simply an HTTP +request. One request can reach multiple checks (for example DNS-name rules, +reserved-address rules, and a final host/IP permission). Those checks cannot be +batched after the request in enforcement mode because each result may control +the next branch. They therefore use synchronous `treetop-client.authorize` +calls. The PostgreSQL queue remains only for asynchronous `shadow` comparisons. + +The current DRF permission stack and Gunicorn workers are synchronous, and the +result is needed before permission evaluation can continue. Using an async HTTP +client would still require blocking at that boundary and would not make the +decision asynchronous. A future end-to-end ASGI conversion could await TreeTop, +but it would still be request-path I/O in `enforce`. + +Legacy permission code is still evaluated in `enforce` so its result can be +compared and so the existing control flow can reach the mapped checkpoint. The +TreeTop result returned by that checkpoint is authoritative. Once the mapping +inventory is complete, legacy computation can be removed or reduced in a +separate change. Until then, use `mreg_policy_queries_per_request` and authorize +latency histograms to find endpoints where multiple dependent checks should be +redesigned into one explicit endpoint-level policy decision. + +Current mapped checkpoints include: + +| Legacy decision | Policy mapping | +| --- | --- | +| Administrative group membership | Explicit `*_admin_access` action on `Generic` | +| CRUD permission after serializer/object resolution | Typed resource plus `_` | +| Host/network regex evaluation | `Host`/record resource with `hostname` and optional typed `ip` | +| DNS wildcard/underscore rules | Explicit membership actions | +| Restricted IP operations | Explicit IP-management actions | +| Host contact reads | Explicit `host_contacts_read` view action | + +This is an incremental mapping boundary, not yet proof that every endpoint in +MREG is policy-backed. Plain `IsAuthenticated` endpoints, host-group ownership, +and legacy branches that do not call the parity mixin remain application-owned +until they receive an explicit contract and Cedar rule. Do not describe a +deployment as globally TreeTop-authoritative until an endpoint inventory shows +that every authorization decision intended for delegation reaches a mapped +checkpoint. MREG authentication and non-authorization validation are expected +to remain local. + +Dynamic user group membership is sent with every TreeTop request. Mutable +database permission rules such as `NetGroupRegexPermission` are not exported +automatically; their Cedar equivalent must be present in the deployed bundle. +This bundle-sync requirement must be part of the mapping/deployment process +before enforcement is enabled. + ## Resource Kind and ID Resolution `ParityMixin` delegates resource kind, ID, and attributes to the typed adapters diff --git a/monitoring/grafana/treetop-parity.json b/monitoring/grafana/treetop-parity.json index f700013a..c43af14a 100644 --- a/monitoring/grafana/treetop-parity.json +++ b/monitoring/grafana/treetop-parity.json @@ -39,6 +39,20 @@ "type": "timeseries", "targets": [{"expr": "sum by (status) (rate(mreg_policy_parity_batches_total[5m]))", "legendFormat": "{{status}}"}], "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8} + }, + { + "id": 6, + "title": "Policy mode", + "type": "stat", + "targets": [{"expr": "max by (mode) (mreg_policy_mode_info)", "legendFormat": "{{mode}}"}], + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 16} + }, + { + "id": 7, + "title": "Enforcement decisions", + "type": "timeseries", + "targets": [{"expr": "sum by (result) (rate(mreg_policy_enforcement_results_total[5m]))", "legendFormat": "{{result}}"}], + "gridPos": {"h": 8, "w": 16, "x": 8, "y": 16} } ], "schemaVersion": 41, diff --git a/monitoring/treetop-alerts.yml b/monitoring/treetop-alerts.yml index 99eeef28..f8708730 100644 --- a/monitoring/treetop-alerts.yml +++ b/monitoring/treetop-alerts.yml @@ -51,3 +51,10 @@ groups: severity: ticket annotations: summary: A policy parity delivery circuit breaker is open + - alert: MregTreeTopEnforcementFailure + expr: increase(mreg_policy_enforcement_results_total{result=~"error_.*"}[5m]) > 0 + for: 0m + labels: + severity: page + annotations: + summary: An authoritative TreeTop decision failed diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index 5b2cfa34..e22ee8b9 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -34,10 +34,15 @@ from mreg.models.auth import User as MregUser from mreg.models.policy import PolicyParityOutbox +from mreg.policy.config import EnforcementFailureMode, PolicyMode logger = structlog.get_logger("mreg.policy.parity") -POLICY_PARITY_ENABLED = getattr(settings, "POLICY_PARITY_ENABLED", True) +POLICY_MODE = PolicyMode(getattr(settings, "POLICY_MODE", "shadow")) +POLICY_PARITY_ENABLED = getattr(settings, "POLICY_PARITY_ENABLED", POLICY_MODE == PolicyMode.SHADOW) +POLICY_ENFORCEMENT_FAILURE_MODE = EnforcementFailureMode( + getattr(settings, "POLICY_ENFORCEMENT_FAILURE_MODE", "deny") +) POLICY_BASE_URL = (getattr(settings, "POLICY_BASE_URL", "") or "").strip() POLICY_NAMESPACE = getattr(settings, "POLICY_NAMESPACE", ["MREG"]) POLICY_PARITY_BATCH_ENABLED = getattr(settings, "POLICY_PARITY_BATCH_ENABLED", True) @@ -84,10 +89,24 @@ POLICY_PARITY_FAILURES_TOTAL = Counter( "mreg_policy_parity_failures_total", - "Policy parity instrumentation failures that did not affect the legacy decision.", + "Policy integration failures by processing stage.", ["stage"], ) +POLICY_ENFORCEMENT_RESULTS_TOTAL = Counter( + "mreg_policy_enforcement_results_total", + "Synchronous authoritative policy outcomes.", + ["result"], +) + +POLICY_MODE_INFO = Gauge( + "mreg_policy_mode_info", + "Configured MREG policy decision mode.", + ["mode"], + multiprocess_mode="livemax", +) +POLICY_MODE_INFO.labels(mode=POLICY_MODE.value).set(1) + POLICY_AUTHORIZE_DURATION_SECONDS = Histogram( "mreg_policy_authorize_duration_seconds", "Duration of policy authorize endpoint calls in seconds.", @@ -534,8 +553,8 @@ def _get_dispatcher() -> _ParityDispatcher: def start_policy_parity_dispatcher() -> None: - """Start a post-fork outbox worker so persisted work survives restarts.""" - if POLICY_PARITY_ENABLED and POLICY_BASE_URL: + """Start the shadow-mode outbox worker after Gunicorn forks.""" + if _is_shadow_enabled(): _get_dispatcher()._ensure_started() @@ -582,7 +601,10 @@ def _submit_policy_batch(items: Sequence[_ParityBatchItem]) -> bool: @contextmanager def batch_policy_parity(): - """Collect one request's parity checks and persist them as one batch.""" + """Track one request's policy work and batch shadow checks.""" + if not _is_policy_enabled(): + yield + return if _request_state.get() is not None: yield return @@ -593,7 +615,7 @@ def batch_policy_parity(): yield finally: try: - if POLICY_PARITY_BATCH_ENABLED and state.items: + if _current_policy_mode() == PolicyMode.SHADOW and POLICY_PARITY_BATCH_ENABLED and state.items: if _submit_policy_batch(state.items): state.submitted_queries += 1 with suppress(Exception): @@ -606,7 +628,11 @@ def batch_policy_parity(): @contextmanager def disable_policy_parity(): - """Temporarily disable parity checks in the current execution context.""" + """Temporarily disable shadow checks in the current execution context. + + Enforcement deliberately ignores this test helper so production code cannot + turn an authoritative decision back into a legacy decision accidentally. + """ token = _parity_disabled_depth.set(_parity_disabled_depth.get() + 1) try: yield @@ -614,8 +640,38 @@ def disable_policy_parity(): _parity_disabled_depth.reset(token) +def _current_policy_mode() -> PolicyMode: + value = POLICY_MODE + return value if isinstance(value, PolicyMode) else PolicyMode(value) + + +def _current_enforcement_failure_mode() -> EnforcementFailureMode: + value = POLICY_ENFORCEMENT_FAILURE_MODE + return value if isinstance(value, EnforcementFailureMode) else EnforcementFailureMode(value) + + +def _is_shadow_enabled() -> bool: + return bool( + _current_policy_mode() == PolicyMode.SHADOW + and POLICY_PARITY_ENABLED + and POLICY_BASE_URL + and _parity_disabled_depth.get() == 0 + ) + + +def _is_enforcement_enabled() -> bool: + return _current_policy_mode() == PolicyMode.ENFORCE + + +def _is_policy_enabled() -> bool: + if _is_enforcement_enabled(): + return True + return _is_shadow_enabled() + + def _is_parity_enabled() -> bool: - return bool(POLICY_PARITY_ENABLED and POLICY_BASE_URL and _parity_disabled_depth.get() == 0) + """Compatibility alias for callers that mean shadow parity.""" + return _is_shadow_enabled() def _corr_id(request: Request) -> str | None: @@ -816,6 +872,130 @@ def flush_policy_parity_batch() -> bool: return submitted +def _build_policy_context( + *, + request: Request, + check: PolicyCheck, + view: View | None, + permission_class: str | None, +) -> dict[str, object]: + policy_action = Action.new(check.action, POLICY_NAMESPACE) + return { + "path": request.path, + "method": request.method, + "permission": permission_class or (view and view.__class__.__name__), + "view": view and view.__class__.__name__, + "model": _model_name_from_view(view), + "action": _fully_qualified_action(policy_action), + "resource_kind": _qualified_resource_kind(check.resource.kind), + "correlation_id": _corr_id(request), + "mode": _current_policy_mode().value, + } + + +def _record_enforcement_result(result: str) -> None: + with suppress(Exception): + POLICY_ENFORCEMENT_RESULTS_TOTAL.labels(result=result).inc() + + +def _enforcement_failure( + *, + decision: bool, + error: str, + context: dict[str, object], + stage: str, +) -> bool: + """Apply the configured fail-closed or transitional legacy fallback.""" + _record_instrumentation_failure(RuntimeError(error), stage=stage) + _log_parity_payload( + _compute_parity_payload( + decision=decision, + policy_allowed=None, + error=error, + context=context, + ) + ) + failure_mode = _current_enforcement_failure_mode() + if failure_mode == EnforcementFailureMode.LEGACY: + result = "error_legacy" + enforced_decision = bool(decision) + else: + result = "error_deny" + enforced_decision = False + _record_enforcement_result(result) + _safe_log( + logging.CRITICAL, + "policy_enforcement_failure", + failure_mode=failure_mode.value, + enforced_decision=enforced_decision, + error=error, + **context, + ) + return enforced_decision + + +def _enforce_policy_decision( + *, + decision: bool, + policy_request: TreeTopRequest, + context: dict[str, object], +) -> bool: + """Synchronously return the authoritative TreeTop decision.""" + if not POLICY_BASE_URL: + return _enforcement_failure( + decision=decision, + error="MREG_POLICY_BASE_URL is not configured", + context=context, + stage="enforce_configuration", + ) + + state = _request_state.get() + if state is not None: + state.submitted_queries += 1 + try: + results, authorize_error = _authorize_with_metrics( + policy_requests=[policy_request], + correlation_id=context.get("correlation_id") + if isinstance(context.get("correlation_id"), str) + else None, + path=context.get("path") if isinstance(context.get("path"), str) else None, + ) + except Exception as exc: + return _enforcement_failure( + decision=decision, + error=f"{type(exc).__name__}: {exc}", + context=context, + stage="enforce_instrumentation", + ) + if authorize_error is not None: + return _enforcement_failure( + decision=decision, + error=authorize_error, + context=context, + stage="enforce_authorize", + ) + + policy_allowed, result_error = _result_to_decision_and_error(results, 0) + if result_error is not None or policy_allowed is None: + return _enforcement_failure( + decision=decision, + error=result_error or "TreeTop returned no decision", + context=context, + stage="enforce_result", + ) + + _log_parity_payload( + _compute_parity_payload( + decision=decision, + policy_allowed=policy_allowed, + error=None, + context=context, + ) + ) + _record_enforcement_result("allow" if policy_allowed else "deny") + return policy_allowed + + def policy_parity( decision: bool, *, @@ -824,24 +1004,29 @@ def policy_parity( view: View | None = None, permission_class: str | None = None, ) -> bool: - """Queue a policy comparison and always preserve the legacy decision.""" - if not _is_parity_enabled(): + """Apply the configured off, shadow, or enforce policy behavior.""" + mode = _current_policy_mode() + if mode == PolicyMode.OFF: + return decision + if mode == PolicyMode.SHADOW and not _is_shadow_enabled(): return decision + context: dict[str, object] = { + "path": request.path, + "method": request.method, + "action": check.action, + "resource_kind": check.resource.kind, + "mode": mode.value, + } try: + context = _build_policy_context( + request=request, + check=check, + view=view, + permission_class=permission_class, + ) muser = MregUser.from_request(request) policy_request = _build_policy_request(muser, check) - policy_action = Action.new(check.action, POLICY_NAMESPACE) - context: dict[str, object] = { - "path": request.path, - "method": request.method, - "permission": permission_class or (view and view.__class__.__name__), - "view": view and view.__class__.__name__, - "model": _model_name_from_view(view), - "action": _fully_qualified_action(policy_action), - "resource_kind": _qualified_resource_kind(check.resource.kind), - "correlation_id": _corr_id(request), - } if POLICY_PARITY_LOG_DETAILS: context.update( { @@ -851,17 +1036,32 @@ def policy_parity( "resource_attrs": dict(check.resource.attrs), } ) + except Exception as exc: + if mode == PolicyMode.ENFORCE: + return _enforcement_failure( + decision=decision, + error=f"{type(exc).__name__}: {exc}", + context=context, + stage="enforce_build", + ) + _record_instrumentation_failure(exc, stage="shadow_build") + return decision - item = _ParityBatchItem( - decision=bool(decision), + if mode == PolicyMode.ENFORCE: + return _enforce_policy_decision( + decision=decision, policy_request=policy_request, context=context, ) - state = _request_state.get() - if state is not None and POLICY_PARITY_BATCH_ENABLED: - state.items.append(item) - elif _submit_policy_batch([item]) and state is not None: - state.submitted_queries += 1 - except Exception as exc: - _record_instrumentation_failure(exc, stage="build") + + item = _ParityBatchItem( + decision=bool(decision), + policy_request=policy_request, + context=context, + ) + state = _request_state.get() + if state is not None and POLICY_PARITY_BATCH_ENABLED: + state.items.append(item) + elif _submit_policy_batch([item]) and state is not None: + state.submitted_queries += 1 return decision diff --git a/mreg/management/commands/check_policy_rollout.py b/mreg/management/commands/check_policy_rollout.py index d3c77946..cccc34e6 100644 --- a/mreg/management/commands/check_policy_rollout.py +++ b/mreg/management/commands/check_policy_rollout.py @@ -21,6 +21,7 @@ def handle(self, *args, **options): # type: ignore[no-untyped-def] max_error_rate=settings.POLICY_ROLLOUT_MAX_ERROR_RATE, max_persist_failures=settings.POLICY_ROLLOUT_MAX_PERSIST_FAILURES, max_dead_letters=settings.POLICY_ROLLOUT_MAX_DEAD_LETTERS, + max_pending_batches=settings.POLICY_ROLLOUT_MAX_PENDING_BATCHES, max_backlog_age_seconds=settings.POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS, ) try: @@ -38,6 +39,7 @@ def handle(self, *args, **options): # type: ignore[no-untyped-def] f"error_rate={snapshot.error_rate:.6f} " f"persist_failures={snapshot.persist_failures:g} " f"dead_letters={snapshot.dead_letters:g} " + f"pending_batches={snapshot.pending_batches:g} " f"backlog_age_seconds={snapshot.backlog_age_seconds:g}" ) if not evaluation.ready: diff --git a/mreg/policy/config.py b/mreg/policy/config.py new file mode 100644 index 00000000..7d495b67 --- /dev/null +++ b/mreg/policy/config.py @@ -0,0 +1,48 @@ +"""Configuration contracts for TreeTop shadow and enforcement modes.""" + +from __future__ import annotations + +from enum import StrEnum + + +class PolicyMode(StrEnum): + """How MREG uses TreeTop decisions.""" + + OFF = "off" + SHADOW = "shadow" + ENFORCE = "enforce" + + +class EnforcementFailureMode(StrEnum): + """Decision used when authoritative TreeTop evaluation fails.""" + + DENY = "deny" + LEGACY = "legacy" + + +def resolve_policy_mode(raw: str | None, *, legacy_parity_enabled: bool) -> PolicyMode: + """Resolve the explicit mode, falling back to the deprecated boolean.""" + candidate = (raw or "").strip().lower() + if not candidate: + candidate = PolicyMode.SHADOW if legacy_parity_enabled else PolicyMode.OFF + try: + return PolicyMode(candidate) + except ValueError as exc: + raise ValueError("MREG_POLICY_MODE must be one of: off, shadow, enforce") from exc + + +def resolve_enforcement_failure_mode(raw: str | None) -> EnforcementFailureMode: + """Parse the explicit behavior used when TreeTop cannot decide.""" + candidate = (raw or EnforcementFailureMode.DENY).strip().lower() + try: + return EnforcementFailureMode(candidate) + except ValueError as exc: + raise ValueError( + "MREG_POLICY_ENFORCEMENT_FAILURE_MODE must be one of: deny, legacy" + ) from exc + + +def validate_policy_configuration(mode: PolicyMode, base_url: str) -> None: + """Reject configurations that cannot provide authoritative decisions.""" + if mode == PolicyMode.ENFORCE and not base_url.strip(): + raise ValueError("MREG_POLICY_BASE_URL is required when MREG_POLICY_MODE=enforce") diff --git a/mreg/policy/rollout.py b/mreg/policy/rollout.py index 00995c9c..e0b86543 100644 --- a/mreg/policy/rollout.py +++ b/mreg/policy/rollout.py @@ -15,6 +15,7 @@ class RolloutThresholds: max_error_rate: float = 0.001 max_persist_failures: int = 0 max_dead_letters: int = 0 + max_pending_batches: int = 0 max_backlog_age_seconds: float = 300.0 @@ -25,6 +26,7 @@ class RolloutSnapshot: errors: float persist_failures: float dead_letters: float + pending_batches: float backlog_age_seconds: float @property @@ -56,6 +58,10 @@ def evaluate_rollout(snapshot: RolloutSnapshot, thresholds: RolloutThresholds) - reasons.append(f"persist failures {snapshot.persist_failures:g} > {thresholds.max_persist_failures}") if snapshot.dead_letters > thresholds.max_dead_letters: reasons.append(f"dead letters {snapshot.dead_letters:g} > {thresholds.max_dead_letters}") + if snapshot.pending_batches > thresholds.max_pending_batches: + reasons.append( + f"pending batches {snapshot.pending_batches:g} > {thresholds.max_pending_batches}" + ) if snapshot.backlog_age_seconds > thresholds.max_backlog_age_seconds: reasons.append( f"oldest backlog age {snapshot.backlog_age_seconds:g}s > {thresholds.max_backlog_age_seconds:g}s" @@ -88,6 +94,7 @@ def fetch_rollout_snapshot( "errors": f'sum(increase(mreg_policy_parity_results_total{{result="error"}}[{window}]))', "persist_failures": f'sum(increase(mreg_policy_parity_batches_total{{status="persist_failed"}}[{window}]))', "dead_letters": 'max(mreg_policy_parity_outbox_entries{status="dead_letter"})', + "pending_batches": 'max(mreg_policy_parity_outbox_entries{status="pending"})', "backlog_age_seconds": "max(mreg_policy_parity_outbox_oldest_seconds)", } values = { diff --git a/mreg/tests/test_policy_config.py b/mreg/tests/test_policy_config.py new file mode 100644 index 00000000..0070dd10 --- /dev/null +++ b/mreg/tests/test_policy_config.py @@ -0,0 +1,51 @@ +from django.test import SimpleTestCase + +from mreg.policy.config import ( + EnforcementFailureMode, + PolicyMode, + resolve_enforcement_failure_mode, + resolve_policy_mode, + validate_policy_configuration, +) + + +class PolicyConfigurationTests(SimpleTestCase): + def test_explicit_policy_mode_takes_precedence(self) -> None: + self.assertEqual( + resolve_policy_mode(" enforce ", legacy_parity_enabled=False), + PolicyMode.ENFORCE, + ) + + def test_deprecated_parity_boolean_maps_to_shadow_or_off(self) -> None: + self.assertEqual( + resolve_policy_mode("", legacy_parity_enabled=True), + PolicyMode.SHADOW, + ) + self.assertEqual( + resolve_policy_mode(None, legacy_parity_enabled=False), + PolicyMode.OFF, + ) + + def test_invalid_policy_mode_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "off, shadow, enforce"): + resolve_policy_mode("invalid", legacy_parity_enabled=True) + + def test_enforcement_failure_mode_defaults_to_deny(self) -> None: + self.assertEqual( + resolve_enforcement_failure_mode(None), + EnforcementFailureMode.DENY, + ) + self.assertEqual( + resolve_enforcement_failure_mode(" legacy "), + EnforcementFailureMode.LEGACY, + ) + + def test_invalid_enforcement_failure_mode_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "deny, legacy"): + resolve_enforcement_failure_mode("allow") + + def test_enforcement_requires_a_base_url(self) -> None: + with self.assertRaisesRegex(ValueError, "MREG_POLICY_BASE_URL"): + validate_policy_configuration(PolicyMode.ENFORCE, "") + validate_policy_configuration(PolicyMode.ENFORCE, "http://policy") + validate_policy_configuration(PolicyMode.SHADOW, "") diff --git a/mreg/tests/test_policy_rollout.py b/mreg/tests/test_policy_rollout.py index 02917df1..0463d0a5 100644 --- a/mreg/tests/test_policy_rollout.py +++ b/mreg/tests/test_policy_rollout.py @@ -35,12 +35,12 @@ def test_prometheus_value_handles_value_empty_and_error_responses(self) -> None: ): _prometheus_value("http://prometheus", "invalid", 2) - @patch("mreg.policy.rollout._prometheus_value", side_effect=[100, 1, 2, 0, 0, 3]) + @patch("mreg.policy.rollout._prometheus_value", side_effect=[100, 1, 2, 0, 0, 0, 3]) def test_fetch_rollout_snapshot_queries_every_gate(self, prometheus_value) -> None: snapshot = fetch_rollout_snapshot("http://prometheus", window="6h", timeout=4) - self.assertEqual(snapshot, RolloutSnapshot(100, 1, 2, 0, 0, 3)) - self.assertEqual(prometheus_value.call_count, 6) + self.assertEqual(snapshot, RolloutSnapshot(100, 1, 2, 0, 0, 0, 3)) + self.assertEqual(prometheus_value.call_count, 7) self.assertTrue(all(call.args[0] == "http://prometheus" for call in prometheus_value.call_args_list)) self.assertTrue(all(call.args[2] == 4 for call in prometheus_value.call_args_list)) self.assertIn("[6h]", prometheus_value.call_args_list[0].args[1]) @@ -53,6 +53,7 @@ def test_ready_snapshot_passes_every_gate(self) -> None: errors=1, persist_failures=0, dead_letters=0, + pending_batches=0, backlog_age_seconds=10, ), RolloutThresholds(), @@ -69,11 +70,12 @@ def test_failed_snapshot_reports_every_broken_gate(self) -> None: errors=5, persist_failures=2, dead_letters=3, + pending_batches=4, backlog_age_seconds=600, ), RolloutThresholds(), ) self.assertFalse(result.ready) - self.assertEqual(len(result.reasons), 6) + self.assertEqual(len(result.reasons), 7) self.assertIn("comparisons", result.reasons[0]) diff --git a/mreg/tests/test_treetop_batching.py b/mreg/tests/test_treetop_batching.py index 99706687..2e65bf66 100644 --- a/mreg/tests/test_treetop_batching.py +++ b/mreg/tests/test_treetop_batching.py @@ -9,7 +9,9 @@ from django.test import TestCase from mreg.api.treetop import ( + EnforcementFailureMode, PolicyCheck, + PolicyMode, PolicyResource, _CircuitBreaker, _ParityBatchItem, @@ -20,7 +22,9 @@ _deserialize_policy_batch, _fully_qualified_action, _get_treetop_client, + _is_enforcement_enabled, _is_parity_enabled, + _is_policy_enabled, _process_policy_parity_batch, _qualified_resource_kind, _request_state, @@ -141,6 +145,41 @@ def test_disable_policy_parity_supports_nesting(self) -> None: self.assertFalse(_is_parity_enabled()) self.assertTrue(_is_parity_enabled()) + def test_policy_modes_distinguish_shadow_and_enforcement(self) -> None: + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.OFF), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + ): + self.assertFalse(_is_parity_enabled()) + self.assertFalse(_is_enforcement_enabled()) + self.assertFalse(_is_policy_enabled()) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + disable_policy_parity(), + ): + self.assertFalse(_is_parity_enabled()) + self.assertTrue(_is_enforcement_enabled()) + self.assertTrue(_is_policy_enabled()) + + def test_off_mode_does_not_build_submit_or_authorize(self) -> None: + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.OFF), + patch("mreg.api.treetop.MregUser.from_request") as from_request, + patch("mreg.api.treetop._submit_policy_batch") as submit, + patch("mreg.api.treetop._get_treetop_client") as get_client, + ): + decision = self._run_parity_check( + self._request(), + decision=True, + hostname="host.example", + ) + + self.assertTrue(decision) + from_request.assert_not_called() + submit.assert_not_called() + get_client.assert_not_called() + def test_build_resource_attrs_detects_ip_values(self) -> None: attrs = _build_resource_attrs({"ip": "192.0.2.1", "name": "host"}) self.assertEqual(attrs["ip"].type.value, "Ip") @@ -297,6 +336,185 @@ def test_policy_parity_is_fail_open_for_build_errors( self.assertTrue(policy_parity(True, request=self._request(), check=self._check())) record_failure.assert_called_once() + @patch("mreg.api.treetop.MregUser.from_request") + def test_enforcement_is_synchronous_and_does_not_use_outbox( + self, + mock_from_request: Mock, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + client = Mock() + client.authorize.return_value = _DummyAuthorizeResponse([True]) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch("mreg.api.treetop._get_treetop_client", return_value=client), + patch("mreg.api.treetop._submit_policy_batch") as submit, + batch_policy_parity(), + ): + enforced = self._run_parity_check( + self._request(), + decision=False, + hostname="host.example", + ) + + self.assertTrue(enforced) + client.authorize.assert_called_once() + submit.assert_not_called() + + @patch("mreg.api.treetop.MregUser.from_request") + def test_enforcement_policy_deny_overrides_legacy_allow( + self, + mock_from_request: Mock, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + client = Mock() + client.authorize.return_value = _DummyAuthorizeResponse([False]) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch("mreg.api.treetop._get_treetop_client", return_value=client), + ): + enforced = self._run_parity_check( + self._request(), + decision=True, + hostname="host.example", + ) + + self.assertFalse(enforced) + + @patch("mreg.api.treetop.MregUser.from_request") + def test_enforcement_errors_deny_by_default( + self, + mock_from_request: Mock, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + client = Mock() + client.authorize.side_effect = RuntimeError("offline") + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch( + "mreg.api.treetop.POLICY_ENFORCEMENT_FAILURE_MODE", + EnforcementFailureMode.DENY, + ), + patch("mreg.api.treetop._get_treetop_client", return_value=client), + patch("mreg.api.treetop._submit_policy_batch") as submit, + ): + enforced = self._run_parity_check( + self._request(), + decision=True, + hostname="host.example", + ) + + self.assertFalse(enforced) + submit.assert_not_called() + + @patch("mreg.api.treetop.MregUser.from_request") + def test_enforcement_can_use_explicit_legacy_failure_fallback( + self, + mock_from_request: Mock, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + client = Mock() + client.authorize.side_effect = RuntimeError("offline") + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch( + "mreg.api.treetop.POLICY_ENFORCEMENT_FAILURE_MODE", + EnforcementFailureMode.LEGACY, + ), + patch("mreg.api.treetop._get_treetop_client", return_value=client), + ): + enforced = self._run_parity_check( + self._request(), + decision=True, + hostname="host.example", + ) + + self.assertTrue(enforced) + + @patch("mreg.api.treetop.MregUser.from_request") + def test_disable_shadow_helper_cannot_bypass_enforcement( + self, + mock_from_request: Mock, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + client = Mock() + client.authorize.return_value = _DummyAuthorizeResponse([False]) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch("mreg.api.treetop._get_treetop_client", return_value=client), + disable_policy_parity(), + ): + enforced = self._run_parity_check( + self._request(), + decision=True, + hostname="host.example", + ) + + self.assertFalse(enforced) + + @patch("mreg.api.treetop.MregUser.from_request", side_effect=RuntimeError("bad principal")) + def test_enforcement_build_errors_fail_closed(self, _from_request: Mock) -> None: + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch( + "mreg.api.treetop.POLICY_ENFORCEMENT_FAILURE_MODE", + EnforcementFailureMode.DENY, + ), + ): + enforced = self._run_parity_check( + self._request(), + decision=True, + hostname="host.example", + ) + + self.assertFalse(enforced) + + @patch("mreg.api.treetop.MregUser.from_request") + def test_enforcement_missing_result_fails_closed( + self, + mock_from_request: Mock, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + client = Mock() + client.authorize.return_value = _DummyAuthorizeResponse([]) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch("mreg.api.treetop._get_treetop_client", return_value=client), + ): + enforced = self._run_parity_check( + self._request(), + decision=True, + hostname="host.example", + ) + + self.assertFalse(enforced) + + @patch("mreg.api.treetop.MregUser.from_request") + def test_enforcement_without_runtime_url_fails_closed( + self, + mock_from_request: Mock, + ) -> None: + mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", ""), + patch("mreg.api.treetop._get_treetop_client") as get_client, + ): + enforced = self._run_parity_check( + self._request(), + decision=True, + hostname="host.example", + ) + + self.assertFalse(enforced) + get_client.assert_not_called() + @patch("mreg.api.treetop.MregUser.from_request") def test_sensitive_log_details_are_disabled_by_default(self, mock_from_request: Mock) -> None: mock_from_request.return_value = SimpleNamespace( @@ -497,6 +715,16 @@ def test_dispatcher_lifecycle_respects_configuration_and_process(self) -> None: stop_policy_parity_dispatcher() dispatcher.shutdown.assert_called_once_with() + def test_dispatcher_does_not_start_in_enforcement_mode(self) -> None: + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch("mreg.api.treetop._get_dispatcher") as get_dispatcher, + ): + start_policy_parity_dispatcher() + + get_dispatcher.assert_not_called() + def test_dispatcher_claims_and_completes_a_durable_batch(self) -> None: from mreg.models.policy import PolicyParityOutbox diff --git a/mregsite/gunicorn_conf.py b/mregsite/gunicorn_conf.py index 13089a00..58b20868 100644 --- a/mregsite/gunicorn_conf.py +++ b/mregsite/gunicorn_conf.py @@ -15,7 +15,7 @@ def _setup_django() -> None: def post_fork(server, worker): # noqa: ARG001 - """Start the durable parity outbox consumer only after worker fork.""" + """Start the shadow-mode outbox consumer only after worker fork.""" _setup_django() from mreg.api.treetop import start_policy_parity_dispatcher diff --git a/mregsite/settings.py b/mregsite/settings.py index b686c41b..b5d09854 100644 --- a/mregsite/settings.py +++ b/mregsite/settings.py @@ -16,10 +16,17 @@ import sys from typing import Literal, TypeVar +from django.core.exceptions import ImproperlyConfigured import structlog import mreg.log_processors import mreg.__about__ +from mreg.policy.config import ( + PolicyMode, + resolve_enforcement_failure_mode, + resolve_policy_mode, + validate_policy_configuration, +) DefaultT = TypeVar("DefaultT", str, int, float, bool) @@ -82,8 +89,26 @@ def parse_protected_attrs(raw: str) -> list[dict[str, str]]: LOG_LEVEL = envvar("MREG_LOG_LEVEL", "CRITICAL").upper() POLICY_PARITY_LOG_LEVEL = envvar("MREG_POLICY_PARITY_LOG_LEVEL", "WARNING").upper() -POLICY_PARITY_ENABLED = envvar("MREG_POLICY_PARITY_ENABLED", True) POLICY_BASE_URL = envvar("MREG_POLICY_BASE_URL", "").strip() +_legacy_policy_parity_enabled = envvar("MREG_POLICY_PARITY_ENABLED", True) +_raw_policy_mode = envvar("MREG_POLICY_MODE", "") +_policy_mode_was_explicit = bool((_raw_policy_mode or "").strip()) +try: + _policy_mode = resolve_policy_mode( + _raw_policy_mode, + legacy_parity_enabled=_legacy_policy_parity_enabled, + ) + _policy_enforcement_failure_mode = resolve_enforcement_failure_mode( + envvar("MREG_POLICY_ENFORCEMENT_FAILURE_MODE", "deny") + ) + validate_policy_configuration(_policy_mode, POLICY_BASE_URL) +except ValueError as exc: + raise ImproperlyConfigured(str(exc)) from exc +POLICY_MODE = _policy_mode.value +POLICY_ENFORCEMENT_FAILURE_MODE = _policy_enforcement_failure_mode.value +# Compatibility for local settings and integrations that still inspect the old +# boolean. Explicit MREG_POLICY_MODE takes precedence over the deprecated flag. +POLICY_PARITY_ENABLED = _policy_mode == PolicyMode.SHADOW raw = (envvar("MREG_POLICY_NAMESPACE", "MREG") or "").strip() # Accept both Cedar-style `org::MREG` and comma-separated `org,MREG`. raw = raw.replace("::", ",") @@ -103,6 +128,7 @@ def parse_protected_attrs(raw: str) -> list[dict[str, str]]: POLICY_ROLLOUT_MAX_ERROR_RATE = envvar("MREG_POLICY_ROLLOUT_MAX_ERROR_RATE", 0.001) POLICY_ROLLOUT_MAX_PERSIST_FAILURES = envvar("MREG_POLICY_ROLLOUT_MAX_PERSIST_FAILURES", 0) POLICY_ROLLOUT_MAX_DEAD_LETTERS = envvar("MREG_POLICY_ROLLOUT_MAX_DEAD_LETTERS", 0) +POLICY_ROLLOUT_MAX_PENDING_BATCHES = envvar("MREG_POLICY_ROLLOUT_MAX_PENDING_BATCHES", 0) POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS = envvar("MREG_POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS", 300.0) REQUESTS_THRESHOLD_SLOW = envvar("MREG_REQUESTS_THRESHOLD_SLOW", 1000) @@ -503,6 +529,30 @@ def parse_protected_attrs(raw: str) -> list[dict[str, str]]: except ImportError: pass +# Validate policy values again because local_settings.py may override the +# environment-derived configuration above. +try: + _post_local_policy_mode = POLICY_MODE + if ( + not _policy_mode_was_explicit + and _post_local_policy_mode == PolicyMode.SHADOW.value + and not POLICY_PARITY_ENABLED + ): + _post_local_policy_mode = "" + _policy_mode = resolve_policy_mode( + _post_local_policy_mode, + legacy_parity_enabled=POLICY_PARITY_ENABLED, + ) + _policy_enforcement_failure_mode = resolve_enforcement_failure_mode( + POLICY_ENFORCEMENT_FAILURE_MODE + ) + validate_policy_configuration(_policy_mode, POLICY_BASE_URL) +except ValueError as exc: + raise ImproperlyConfigured(str(exc)) from exc +POLICY_MODE = _policy_mode.value +POLICY_ENFORCEMENT_FAILURE_MODE = _policy_enforcement_failure_mode.value +POLICY_PARITY_ENABLED = _policy_mode == PolicyMode.SHADOW + if TESTING or "CI" in os.environ: SUPERUSER_GROUP = "default-super-group" ADMINUSER_GROUP = "default-admin-group" From ad3260ca1c8244781b45fafcb5a12af2cc997fd2 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 19 Aug 2026 01:21:08 +0200 Subject: [PATCH 30/34] Refactor TreeTop authorization into request-scoped stacks --- README.md | 17 +- docs/env.md | 73 +- docs/metrics.md | 413 ++---- docs/parity_testing.md | 31 +- docs/policies.md | 401 +++--- hostpolicy/api/permissions.py | 159 ++- monitoring/grafana/treetop-parity.json | 35 +- monitoring/treetop-alerts.yml | 42 +- mreg/api/permissions.py | 985 ++++++++++---- mreg/api/treetop.py | 1154 +++++------------ mreg/api/v1/tests/test_logging.py | 12 +- mreg/api/v1/views.py | 49 +- mreg/api/v1/views_bacnet.py | 6 +- mreg/api/v1/views_hostgroups.py | 10 +- mreg/api/v1/views_network_policy.py | 13 +- mreg/api/v1/views_zones.py | 12 +- mreg/api/views.py | 23 +- .../commands/check_policy_rollout.py | 10 +- mreg/middleware/logging_http.py | 4 +- mreg/migrations/0017_policyparityoutbox.py | 35 - mreg/models/__init__.py | 1 - mreg/models/policy.py | 26 - mreg/policy/config.py | 18 - mreg/policy/contracts.py | 43 +- mreg/policy/rollout.py | 26 +- mreg/tests/test_gunicorn_conf.py | 35 +- mreg/tests/test_policy_config.py | 16 - mreg/tests/test_policy_rollout.py | 16 +- mreg/tests/test_treetop.py | 283 ++++ mreg/tests/test_treetop_batching.py | 808 ------------ mregsite/gunicorn_conf.py | 26 +- mregsite/settings.py | 23 +- treetop/data/labels.json | 114 +- treetop/data/mreg-bundle.tar.gz | Bin 3477 -> 5102 bytes treetop/data/mreg.cedar | 489 ++++--- treetop/data/mreg.cedarschema | 680 +++++++++- 36 files changed, 3038 insertions(+), 3050 deletions(-) delete mode 100644 mreg/migrations/0017_policyparityoutbox.py delete mode 100644 mreg/models/policy.py create mode 100644 mreg/tests/test_treetop.py delete mode 100644 mreg/tests/test_treetop_batching.py diff --git a/README.md b/README.md index 0a30888b..c40e3ddc 100644 --- a/README.md +++ b/README.md @@ -182,29 +182,18 @@ mreg supports configuration via environment variables with the `MREG_` prefix. T | Variable | Default | Description | | -------- | ------- | ----------- | -| `MREG_POLICY_MODE` | `shadow` | `off`, asynchronous `shadow`, or synchronous authoritative `enforce` | +| `MREG_POLICY_MODE` | `shadow` | `off`, synchronous observational `shadow`, or synchronous authoritative `enforce` | | `MREG_POLICY_PARITY_ENABLED` | `True` | Deprecated compatibility flag used only when `MREG_POLICY_MODE` is unset | | `MREG_POLICY_BASE_URL` | `""` | TreeTop REST base URL; an empty value disables calls | | `MREG_POLICY_NAMESPACE` | `MREG` | Cedar namespace used for principals, actions, and resources | -| `MREG_POLICY_PARITY_BATCH_ENABLED` | `True` | Persist one durable parity batch per HTTP request | | `MREG_POLICY_TIMEOUT_SECONDS` | `5.0` | TreeTop client timeout in seconds | -| `MREG_POLICY_ENFORCEMENT_FAILURE_MODE` | `deny` | Deny on enforcement error, or use the transitional `legacy` fallback | -| `MREG_POLICY_PARITY_MAX_ATTEMPTS` | `8` | Delivery attempts before retaining a dead letter | -| `MREG_POLICY_PARITY_RETRY_BASE_SECONDS` | `2.0` | Initial durable-outbox retry delay | -| `MREG_POLICY_PARITY_RETRY_MAX_SECONDS` | `300.0` | Maximum durable-outbox retry delay | -| `MREG_POLICY_PARITY_LEASE_SECONDS` | `60.0` | Time before another worker may reclaim an abandoned row | -| `MREG_POLICY_PARITY_POLL_SECONDS` | `1.0` | Durable-outbox polling interval | -| `MREG_POLICY_PARITY_CIRCUIT_FAILURES` | `5` | Consecutive failures that open the delivery circuit | -| `MREG_POLICY_PARITY_CIRCUIT_RESET_SECONDS` | `30.0` | Open-circuit cooldown | +| `MREG_POLICY_CIRCUIT_FAILURES` | `5` | Consecutive synchronous failures that open a worker circuit | +| `MREG_POLICY_CIRCUIT_RESET_SECONDS` | `30.0` | Open-circuit cooldown | | `MREG_POLICY_PARITY_LOG_LEVEL` | `WARNING` | Dedicated parity logger level | | `MREG_POLICY_PARITY_LOG_DETAILS` | `False` | Include sensitive principal/resource details in parity logs | | `MREG_POLICY_ROLLOUT_MIN_COMPARISONS` | `10000` | Minimum observations required by the enforcement gate | | `MREG_POLICY_ROLLOUT_MAX_MISMATCH_RATE` | `0.001` | Maximum accepted mismatch ratio | | `MREG_POLICY_ROLLOUT_MAX_ERROR_RATE` | `0.001` | Maximum accepted policy error ratio | -| `MREG_POLICY_ROLLOUT_MAX_PERSIST_FAILURES` | `0` | Maximum accepted outbox persistence failures | -| `MREG_POLICY_ROLLOUT_MAX_DEAD_LETTERS` | `0` | Maximum accepted dead letters | -| `MREG_POLICY_ROLLOUT_MAX_PENDING_BATCHES` | `0` | Maximum pending shadow batches before enforcement | -| `MREG_POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS` | `300.0` | Maximum age of the oldest pending batch | ### Network Policy Configuration diff --git a/docs/env.md b/docs/env.md index 6e7c1143..991aceaa 100644 --- a/docs/env.md +++ b/docs/env.md @@ -33,10 +33,9 @@ Must be one of the following: Controls how MREG uses TreeTop. Default: `shadow` - `off`: use legacy permissions and make no TreeTop calls. -- `shadow`: keep legacy permissions authoritative and submit comparisons - asynchronously through the durable PostgreSQL outbox. -- `enforce`: call TreeTop synchronously at each mapped authorization checkpoint - and use its result. The shadow outbox and dispatcher are not used. +- `shadow`: call TreeTop synchronously once per protected request, compare the + complete endpoint decision, and return the legacy decision. +- `enforce`: make that same synchronous endpoint decision authoritative. `enforce` requires a non-empty `MREG_POLICY_BASE_URL`; invalid values or a missing enforcement URL stop Django during configuration rather than silently @@ -79,59 +78,21 @@ console and rotating `MREG_LOG_FILE_NAME` handlers. Timeout in seconds for calls to TreeTop. Default: `5.0` -These calls run in the background in `shadow` and synchronously on the request -path in `enforce`. +Both active modes wait for the result because authorization must finish before +request processing continues. `shadow` differs only in which decision is +returned. `enforce` always fails closed on timeout, invalid response, circuit +rejection, or other TreeTop failure; there is no legacy fallback. -## `MREG_POLICY_ENFORCEMENT_FAILURE_MODE` +## Synchronous circuit breaker -Decision used when a synchronous authoritative TreeTop call cannot return a -valid result. Default: `deny` +- `MREG_POLICY_CIRCUIT_FAILURES` (`5`): consecutive failures before the + process-local worker circuit opens. +- `MREG_POLICY_CIRCUIT_RESET_SECONDS` (`30.0`): cooldown before one half-open + probe is allowed. -- `deny`: fail closed. This is the production enforcement default. -- `legacy`: return the already-computed legacy decision. This is a transitional - rollout fallback and is not fully authoritative. - -Explicit TreeTop allow/deny responses are always authoritative in `enforce`; -this setting applies only to transport, serialization, configuration, or -invalid-result failures. - -## `MREG_POLICY_PARITY_BATCH_ENABLED` - -Boolean flag controlling request-scoped batching of parity authorize checks. -Default: `True` - -When enabled, parity checks are collected during request handling and persisted -to the PostgreSQL outbox as one batch. Requests never wait for TreeTop. When -disabled, each check is persisted as its own durable batch. This setting applies -only to `shadow`; authoritative checks are necessarily synchronous and are not -queued. - -## Durable parity delivery - -The following settings control the shared PostgreSQL outbox in `shadow` mode: - -- `MREG_POLICY_PARITY_MAX_ATTEMPTS` (`8`): delivery attempts before a row is - retained as a dead letter. -- `MREG_POLICY_PARITY_RETRY_BASE_SECONDS` (`2.0`): initial exponential-backoff - delay. -- `MREG_POLICY_PARITY_RETRY_MAX_SECONDS` (`300.0`): retry delay cap. -- `MREG_POLICY_PARITY_LEASE_SECONDS` (`60.0`): time before an abandoned claim - can be reclaimed by another worker. -- `MREG_POLICY_PARITY_POLL_SECONDS` (`1.0`): worker polling interval. -- `MREG_POLICY_PARITY_CIRCUIT_FAILURES` (`5`): consecutive delivery failures - that open a worker's circuit breaker. -- `MREG_POLICY_PARITY_CIRCUIT_RESET_SECONDS` (`30.0`): circuit cooldown. - -Successful rows are deleted. Exhausted rows remain in -`mreg_policyparityoutbox` with `failed_at` and `last_error` populated. The -outbox necessarily contains the principal, groups, resource identifier, and -resource attributes required for a later authorization call. Protect database -access accordingly and establish an operational dead-letter retention policy. - -Before switching from `shadow` to `enforce`, drain pending rows and resolve dead -letters. Enforcement does not start the dispatcher or consume old shadow rows; -re-evaluating them against a later bundle would not represent the decision that -was available when the original request ran. +The client timeout remains `MREG_POLICY_TIMEOUT_SECONDS` (`5.0`). Each Gunicorn +worker owns its client and thread-safe circuit state. An open circuit returns +the legacy decision in `shadow` and denies in `enforce`. The container sets `PROMETHEUS_MULTIPROC_DIR` to an isolated directory so metrics from every Gunicorn worker are aggregated. Custom Gunicorn deployments @@ -145,10 +106,6 @@ operator enables policy enforcement. Defaults can be tuned with: - `MREG_POLICY_ROLLOUT_MIN_COMPARISONS` (`10000`) - `MREG_POLICY_ROLLOUT_MAX_MISMATCH_RATE` (`0.001`) - `MREG_POLICY_ROLLOUT_MAX_ERROR_RATE` (`0.001`) -- `MREG_POLICY_ROLLOUT_MAX_PERSIST_FAILURES` (`0`) -- `MREG_POLICY_ROLLOUT_MAX_DEAD_LETTERS` (`0`) -- `MREG_POLICY_ROLLOUT_MAX_PENDING_BATCHES` (`0`) -- `MREG_POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS` (`300.0`) ## `MREG_LOG_FILE_SIZE` diff --git a/docs/metrics.md b/docs/metrics.md index f39e97f9..e6cdab85 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,331 +1,90 @@ # Metrics Overview -This document describes the Prometheus metrics exposed by MREG, their purpose, labels, and units. Labels are chosen to keep cardinality low and operationally useful. - -## Endpoint - -Metrics are exposed at the following endpoint: `/api/meta/metrics`. - -## HTTP Metrics - -- Name: mreg_http_requests_total - - Type: Counter - - Labels: method, path, status - - Unit: requests - - Description: Total number of HTTP requests, partitioned by method, normalized path (view name/route), and status code. - -- Name: mreg_http_request_duration_seconds - - Type: Histogram - - Labels: method, path, status - - Unit: seconds - - Description: Request latency from middleware entry to response. - - Buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] - -- Name: mreg_http_inprogress_requests - - Type: Gauge - - Labels: method, path - - Unit: requests - - Description: Number of requests in-flight. - -- Name: mreg_http_request_size_bytes - - Type: Histogram - - Labels: method, path - - Unit: bytes - - Description: Size of HTTP request payload. Uses `CONTENT_LENGTH` when present; otherwise not observed to avoid loading bodies. - - Buckets: [512, 1k, 2k, 4k, 8k, 16k, 64k, 256k, 1M, 4M] - -- Name: mreg_http_response_size_bytes - - Type: Histogram - - Labels: method, path, status - - Unit: bytes - - Description: Size of HTTP response payload. Uses `Content-Length` when set; skips observation for streaming or unknown sizes. - - Buckets: [512, 1k, 2k, 4k, 8k, 16k, 64k, 256k, 1M, 4M] - -- Name: mreg_http_exceptions_total - - Type: Counter - - Labels: method, path, exception - - Unit: exceptions - - Description: Total number of uncaught application exceptions that resulted in 500 responses, partitioned by exception class name. - -- Name: mreg_http_unresolved_requests_total - - Type: Counter - - Labels: method, status - - Unit: requests - - Description: Requests whose normalized path could not be resolved (e.g., 404s). Useful for monitoring spikes in unresolved routes. - -## Database Metrics - -- Name: mreg_db_query_duration_seconds - - Type: Histogram - - Labels: method, path - - Unit: seconds - - Description: Duration of each DB query executed during a request. - -- Name: mreg_db_request_duration_seconds - - Type: Histogram - - Labels: method, path, status - - Unit: seconds - - Description: Total DB time aggregated per HTTP request. - -- Name: mreg_db_queries_per_request - - Type: Histogram - - Labels: method, path, status - - Unit: queries - - Description: Number of DB queries attempted during a single HTTP request (includes attempted queries even if they error). - - Buckets: [1, 2, 3, 5, 8, 13, 21, 34, 55] - -- Name: mreg_db_queries_total - - Type: Counter - - Labels: method, path - - Unit: queries - - Description: Total number of DB queries attempted across all requests. - -- Name: mreg_db_errors_total - - Type: Counter - - Labels: method, path, exception - - Unit: errors - - Description: Total number of DB errors, partitioned by exception class name. - -## LDAP Metrics - -- Name: mreg_ldap_call_duration_seconds - - Type: Histogram - - Labels: operation - - Unit: seconds - - Description: Duration of LDAP operations (initialize, bind, unbind) invoked by the health check. - - Buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5] - -- Name: mreg_ldap_call_failures_total - - Type: Counter - - Labels: operation, exception - - Unit: failures - - Description: LDAP operation failures by operation and exception class (e.g., bind LDAPError). Useful to see if LDAP is flapping or credential/ACL issues arise. - -## Policy Engine Metrics - -- Name: mreg_policy_decisions_total - - Type: Counter - - Labels: decision - - Unit: decisions - - Description: Policy-engine decisions recorded during parity checks. - - Label values: `allow`, `deny`, `error` - -- Name: mreg_policy_legacy_decisions_total - - Type: Counter - - Labels: decision - - Unit: decisions - - Description: Legacy permission decisions compared against policy parity. - - Label values: `allow`, `deny` - -- Name: mreg_policy_parity_results_total - - Type: Counter - - Labels: result - - Unit: comparisons - - Description: Outcome of legacy vs policy parity comparisons. - - Label values: `match`, `mismatch`, `error` - -- Name: mreg_policy_authorize_calls_total - - Type: Counter - - Labels: status - - Unit: calls - - Description: Calls made to the policy `authorize` endpoint. - - Label values: `success`, `exception` - -- Name: mreg_policy_parity_batches_total - - Type: Counter - - Labels: status - - Unit: batches - - Description: Durable outbox lifecycle events. - - Label values: `persisted`, `processed`, `retried`, `dead_letter`, `persist_failed` - -- Name: mreg_policy_parity_outbox_entries - - Type: Gauge - - Labels: status - - Description: Current shared outbox rows by `pending` or `dead_letter` status. - -- Name: mreg_policy_parity_outbox_oldest_seconds - - Type: Gauge - - Labels: none - - Unit: seconds - - Description: Age of the oldest pending durable batch. - -- Name: mreg_policy_parity_circuit_open - - Type: Gauge - - Labels: none - - Description: `1` while a worker's TreeTop delivery circuit is open, otherwise `0`. - -- Name: mreg_policy_parity_failures_total - - Type: Counter - - Labels: stage - - Unit: failures - - Description: Policy integration failures by processing stage. Shadow failures are fail-open; enforcement failures follow the configured failure mode. - - Typical label values: `shadow_build`, `persist`, `request_exit`, `worker`, `result_logging`, `enforce_build`, `enforce_authorize`, `enforce_result` - -- Name: mreg_policy_enforcement_results_total - - Type: Counter - - Labels: result - - Unit: decisions - - Description: Synchronous authoritative outcomes in `enforce` mode. - - Label values: `allow`, `deny`, `error_deny`, `error_legacy` - -- Name: mreg_policy_mode_info - - Type: Gauge - - Labels: mode - - Description: Configured policy mode for the worker. - - Label values: `off`, `shadow`, `enforce` - -- Name: mreg_policy_authorize_duration_seconds - - Type: Histogram - - Labels: status - - Unit: seconds - - Description: Duration of policy `authorize` calls. - - Buckets/ranges: `0-1ms`, `1-2.5ms`, `2.5-5ms`, `5-10ms`, `10-25ms`, `25-50ms`, `50-100ms`, `100-250ms`, `250-500ms`, `500ms-1s`, `1-2.5s`, `2.5-5s`, `5s+` - - Prometheus boundaries: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, +Inf] - -- Name: mreg_policy_requests_per_authorize - - Type: Histogram - - Labels: none - - Unit: policy requests - - Description: Number of policy requests included in each `authorize` call. - - Buckets/ranges: `0`, `1`, `2`, `3`, `4-5`, `6-8`, `9+` - - Prometheus boundaries: [0, 1, 2, 3, 5, 8, +Inf] - -- Name: mreg_policy_queries_per_request - - Type: Histogram - - Labels: none - - Unit: submitted batches - - Description: Number of shadow batches submitted or synchronous enforcement calls made by each HTTP request. - - Buckets/ranges: `0`, `1`, `2`, `3`, `4-5`, `6-8`, `9+` - - Prometheus boundaries: [0, 1, 2, 3, 5, 8, +Inf] - -In `shadow`, request batching normally produces `0` (no checks) or `1` (one -durable batch) and the worker authorizes it later. In `enforce`, the value is the -number of synchronous mapped authorization checkpoints reached by the request; -these cannot be delayed or combined after the request because their result -controls permission flow. - -## TreeTop rollout dashboard and alerts - -- Grafana dashboard: `monitoring/grafana/treetop-parity.json` -- Prometheus alerts: `monitoring/treetop-alerts.yml` -- Executable gate: `python manage.py check_policy_rollout --prometheus-url URL` - -The default gate requires at least 10,000 comparisons over the selected window, -at most 0.1% mismatches, at most 0.1% errors, zero persistence failures, zero -dead letters, zero pending shadow batches, and a pending backlog younger than -five minutes. - -`MregTreeTopEnforcementFailure` pages on any `error_deny` or `error_legacy` -outcome. `error_legacy` means TreeTop was not authoritative for that request and -should only exist during a deliberate transitional rollout. - -## Labeling Strategy - -- path: normalized using Django URL resolution to view name (preferred) or route pattern. Falls back to "unresolved" to avoid cardinality explosion from raw paths with IDs. -- method: HTTP method (GET, POST, etc.). -- status: HTTP status code as string (e.g., "200", "404"). -- exception: Python exception class name. We do not include messages or stack traces. - -## Notes - -- Timing uses monotonic clocks to avoid wall-clock skew. -- The metrics endpoint (/api/meta/metrics) is not instrumented and is tolerant to a trailing slash. -- Gauges are carefully paired to prevent underflow. -- The container configures Prometheus client multiprocess mode and cleans its - per-process files before Gunicorn starts. Custom process managers must set - `PROMETHEUS_MULTIPROC_DIR` to a clean, writable directory before Python - starts and call `prometheus_client.multiprocess.mark_process_dead` when a - worker exits. -- Avoid building dashboards/alerts on high-cardinality labels; stick to method/path/status/exception. - -## Alerting Examples - -- N+1 query detection - - Goal: Detect endpoints where average queries per request spike above a threshold. - - PromQL: - - Average queries per request per path/method over 5m: - `sum by (method, path) (rate(mreg_db_queries_per_request_sum[5m])) / sum by (method, path) (rate(mreg_db_queries_per_request_count[5m]))` - - Alert when `> 20` (tune to your baseline) - -- Payload size anomalies (response size) - - Goal: Detect endpoints returning unusually large payloads. - - PromQL: - - Average response size bytes per path/method over 5m: - `sum by (method, path) (rate(mreg_http_response_size_bytes_sum[5m])) / sum by (method, path) (rate(mreg_http_response_size_bytes_count[5m]))` - - Alert when `> 1048576` (1 MiB) or when deviates from a baseline (use recording rules or anomaly detection plugins) - -- 5xx spikes by view/exception - - Goal: Track and alert on failures grouped by normalized path and exception type. - - PromQL: - - 5xx rate per path/method over 5m: - `sum by (method, path) (rate(mreg_http_requests_total{status=~"5.."}[5m]))` - - Exceptions by type per path/method over 5m: - `sum by (method, path, exception) (rate(mreg_http_exceptions_total[5m]))` - - Alert on sustained spikes above baseline (e.g., `> 0.1 rps` for 10m) - -## Sample Prometheus alert rules (starter set) - -Tune thresholds to your baseline; these are illustrative. - -```yaml -groups: - - name: mreg-alerts - rules: - - alert: Mreg5xxSpike - expr: sum by (method, path) (rate(mreg_http_requests_total{status=~"5.."}[5m])) > 0.1 - for: 10m - labels: - severity: page - annotations: - summary: "5xx spike on {{ $labels.method }} {{ $labels.path }}" - - - alert: MregLDAPFailures - expr: sum by (operation, exception) (rate(mreg_ldap_call_failures_total[5m])) > 0 - for: 5m - labels: - severity: page - annotations: - summary: "LDAP failures {{ $labels.operation }} {{ $labels.exception }}" - - - alert: MregLDAPLatencyHigh - expr: histogram_quantile( - 0.95, - sum by (le) (rate(mreg_ldap_call_duration_seconds_bucket[5m])) - ) > 1 - for: 5m - labels: - severity: ticket - annotations: - summary: "LDAP latency p95 > 1s" - - - alert: MregNPlusOneSuspect - expr: ( - sum by (method, path) (rate(mreg_db_queries_per_request_sum[5m])) - / sum by (method, path) (rate(mreg_db_queries_per_request_count[5m])) - ) > 20 - for: 10m - labels: - severity: ticket - annotations: - summary: "High queries/request on {{ $labels.method }} {{ $labels.path }}" - - - alert: MregResponseSizeAnomaly - expr: ( - sum by (method, path) (rate(mreg_http_response_size_bytes_sum[5m])) - / sum by (method, path) (rate(mreg_http_response_size_bytes_count[5m])) - ) > 1048576 - for: 10m - labels: - severity: ticket - annotations: - summary: "Large responses on {{ $labels.method }} {{ $labels.path }} (>1MiB avg over 5m)" +MREG exposes Prometheus metrics at `/api/meta/metrics`. The endpoint itself is +not instrumented. Labels intentionally avoid usernames, raw URLs, query +parameters, remote addresses, SQL, and policy resource data. + +## HTTP and dependency metrics + +| Metric | Type | Labels | Meaning | +| --- | --- | --- | --- | +| `mreg_http_requests_total` | Counter | `method`, `path`, `status` | Requests by normalized route and status | +| `mreg_http_request_duration_seconds` | Histogram | `method`, `path`, `status` | End-to-end request latency | +| `mreg_http_inprogress_requests` | Gauge | `method`, `path` | Requests currently in flight | +| `mreg_http_request_size_bytes` | Histogram | `method`, `path` | Known request payload sizes | +| `mreg_http_response_size_bytes` | Histogram | `method`, `path`, `status` | Known non-streaming response sizes | +| `mreg_http_exceptions_total` | Counter | `method`, `path`, `exception` | Uncaught application exceptions | +| `mreg_http_unresolved_requests_total` | Counter | `method`, `status` | Requests whose route could not be normalized | +| `mreg_db_query_duration_seconds` | Histogram | `method`, `path` | Individual database query latency | +| `mreg_db_request_duration_seconds` | Histogram | `method`, `path`, `status` | Database time per request | +| `mreg_db_queries_per_request` | Histogram | `method`, `path`, `status` | Attempted queries per request | +| `mreg_db_queries_total` | Counter | `method`, `path` | Attempted database queries | +| `mreg_db_errors_total` | Counter | `method`, `path`, `exception` | Database errors | +| `mreg_ldap_call_duration_seconds` | Histogram | `operation` | LDAP health-check call latency | +| `mreg_ldap_call_failures_total` | Counter | `operation`, `exception` | LDAP health-check failures | + +`path` is a resolved view name or route pattern, never a raw object URL. Timing +uses monotonic clocks. + +## TreeTop metrics + +| Metric | Type | Labels | Meaning | +| --- | --- | --- | --- | +| `mreg_policy_decisions_total` | Counter | `decision` | Composite TreeTop result: `allow`, `deny`, or `error` | +| `mreg_policy_legacy_decisions_total` | Counter | `decision` | Composite legacy result used for comparison | +| `mreg_policy_parity_results_total` | Counter | `result` | Endpoint comparison: `match`, `mismatch`, or `error` | +| `mreg_policy_authorize_calls_total` | Counter | `status` | Synchronous authorize calls: `success` or `exception` | +| `mreg_policy_authorize_duration_seconds` | Histogram | `status` | Synchronous authorize latency | +| `mreg_policy_failures_total` | Counter | `stage` | Integration failures, currently the `authorize` stage | +| `mreg_policy_enforcement_results_total` | Counter | `result` | Authoritative `allow`, `deny`, or fail-closed `error_deny` | +| `mreg_policy_mode_info` | Gauge | `mode` | Active `off`, `shadow`, or `enforce` mode | +| `mreg_policy_stack_size` | Histogram | none | Cedar leaves in the endpoint stack sent by one call | +| `mreg_policy_authorize_calls_per_request` | Histogram | none | TreeTop HTTP calls per MREG request; protected requests should be `1` | +| `mreg_policy_circuit_open` | Gauge | none | Whether a worker's synchronous circuit is open | + +All protected endpoint checks are synchronous in both active modes. `shadow` +returns the legacy result after recording the comparison; `enforce` returns the +TreeTop composite and fails closed. There is no queue, retry worker, persistence +metric, and enforcement failures never return the legacy decision. + +The two design-invariant metrics are: + +- `mreg_policy_authorize_calls_per_request`: alert if observations exceed one. +- `mreg_policy_stack_size`: identify high-count endpoints whose semantic policy + can be simplified even though transport is already consolidated. + +## Rollout dashboard, alerts, and gate + +- Dashboard: `monitoring/grafana/treetop-parity.json` +- Rules: `monitoring/treetop-alerts.yml` +- Gate: `python manage.py check_policy_rollout --prometheus-url URL` + +The default gate requires at least 10,000 endpoint comparisons, no more than +0.1% mismatches, and no more than 0.1% errors over the selected window. The +alerts cover mismatch/error rates, any authoritative failure, an open circuit, +and violations of the one-call-per-request invariant. + +Useful PromQL: + +```promql +# Policy mismatch rate +sum(rate(mreg_policy_parity_results_total{result="mismatch"}[30m])) +/ +clamp_min(sum(rate(mreg_policy_parity_results_total{result=~"match|mismatch"}[30m])), 1) + +# p95 synchronous TreeTop latency +histogram_quantile( + 0.95, + sum by (le) (rate(mreg_policy_authorize_duration_seconds_bucket[5m])) +) + +# Average checks in each endpoint stack +rate(mreg_policy_stack_size_sum[5m]) +/ +clamp_min(rate(mreg_policy_stack_size_count[5m]), 1) ``` -## What's intentionally not labeled - -Almost all labels that could lead to high cardinality or sensitive data exposure are avoided, including but not limited to: - -- User-specific labels (e.g., user ID) to prevent cardinality explosion and privacy concerns. -- Query parameters in paths to avoid high cardinality from unique URLs. -- Remote IP addresses for privacy and cardinality reasons. -- Detailed SQL query information to prevent high cardinality and sensitive data exposure. +The container configures Prometheus multiprocess mode and clears its directory +before Gunicorn starts. Other process managers must provide a clean writable +`PROMETHEUS_MULTIPROC_DIR` and call +`prometheus_client.multiprocess.mark_process_dead` when a worker exits. diff --git a/docs/parity_testing.md b/docs/parity_testing.md index 34fb02e8..6d043ef1 100644 --- a/docs/parity_testing.md +++ b/docs/parity_testing.md @@ -90,18 +90,16 @@ contexts and concurrently handled requests are isolated from one another. It disables only `shadow` checks; it is deliberately ignored in `enforce` so test or application code cannot bypass an authoritative decision accidentally. -In `shadow`, parity batches are persisted in a shared PostgreSQL outbox. -Post-fork workers claim rows with database locks, retry with exponential -backoff, and retain dead letters after the configured attempt limit. A circuit -breaker protects an unavailable TreeTop service. Client, serialization, -persistence, logging, and TreeTop failures remain fail-open and never replace -the legacy decision. - -The outbox and its worker exist only in `shadow`. In `enforce`, each mapped -permission checkpoint calls TreeTop synchronously, records the comparison -immediately, and returns the policy decision. No enforcement request is queued -for later re-authorization. Enforcement failures deny by default; the explicit -`legacy` failure mode is available only as a transitional fallback. +In both `shadow` and `enforce`, one complete endpoint stack is sent +synchronously in one `authorize` call. A stack can contain nested AND/OR rules; +TreeTop evaluates all leaves and MREG composes their results locally. `shadow` +records the comparison and returns the legacy result. `enforce` returns the +TreeTop result and fails closed on every integration failure. + +The request scope rejects a second different stack, making accidental +checkpoint-by-checkpoint calls visible during development instead of quietly +adding request-path latency. A thread-safe circuit breaker prevents every +request from waiting for the full timeout during an outage. ## Parity Runbook @@ -122,13 +120,13 @@ mreg_policy_parity_results_total{result="mismatch"} 3. List mismatch events in the configured application log. ```bash -rg -n '"event": "policy_parity_mismatch"' logs/app.log +rg -n '"event": "policy_stack_result".*"parity": false' logs/app.log ``` 4. Optional: inspect actions seen in mismatch events. ```bash -jq -r 'select(.event == "policy_parity_mismatch") | .context.action // empty' logs/app.log \ +jq -r 'select(.event == "policy_stack_result" and .parity == false) | .context.path' logs/app.log \ | sort | uniq -c | sort -nr ``` @@ -150,8 +148,9 @@ Do not enable enforcement until this command passes. Import ## Mismatch Triage Guide -Use `legacy_decision`, `policy_decision`, and `context.action`. Detailed resource -attributes are available only when `MREG_POLICY_PARITY_LOG_DETAILS` is enabled. +Use `legacy_decision`, `policy_decision`, and the request context. Detailed +leaf actions and resource attributes are available only when +`MREG_POLICY_PARITY_LOG_DETAILS` is enabled. - `legacy_decision=true`, `policy_decision=false`: - Missing/too-narrow Cedar allow rule. diff --git a/docs/policies.md b/docs/policies.md index 06350b37..feafa5f6 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -1,236 +1,191 @@ +# TreeTop Authorization -# Policy Actions +MREG can evaluate authorization with the TreeTop Cedar policy engine. The +integration is request-scoped, synchronous, bundle-based, and has three modes: -This section documents the policy actions currently used by MREG permission checks. - -Related documentation: - -- Parity test workflow and triage: [`parity_testing.md`](./parity_testing.md) +| Mode | TreeTop call | Returned decision | +| --- | --- | --- | +| `off` | none | legacy MREG permission | +| `shadow` | one synchronous call per protected endpoint | legacy MREG permission | +| `enforce` | one synchronous call per protected endpoint | TreeTop composite; errors deny | + +An empty `MREG_POLICY_BASE_URL` makes the default `shadow` mode behave like +`off`. `enforce` requires a URL at startup and has no legacy error fallback. +Authentication remains local; token acquisition, health checks, metrics, schema, +and admin pages are the explicit policy exemptions. + +## Why there is no queue or async dispatcher + +Authorization must complete before request processing can continue. Queuing the +work would either allow an unauthorised request to proceed or still require the +request to wait for the queue result. An async HTTP client would change how the +thread waits, not remove the dependency. The DRF/Gunicorn application is +synchronous, so MREG uses the synchronous `treetop-client` API directly. + +Shadow mode also waits. This ensures its comparison uses the policy bundle that +was active for the request and exercises the exact latency, timeout, circuit, +and response-validation path that enforcement will use. The former PostgreSQL +outbox, migration, dispatcher, retry/dead-letter state, and Gunicorn background +thread are intentionally absent. + +## One endpoint stack and one HTTP call + +An endpoint builds a tree of `PolicyLeaf`, `PolicyAll`, and `PolicyAny` nodes. +Every leaf is included in one batched `authorize` request. MREG then composes the +ordered results locally using the tree's AND/OR structure. Examples include: + +- all old and new targets required for a hostname rename; +- any IP attached to a host matching a NetGroup rule; +- any host-policy role label matching the host's derived labels; +- DNS-name, reserved-address, ownership, and target facts in the same endpoint + decision. + +The request scope caches an identical repeated stack and rejects a second +different stack. `mreg_policy_authorize_calls_per_request` makes violations of +the one-call invariant observable. + +## Principal, action, resource, and facts + +Each leaf sends: + +- a qualified principal such as `MREG::User::"alice"`, with current group + memberships; +- one explicit action such as `MREG::Action::"host_update"`; +- a typed resource such as `MREG::Host::"host.example.org"`; +- raw facts needed by Cedar, including hostname, IP, network, DNS-name shape, + target/self relationship, host-group ownership, or host-policy role label. + +MREG does not send a precomputed `allow` fact. Relationship booleans such as +`selfAccess` and `requesterIsOwner` describe request state; Cedar decides what +those facts mean. + +`mreg/policy/contracts.py` is the dependency-free source of truth for resource +kinds, optional attributes, operations, and actions. `mreg/policy/resources.py` +resolves model/view data to stable IDs and normalized attributes. Unknown kinds +must be registered explicitly; view class names are not an authority fallback. +The generated schema is checked in CI. + +## Mapping mutable database permissions + +User group membership remains dynamic and is sent on every call. Mutable +`NetGroupRegexPermission` rows cannot remain an independent authority when +TreeTop is authoritative. Their equivalents must be reviewed and added to the +deployed bundle: + +| Database field | Bundle representation | +| --- | --- | +| `group` | Cedar principal group | +| `range` | Cedar `ip.isInRange(...)` or exact network condition | +| `regex` | named pattern in `labels.json` | +| `labels` | derived label name used by Cedar/host-policy rules | + +TreeTop applies all regexes in the bundle to the raw `hostname` fact and adds +`nameLabels`. Cedar checks labels such as `netgroup_example_org`; MREG neither +runs the bundle regex nor invents the label. This is the intended TreeTop label +boundary. + +In `enforce`, the NetGroupRegexPermission API remains readable but returns HTTP +409 for POST, PUT, PATCH, and DELETE. This prevents the database from appearing +to change authoritative policy. In `off` and `shadow`, writes retain their +legacy behavior so policy authors can stage and compare a migration. Bundle +publication is a separate reviewed deployment operation. + +## Local responsibilities and Cedar responsibilities + +MREG still owns authentication, serializer validation, object lookup, database +transactions, conflicts, and business invariants. Cedar owns authorization for +protected endpoints in `enforce`, including: + +- authenticated reads and explicit introspection actions; +- super/admin/network/group/host-policy roles; +- host, record, BACnet, network, community, zone, label, and host-policy CRUD; +- NetGroup hostname/range rules through derived labels; +- DNS wildcard/underscore restrictions; +- restricted IP assignment; +- host-group ownership and membership changes; +- host-policy role-to-host label matching. + +## Failure behavior + +The timeout defaults to five seconds. Each Gunicorn worker owns a reusable +client and a thread-safe closed/open/half-open circuit breaker. After the +configured consecutive failures, the circuit rejects calls until its cooldown; +one request then probes the service. + +- `shadow`: log/metric the error and return the legacy result. +- `enforce`: log at critical severity, increment `error_deny`, and deny. + +Malformed result counts and per-result errors are failures just like transport +exceptions. `disable_policy_parity()` can suppress only shadow calls in narrow +test scopes; it cannot bypass enforcement. + +## Bundle source and build + +| Artifact | Path | +| --- | --- | +| Organization manifest | `treetop/data/treetop-bundle.toml` | +| MREG module manifest | `treetop/data/treetop-mreg-module.toml` | +| Global module manifest | `treetop/data/treetop-global-module.toml` | +| Cedar policy | `treetop/data/mreg.cedar` | +| Global super policy | `treetop/data/global.cedar` | +| Derived labels | `treetop/data/labels.json` | +| Generated schema | `treetop/data/mreg.cedarschema` | +| Generated archive | `treetop/data/mreg-bundle.tar.gz` | + +Build with `treetop-bundle` 0.0.5: + +```bash +python scripts/generate-treetop-schema.py --check +treetop-bundle check bundle treetop/data/treetop-bundle.toml +treetop-bundle build \ + --manifest treetop/data/treetop-bundle.toml \ + --output treetop/data/mreg-bundle.tar.gz +TREETOP_BUNDLE_BIN=treetop-bundle scripts/check-treetop-bundle.sh +``` -## Source of Truth +Bundle output is deterministic and CI compares it byte-for-byte. Bundles are +currently unsigned; the development server explicitly uses +`TREETOP_BUNDLE_SIGNATURE_POLICY=allow-unsigned`. -- Bundle manifest: `treetop/data/treetop-bundle.toml` -- Policy module: `treetop/data/treetop-mreg-module.toml` -- Global policy module: `treetop/data/treetop-global-module.toml` -- Policy definitions: `treetop/data/mreg.cedar` and `treetop/data/global.cedar` -- Python resource/action contracts: `mreg/policy/contracts.py` -- Typed resource adapters: `mreg/policy/resources.py` -- Generated Cedar schema: `treetop/data/mreg.cedarschema` -- Derived labels: `treetop/data/labels.json` -- Generated bundle: `treetop/data/mreg-bundle.tar.gz` -- Parity transport/logging: `mreg/api/treetop.py` +No `treetop-client` change is required for bundle support. MREG sends ordinary +authorization requests to `treetop-rest`; the REST server downloads, validates, +atomically loads, and refreshes the bundle. -## Building the Bundle +## Local setup and rollout -Install `treetop-bundle` 0.0.5 from the -[`treetop-bundle` releases](https://github.com/treetop-policy-engine/treetop-bundle/releases/tag/v0.0.5), -which matches the bundle format and Treetop Core version supported by the -pinned REST server. Then validate and build the bundle from the repository -root: +Start `treetop-rest` 0.0.14 and the bundle file server: -```console -$ python scripts/generate-treetop-schema.py --check -$ treetop-bundle build \ - --manifest treetop/data/treetop-bundle.toml \ - --output treetop/data/mreg-bundle.tar.gz -$ TREETOP_BUNDLE_BIN=treetop-bundle scripts/check-treetop-bundle.sh +```bash +docker compose -f treetop/docker-compose.yml up -d ``` -The schema's entities and action declarations are generated from the Python -contracts. Add a `ResourceContract` and its adapter before changing policies; -CI rejects a stale generated schema. Bundle output is deterministic. Commit the -regenerated archive whenever a module manifest, Cedar policy, contract/schema, -or label definition changes. The local -TreeTop stack loads the archive atomically through `TREETOP_BUNDLE_URL`. MREG -currently uses unsigned bundles, verified with the explicit `allow-unsigned` -signature policy. - -## Adding a New Protected Resource - -When introducing a new resource that should be parity-checked, use this checklist: - -1. Ensure the permission path reaches `ParityMixin.pp()` or `pp_generic_action()`. -2. Add a `ResourceContract` and registered adapter in `mreg/policy/`. -3. Confirm CRUD action dispatch is used (`_`). -4. Define the resource kind contract for the endpoint: - - Use serializer `Meta.model` for model-backed views. - - Set `policy_resource_kind` explicitly on non-model views. - - Set a `policy_actions` operation mapping when an endpoint action is not - the model's conventional CRUD action. -5. Verify resource ID resolution produces stable IDs for list/detail/custom views. -6. Regenerate `treetop/data/mreg.cedarschema` and update Cedar rules. -7. If policy conditions depend on derived labels, update `treetop/data/labels.json`. -8. Rebuild `treetop/data/mreg-bundle.tar.gz`. -9. Add tests for create/read/update/delete behavior and group/admin overrides. -10. Run parity checks and confirm zero mismatches. -11. If tests mutate permissions mid-test, scope `disable_policy_parity()` as narrowly as possible. - -## Enforcement Mapping Boundary - -`MREG_POLICY_MODE=enforce` makes TreeTop synchronous and authoritative wherever -the permission path reaches `ParityMixin.pp()` or `pp_generic_action()`. The -return value at that checkpoint becomes the TreeTop decision. Authentication, -serializer validation, object lookup, and business invariants remain MREG -responsibilities. - -The mapping unit is a semantic permission checkpoint, not simply an HTTP -request. One request can reach multiple checks (for example DNS-name rules, -reserved-address rules, and a final host/IP permission). Those checks cannot be -batched after the request in enforcement mode because each result may control -the next branch. They therefore use synchronous `treetop-client.authorize` -calls. The PostgreSQL queue remains only for asynchronous `shadow` comparisons. - -The current DRF permission stack and Gunicorn workers are synchronous, and the -result is needed before permission evaluation can continue. Using an async HTTP -client would still require blocking at that boundary and would not make the -decision asynchronous. A future end-to-end ASGI conversion could await TreeTop, -but it would still be request-path I/O in `enforce`. - -Legacy permission code is still evaluated in `enforce` so its result can be -compared and so the existing control flow can reach the mapped checkpoint. The -TreeTop result returned by that checkpoint is authoritative. Once the mapping -inventory is complete, legacy computation can be removed or reduced in a -separate change. Until then, use `mreg_policy_queries_per_request` and authorize -latency histograms to find endpoints where multiple dependent checks should be -redesigned into one explicit endpoint-level policy decision. - -Current mapped checkpoints include: - -| Legacy decision | Policy mapping | -| --- | --- | -| Administrative group membership | Explicit `*_admin_access` action on `Generic` | -| CRUD permission after serializer/object resolution | Typed resource plus `_` | -| Host/network regex evaluation | `Host`/record resource with `hostname` and optional typed `ip` | -| DNS wildcard/underscore rules | Explicit membership actions | -| Restricted IP operations | Explicit IP-management actions | -| Host contact reads | Explicit `host_contacts_read` view action | - -This is an incremental mapping boundary, not yet proof that every endpoint in -MREG is policy-backed. Plain `IsAuthenticated` endpoints, host-group ownership, -and legacy branches that do not call the parity mixin remain application-owned -until they receive an explicit contract and Cedar rule. Do not describe a -deployment as globally TreeTop-authoritative until an endpoint inventory shows -that every authorization decision intended for delegation reaches a mapped -checkpoint. MREG authentication and non-authorization validation are expected -to remain local. - -Dynamic user group membership is sent with every TreeTop request. Mutable -database permission rules such as `NetGroupRegexPermission` are not exported -automatically; their Cedar equivalent must be present in the deployed bundle. -This bundle-sync requirement must be part of the mapping/deployment process -before enforcement is enabled. - -## Resource Kind and ID Resolution - -`ParityMixin` delegates resource kind, ID, and attributes to the typed adapters -in `mreg/policy/resources.py`. - -Resource kind fallback order (`_resource_kind_from_view`): - -1. `obj.__class__.__name__` when object is available -2. `validated_serializer.Meta.model.__name__` -3. `validated_serializer.instance.__class__.__name__` -4. Explicit `view.policy_resource_kind` -5. `view.get_serializer_class().Meta.model.__name__` - -There is no view-class-name fallback. Renaming a view must not silently change -authorization behavior. Resource and principal entity types are qualified in -wire requests and the schema, for example `MREG::Host` and `MREG::User`. - -Custom actions are declared explicitly on views. For example, the host contacts -endpoint uses `policy_resource_kind = "Host"` with -`policy_actions = {"read": "host_contacts_read"}`. - -Resource ID fallback order (`_resource_id_from_view`): - -1. Object attributes: `pk`, `id`, `name` -2. Request/serializer data keys: `pk`, `id`, `name` -3. `validated_serializer.instance` attributes: `pk`, `id`, `name` -4. URL kwargs: `pk`, `id`, `name`, `cpk`, `hostpk`, `network` -5. Default `"any"` - -## CRUD Action Naming - -For model-backed checks, action names are generated as: - -`_` - -Where operation is mapped from HTTP method: - -- `GET`, `HEAD`, `OPTIONS` -> `read` -- `POST` -> `create` -- `PUT`, `PATCH` -> `update` -- `DELETE` -> `delete` - -## CRUD Actions Declared in Cedar - -- `host_create`, `host_read`, `host_update`, `host_delete` -- `host_contacts_read` -- `ipaddress_create`, `ipaddress_read`, `ipaddress_update`, `ipaddress_delete` -- `cname_create`, `cname_read`, `cname_update`, `cname_delete` -- `hinfo_create`, `hinfo_read`, `hinfo_update`, `hinfo_delete` -- `loc_create`, `loc_read`, `loc_update`, `loc_delete` -- `mx_create`, `mx_read`, `mx_update`, `mx_delete` -- `naptr_create`, `naptr_read`, `naptr_update`, `naptr_delete` -- `name_server_create`, `name_server_read`, `name_server_update`, `name_server_delete` -- `ptr_override_create`, `ptr_override_read`, `ptr_override_update`, `ptr_override_delete` -- `sshfp_create`, `sshfp_read`, `sshfp_update`, `sshfp_delete` -- `srv_create`, `srv_read`, `srv_update`, `srv_delete` -- `txt_create`, `txt_read`, `txt_update`, `txt_delete` -- `bacnet_id_create`, `bacnet_id_read`, `bacnet_id_update`, `bacnet_id_delete` -- `community_create`, `community_read`, `community_update`, `community_delete` - -## Non-CRUD Actions Declared in Cedar - -- `admin_access` -- `network_admin_access` -- `hostgroup_admin_access` -- `hostpolicy_admin_access` -- `dns_wildcard_admin_access` -- `dns_underscore_admin_access` -- `ip_gw_management` -- `ip_broadcast_management` -- `ip_network_management` -- `ip_reserved_management` -- `ip_restricted_management` -- `create_label` -- `delete_label` -- `view_label` -- `edit_label` - -## Attribute Contract for Policy Checks - -All resource attributes are normalized through the registered resource adapter: - -- `kind` is always added using snake_case resource kind. -- Attribute values are stringified. -- In `policy_parity`, string values that parse as IPs are sent as IP-typed attributes, otherwise as string attributes. - -Common attribute payloads in current checks: - -| Context | Typical action(s) | Attributes sent | -| --- | --- | --- | -| Safe/read precheck in `IsGrantedNetGroupRegexPermission.has_permission` | `_read` | `kind`, `path` | -| Host/IP netgroup evaluation (`has_perm`) | CRUD action from method | `kind`, `hostname`, optional `ip` | -| Create admin parity check | `_create` | `kind` + flattened serializer data | -| Update admin parity check | `_update` | `kind` + stringified validated data | -| Destroy admin parity check | `_delete` | `kind`, `id` | - -## Wildcard Action Rules - -These global-module rules do not enumerate action names and therefore match any -action: +Observe synchronously first: -- `global.mreg_superadmin`: principal in `MREG::Group::"default-super-group"` - may perform any action. -- `global.super_admin_allow_all_policy`: principal `MREG::User::"super"` may - perform any action. - -## Code-Emitted Parity Actions +```bash +export MREG_POLICY_MODE=shadow +export MREG_POLICY_BASE_URL=http://localhost:9999 +export MREG_POLICY_NAMESPACE=MREG +``` -The code also emits parity checks for: +After the bundle mapping is reviewed and the rollout gate passes, enable +authority and restart all workers: -- `superuser_access` -- `is_superuser` +```bash +export MREG_POLICY_MODE=enforce +``` -These are covered by wildcard superadmin rules in Cedar. +Roll back by setting the mode to `shadow` or `off` and restarting workers. If +MREG itself runs in a container, use a TreeTop URL reachable from that +container—not its own `localhost`. + +## Adding a protected endpoint + +1. Register its typed resource contract and any custom action. +2. Choose the final semantic authorization point. Use the early DRF permission + hook only when all facts are available there; otherwise authorize after + serializer/object resolution. +3. Build the complete AND/OR stack and call `authorize_policy_stack()` once. +4. Add Cedar permits/forbids and derived label rules together. +5. Regenerate the schema and archive. +6. Test legacy behavior, shadow comparison, enforce allow/deny/error behavior, + and the one-call invariant. diff --git a/hostpolicy/api/permissions.py b/hostpolicy/api/permissions.py index c7f8c418..6aa0cccb 100644 --- a/hostpolicy/api/permissions.py +++ b/hostpolicy/api/permissions.py @@ -1,5 +1,7 @@ -from rest_framework.permissions import IsAuthenticated, SAFE_METHODS +from rest_framework.permissions import SAFE_METHODS +from mreg.api.permissions import IsAuthenticated +from mreg.api.treetop import authorize_policy_stack, policy_any, policy_leaf from mreg.models.auth import User from mreg.models.host import Host from mreg.models.network import NetGroupRegexPermission @@ -14,59 +16,128 @@ class IsSuperOrHostPolicyAdminOrReadOnly(IsAuthenticated): def has_permission(self, request, view): if not super().has_permission(request, view): - # Not even reading is allowed if you're not authenticated return False - + user = User.from_request(request) - if request.method in SAFE_METHODS: - return True - if user.is_mreg_superuser_or_hostpolicy_admin: - return True + legacy = True + elif user.is_mreg_superuser_or_hostpolicy_admin: + legacy = True + else: + legacy = self._legacy_role_host_permission(request, view) - # Handle the (possible) absence of 'name' during schema generation - name = view.kwargs.get('name') - if name is None: # pragma: no cover - return False + if request.method not in SAFE_METHODS and view.__class__.__name__ in { + "HostPolicyRoleHostsDetail", + "HostPolicyRoleHostsList", + }: + return self._authorize_role_host_membership( + request=request, + view=view, + legacy=legacy, + ) + if request.method not in SAFE_METHODS and view.__class__.__name__ in { + "HostPolicyRoleAtomsDetail", + "HostPolicyRoleAtomsList", + }: + role_name = str(view.kwargs.get("name") or "any") + return authorize_policy_stack( + legacy, + request=request, + root=policy_leaf( + action="hostpolicy_role_atom_membership_update", + resource_kind="HostPolicyRole", + resource_id=role_name, + resource_attrs={"kind": "host_policy_role", "name": role_name}, + ), + view=view, + permission_class=self.__class__.__name__, + ) + return self.authorize_endpoint( + legacy_decision=legacy, + request=request, + view=view, + data=request.data if isinstance(request.data, dict) else None, + fallback_action="hostpolicy_admin_access", + ) - # Is this request about atoms or something else that isn't a role? - # In that case, non-admin-users shouldn't have access anyway, and we can deny the request. - if not (view.__class__.__name__ == 'HostPolicyRoleHostsDetail' or - view.__class__.__name__ == 'HostPolicyRoleHostsList'): - return False + def _authorize_role_host_membership(self, *, request, view, legacy: bool) -> bool: + role_name = str(view.kwargs.get("name") or "") + hostname = str(view.kwargs.get("host") or request.data.get("name") or "") + role_labels = tuple( + HostPolicyRole.objects.filter(name=role_name).values_list( + "labels__name", flat=True + ) + ) + ips = tuple( + str(ip) + for ip in Host.objects.filter(name=hostname) + .exclude(ipaddresses__ipaddress=None) + .values_list("ipaddresses__ipaddress", flat=True) + ) + leaves = tuple( + policy_leaf( + action="hostpolicy_role_host_membership_update", + resource_kind="Host", + resource_id=hostname or "any", + resource_attrs={ + "kind": "host", + "name": hostname, + "hostname": hostname, + "ip": ip, + "roleLabel": str(label), + }, + ) + for label in role_labels + for ip in ips + ) + root = ( + policy_any(*leaves) + if leaves + else policy_leaf( + action="hostpolicy_role_host_membership_update", + resource_kind="Host", + resource_id=hostname or "any", + resource_attrs={ + "kind": "host", + "name": hostname, + "hostname": hostname, + }, + ) + ) + return authorize_policy_stack( + legacy, + request=request, + root=root, + view=view, + permission_class=self.__class__.__name__, + ) - # Find out which labels are attached to this role - role_labels = HostPolicyRole.objects.filter(name=name).values_list('labels__name', flat=True) + @staticmethod + def _legacy_role_host_permission(request, view) -> bool: + name = view.kwargs.get("name") + if name is None: # pragma: no cover + return False + if view.__class__.__name__ not in { + "HostPolicyRoleHostsDetail", + "HostPolicyRoleHostsList", + }: + return False + role_labels = HostPolicyRole.objects.filter(name=name).values_list("labels__name", flat=True) if not any(role_labels): - # if the role doesn't have any labels, there's no possibility of access at this point return False - - # Find all the NetGroupRegexPermission objects that correspond with - # the ipaddress, hostname, and the groups that the user is a member of - # Also, ensure that the hostname is not empty. - hostname = view.kwargs.get('host', request.data.get("name")) - if not hostname: # pragma: no cover + hostname = view.kwargs.get("host", request.data.get("name")) + if not hostname: # pragma: no cover return False - - ips = list(Host.objects.filter( - name=hostname - ).exclude( - ipaddresses__ipaddress=None - ).values_list('ipaddresses__ipaddress', flat=True)) - qs = NetGroupRegexPermission.find_perm(request.user.group_list, hostname, ips) - - # If no permissions matched the host/ip, we deny access - if not qs.exists(): + ips = list( + Host.objects.filter(name=hostname) + .exclude(ipaddresses__ipaddress=None) + .values_list("ipaddresses__ipaddress", flat=True) + ) + permissions = NetGroupRegexPermission.find_perm(request.user.group_list, hostname, ips) + if not permissions.exists(): return False - - # Do any of those permissions have labels that match the labels attached to this role? - # If so, access is granted - perm_labels = qs.values_list('labels__name', flat=True) - if any(label in perm_labels for label in role_labels): - return True - - # If the code got to this point, it means none of the labels matched. - return False + permission_labels = permissions.values_list("labels__name", flat=True) + return any(label in permission_labels for label in role_labels) def has_m2m_change_permission(self, request, view): return True diff --git a/monitoring/grafana/treetop-parity.json b/monitoring/grafana/treetop-parity.json index c43af14a..a3b78461 100644 --- a/monitoring/grafana/treetop-parity.json +++ b/monitoring/grafana/treetop-parity.json @@ -4,7 +4,7 @@ "panels": [ { "id": 1, - "title": "Parity mismatch rate", + "title": "Endpoint parity mismatch rate", "type": "timeseries", "targets": [{"expr": "sum(rate(mreg_policy_parity_results_total{result=\"mismatch\"}[5m])) / clamp_min(sum(rate(mreg_policy_parity_results_total{result=~\"match|mismatch\"}[5m])), 1)", "legendFormat": "mismatch"}], "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"steps": [{"color": "green", "value": null}, {"color": "red", "value": 0.001}]}}, "overrides": []}, @@ -12,7 +12,7 @@ }, { "id": 2, - "title": "Parity error rate", + "title": "Endpoint parity error rate", "type": "timeseries", "targets": [{"expr": "sum(rate(mreg_policy_parity_results_total{result=\"error\"}[5m])) / clamp_min(sum(rate(mreg_policy_parity_results_total[5m])), 1)", "legendFormat": "errors"}], "fieldConfig": {"defaults": {"unit": "percentunit", "thresholds": {"steps": [{"color": "green", "value": null}, {"color": "red", "value": 0.001}]}}, "overrides": []}, @@ -20,24 +20,25 @@ }, { "id": 3, - "title": "Outbox entries", + "title": "Authorize calls per request", "type": "timeseries", - "targets": [{"expr": "max by (status) (mreg_policy_parity_outbox_entries)", "legendFormat": "{{status}}"}], + "targets": [{"expr": "rate(mreg_policy_authorize_calls_per_request_sum[5m]) / clamp_min(rate(mreg_policy_authorize_calls_per_request_count[5m]), 1)", "legendFormat": "average"}], + "fieldConfig": {"defaults": {"thresholds": {"steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}, "overrides": []}, "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8} }, { "id": 4, - "title": "Oldest pending batch", - "type": "stat", - "targets": [{"expr": "max(mreg_policy_parity_outbox_oldest_seconds)"}], - "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"steps": [{"color": "green", "value": null}, {"color": "red", "value": 300}]}}, "overrides": []}, + "title": "Checks per endpoint stack", + "type": "timeseries", + "targets": [{"expr": "rate(mreg_policy_stack_size_sum[5m]) / clamp_min(rate(mreg_policy_stack_size_count[5m]), 1)", "legendFormat": "average"}], "gridPos": {"h": 8, "w": 8, "x": 8, "y": 8} }, { "id": 5, - "title": "Delivery lifecycle", + "title": "Authorize p95 latency", "type": "timeseries", - "targets": [{"expr": "sum by (status) (rate(mreg_policy_parity_batches_total[5m]))", "legendFormat": "{{status}}"}], + "targets": [{"expr": "histogram_quantile(0.95, sum by (le) (rate(mreg_policy_authorize_duration_seconds_bucket[5m])))", "legendFormat": "p95"}], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8} }, { @@ -52,14 +53,22 @@ "title": "Enforcement decisions", "type": "timeseries", "targets": [{"expr": "sum by (result) (rate(mreg_policy_enforcement_results_total[5m]))", "legendFormat": "{{result}}"}], - "gridPos": {"h": 8, "w": 16, "x": 8, "y": 16} + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 16} + }, + { + "id": 8, + "title": "Worker circuit", + "type": "stat", + "targets": [{"expr": "max(mreg_policy_circuit_open)", "legendFormat": "open"}], + "fieldConfig": {"defaults": {"thresholds": {"steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 16} } ], "schemaVersion": 41, "tags": ["mreg", "treetop", "rollout"], "templating": {"list": []}, "time": {"from": "now-24h", "to": "now"}, - "title": "MREG TreeTop parity rollout", + "title": "MREG TreeTop endpoint rollout", "uid": "mreg-treetop-parity", - "version": 1 + "version": 2 } diff --git a/monitoring/treetop-alerts.yml b/monitoring/treetop-alerts.yml index f8708730..b9bc323d 100644 --- a/monitoring/treetop-alerts.yml +++ b/monitoring/treetop-alerts.yml @@ -11,7 +11,7 @@ groups: labels: severity: page annotations: - summary: TreeTop parity mismatch rate exceeds 0.1% + summary: TreeTop endpoint parity mismatch rate exceeds 0.1% - alert: MregTreeTopParityErrorRateHigh expr: | sum(rate(mreg_policy_parity_results_total{result="error"}[30m])) @@ -22,39 +22,29 @@ groups: labels: severity: page annotations: - summary: TreeTop parity error rate exceeds 0.1% - - alert: MregTreeTopParityPersistenceFailure - expr: increase(mreg_policy_parity_batches_total{status="persist_failed"}[5m]) > 0 - for: 0m - labels: - severity: page - annotations: - summary: A policy parity batch could not be persisted - - alert: MregTreeTopParityDeadLetter - expr: max(mreg_policy_parity_outbox_entries{status="dead_letter"}) > 0 - for: 0m + summary: TreeTop endpoint parity error rate exceeds 0.1% + - alert: MregTreeTopCircuitOpen + expr: max(mreg_policy_circuit_open) > 0 + for: 2m labels: severity: page annotations: - summary: The policy parity outbox contains dead letters - - alert: MregTreeTopParityBacklogStale - expr: max(mreg_policy_parity_outbox_oldest_seconds) > 300 - for: 10m + summary: A synchronous TreeTop worker circuit is open + - alert: MregTreeTopMultipleCallsPerRequest + expr: | + sum(rate(mreg_policy_authorize_calls_per_request_count[5m])) + - + sum(rate(mreg_policy_authorize_calls_per_request_bucket{le="1.0"}[5m])) + > 0 + for: 5m labels: severity: page annotations: - summary: The oldest policy parity batch is over five minutes old - - alert: MregTreeTopParityCircuitOpen - expr: max(mreg_policy_parity_circuit_open) > 0 - for: 2m - labels: - severity: ticket - annotations: - summary: A policy parity delivery circuit breaker is open + summary: A request made more than one TreeTop authorize call - alert: MregTreeTopEnforcementFailure - expr: increase(mreg_policy_enforcement_results_total{result=~"error_.*"}[5m]) > 0 + expr: increase(mreg_policy_enforcement_results_total{result="error_deny"}[5m]) > 0 for: 0m labels: severity: page annotations: - summary: An authoritative TreeTop decision failed + summary: An authoritative TreeTop request failed closed diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index ba0f11d6..9288a829 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -1,7 +1,7 @@ from __future__ import annotations import ipaddress -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any from rest_framework import exceptions from rest_framework.permissions import IsAuthenticated as DRFIsAuthenticated, SAFE_METHODS @@ -11,11 +11,21 @@ from mreg.api.responses import error_body from mreg.api.v1.serializers import HostSerializer -from mreg.models.host import HostGroup +from mreg.models.host import Host, HostGroup from mreg.models.network import NetGroupRegexPermission, Network from mreg.models.auth import User, MregAdminGroup -from mreg.api.treetop import PolicyCheck, PolicyResource, policy_parity +from mreg.api.treetop import ( + PolicyCheck, + PolicyResource, + authorize_policy_stack, + policy_all, + policy_any, + policy_enforcement_enabled, + policy_leaf, + policy_parity, + policy_shadow_enabled, +) from mreg.policy.contracts import MEMBERSHIP_ACTIONS, snake_case from mreg.policy.resources import ( adapter_for_kind, @@ -184,12 +194,7 @@ def user_has_permission( is_member = user.is_member_of_any(memberlist) - return self.pp( - decision=is_member, - action=self._MEMBERSHIP_ACTIONS[membership], - request=request, - view=view, - ) + return is_member def user_is_superuser(self, request: Request, view: GenericAPIView) -> bool: """ @@ -260,6 +265,84 @@ def user_is_any(self, *memberships: MregAdminGroup, request: Request, view: Gene return True return False + def authorize_memberships( + self, + *memberships: MregAdminGroup, + legacy_decision: bool, + request: Request, + view: GenericAPIView, + ) -> bool: + """Authorize an OR of membership actions in one TreeTop call.""" + leaves = tuple( + policy_leaf( + action=self._MEMBERSHIP_ACTIONS[membership], + resource_kind="Generic", + resource_id="any", + resource_attrs=DEFAULT_RESOURCE_ATTRS, + ) + for membership in memberships + ) + root = leaves[0] if len(leaves) == 1 else policy_any(*leaves) + return authorize_policy_stack( + legacy_decision, + request=request, + root=root, + view=view, + permission_class=self.__class__.__name__, + ) + + def authorize_endpoint( + self, + *, + legacy_decision: bool, + request: Request, + view: GenericAPIView, + validated_serializer: Serializer | None = None, + obj: Any = None, + data: Mapping[str, Any] | None = None, + fallback_action: str = "authenticated_access", + ) -> bool: + """Authorize one ordinary endpoint operation as a single-leaf stack.""" + try: + resource_kind = self._resource_kind_from_view( + view=view, + validated_serializer=validated_serializer, + obj=obj, + ) + operation = self._crud_operation_from_method(request.method) + action = self._policy_action_from_view( + view=view, + resource_kind=resource_kind, + operation=operation, + ) + resource_id = self._resource_id_from_view( + view=view, + validated_serializer=validated_serializer, + obj=obj, + data=data, + ) + attrs = self._normalize_resource_attrs( + resource_kind=resource_kind, + attrs=data, + ) + except ValueError: + resource_kind = "Generic" + action = fallback_action + resource_id = str(next(iter(getattr(view, "kwargs", {}).values()), "any")) + attrs = DEFAULT_RESOURCE_ATTRS + return authorize_policy_stack( + legacy_decision, + request=request, + root=policy_leaf( + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + resource_attrs=attrs, + ), + view=view, + permission_class=self.__class__.__name__, + ) + class CRUDPermissionsMixin: """ @@ -332,14 +415,77 @@ def deny_reserved_ipaddress(self, ip: str, request: Request, view: GenericAPIVie return network.is_reserved_ipaddress(ip) + def deny_restricted_ipaddress(self, ip: str, request: Request, view: GenericAPIView) -> bool: + """Check all IP restrictions applied while assigning an address.""" + if self.deny_reserved_ipaddress(ip, request, view): + return True + if self.user_is_network_admin(request, view): + return False + network = Network.objects.filter(network__net_contains=ip).first() + if not network: + return False + address = ipaddress.ip_address(ip) + return address in { + network.network.network_address, + network.network.broadcast_address, + } + pass +class IsAuthenticatedWithPolicy(IsAuthenticated): + """Authenticate locally, then authorize the endpoint once in TreeTop.""" + + def has_permission(self, request, view): + if not super().has_permission(request, view): + return False + return self.authorize_endpoint( + legacy_decision=True, + request=request, + view=view, + data=request.data if isinstance(request.data, Mapping) else None, + ) + + +class UserInfoPermission(IsAuthenticated): + """Authorize access to the requesting user's or another user's details.""" + + def has_permission(self, request, view): + if not super().has_permission(request, view): + return False + user = User.from_request(request) + target_username = request.query_params.get("username") or user.username + self_access = target_username == user.username + legacy = self_access or user.is_mreg_superuser_or_admin or user.is_mreg_hostgroup_admin + return authorize_policy_stack( + legacy, + request=request, + root=policy_leaf( + action="user_info_read", + resource_kind="Generic", + resource_id=str(target_username), + resource_attrs={ + "kind": "generic", + "name": str(target_username), + "selfAccess": str(self_access).lower(), + }, + ), + view=view, + permission_class=self.__class__.__name__, + ) + + class IsAuthenticatedAndReadOnly(IsAuthenticated): def has_permission(self, request, view): if not super().has_permission(request, view): return False - return request.method in SAFE_METHODS + if request.method not in SAFE_METHODS: + return False + return self.authorize_endpoint( + legacy_decision=True, + request=request, + view=view, + ) class IsSuperGroupMember(IsAuthenticated): @@ -368,8 +514,15 @@ def has_permission(self, request, view): if not super().has_permission(request, view): return False if request.method in SAFE_METHODS: - return True - return self.user_is_admin(request=request, view=view) + return self.authorize_endpoint(legacy_decision=True, request=request, view=view) + legacy = self.user_is_admin(request=request, view=view) + return self.authorize_endpoint( + legacy_decision=legacy, + request=request, + view=view, + data=request.data if isinstance(request.data, Mapping) else None, + fallback_action=self._MEMBERSHIP_ACTIONS[MregAdminGroup.ADMINUSER], + ) class IsSuperOrNetworkAdminMember(IsAuthenticated): @@ -381,7 +534,56 @@ def has_permission(self, request, view): if not super().has_permission(request, view): return False - return self.user_is_any(MregAdminGroup.SUPERUSER, MregAdminGroup.NETWORK_ADMIN, request=request, view=view) + legacy = self.user_is_any( + MregAdminGroup.SUPERUSER, + MregAdminGroup.NETWORK_ADMIN, + request=request, + view=view, + ) + return self.authorize_endpoint( + legacy_decision=legacy, + request=request, + view=view, + data=request.data if isinstance(request.data, Mapping) else None, + fallback_action=self._MEMBERSHIP_ACTIONS[MregAdminGroup.NETWORK_ADMIN], + ) + + +class IsSuperOrReadOnly(IsAuthenticated): + """Authorize safe reads or superuser-only mutations with one stack.""" + + def has_permission(self, request, view): + if not super().has_permission(request, view): + return False + legacy = request.method in SAFE_METHODS or self.user_is_superuser(request, view) + return self.authorize_endpoint( + legacy_decision=legacy, + request=request, + view=view, + data=request.data if isinstance(request.data, Mapping) else None, + fallback_action=self._MEMBERSHIP_ACTIONS[MregAdminGroup.SUPERUSER], + ) + + +class IsNetworkAdminOrReadOnly(IsAuthenticated): + """Authorize safe reads or network-admin mutations with one stack.""" + + def has_permission(self, request, view): + if not super().has_permission(request, view): + return False + legacy = request.method in SAFE_METHODS or self.user_is_any( + MregAdminGroup.SUPERUSER, + MregAdminGroup.NETWORK_ADMIN, + request=request, + view=view, + ) + return self.authorize_endpoint( + legacy_decision=legacy, + request=request, + view=view, + data=request.data if isinstance(request.data, Mapping) else None, + fallback_action=self._MEMBERSHIP_ACTIONS[MregAdminGroup.NETWORK_ADMIN], + ) class IsSuperOrGroupAdminOrReadOnly(IsAuthenticated): @@ -393,9 +595,21 @@ def has_permission(self, request, view): if not super().has_permission(request, view): return False if request.method in SAFE_METHODS: - return True + return self.authorize_endpoint(legacy_decision=True, request=request, view=view) - return self.user_is_any(MregAdminGroup.SUPERUSER, MregAdminGroup.GROUP_ADMIN, request=request, view=view) + legacy = self.user_is_any( + MregAdminGroup.SUPERUSER, + MregAdminGroup.GROUP_ADMIN, + request=request, + view=view, + ) + return self.authorize_endpoint( + legacy_decision=legacy, + request=request, + view=view, + data=request.data if isinstance(request.data, Mapping) else None, + fallback_action=self._MEMBERSHIP_ACTIONS[MregAdminGroup.GROUP_ADMIN], + ) class IsGrantedNetGroupRegexPermission(IsAuthenticated): @@ -434,6 +648,9 @@ def has_permission(self, request, view): if user.is_mreg_superuser_or_admin: return True + if policy_enforcement_enabled() or policy_shadow_enabled(): + return True + # Will do do more object checks later, but initially refuse any # unwarranted requests. qs = NetGroupRegexPermission.objects.filter(group__in=user.group_list) @@ -447,6 +664,76 @@ def has_permission(self, request, view): return True return False + def _target_policy_node( + self, + *, + hostname: str, + ips: Sequence[str], + action: str, + resource_kind: str, + resource_id: str, + policy_name: str | None = None, + extra_attrs: Mapping[str, str] | None = None, + ): + """Build an OR of target-IP leaves with raw authorization facts.""" + checked_name = str(policy_name or hostname) + values = tuple(ips) or (None,) + leaves = [] + for ip in values: + attrs = { + "kind": self._snake_case(resource_kind), + "name": checked_name, + "hostname": str(hostname), + "dnsWildcard": str("*" in checked_name).lower(), + "dnsWildcardValidDepth": str(checked_name.count(".") >= 3).lower(), + "dnsUnderscore": str("_" in checked_name).lower(), + } + if ip is not None: + attrs["ip"] = str(ip) + network = Network.objects.filter(network__net_contains=str(ip)).first() + attrs["ipReserved"] = str(bool(network and network.is_reserved_ipaddress(str(ip)))).lower() + attrs["ipRestricted"] = str( + bool( + network + and ( + network.is_reserved_ipaddress(str(ip)) + or ipaddress.ip_address(ip) + in { + network.network.network_address, + network.network.broadcast_address, + } + ) + ) + ).lower() + if extra_attrs: + attrs.update(extra_attrs) + leaves.append( + policy_leaf( + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + resource_attrs=attrs, + ) + ) + return leaves[0] if len(leaves) == 1 else policy_any(*leaves) + + def _required_resource_kind( + self, + *, + view: GenericAPIView, + validated_serializer: Serializer | None = None, + obj: Any = None, + ) -> str: + """Translate an unregistered target into the legacy permission error.""" + try: + return self._resource_kind_from_view( + view=view, + validated_serializer=validated_serializer, + obj=obj, + ) + except ValueError as exc: + raise exceptions.PermissionDenied(f"Unhandled view: {view}") from exc + def has_perm( self, user, @@ -458,41 +745,31 @@ def has_perm( action: str | None = None, resource_kind: str = "Host", resource_id: str | None = None, + legacy_decision: bool | None = None, ): - """Evaluate NetGroupRegexPermission and parity for hostname/IP tuples.""" - legacy = bool(NetGroupRegexPermission.find_perm(user.group_list, hostname, ips, require_ip)) + """Evaluate all hostname/IP candidates in one synchronous policy stack.""" + legacy = ( + bool(NetGroupRegexPermission.find_perm(user.group_list, hostname, ips, require_ip)) + if legacy_decision is None + else bool(legacy_decision) + ) operation = self._crud_operation_from_method(request.method) resolved_action = action or self._crud_action(resource_kind, operation) resolved_resource_id = str(resource_id or hostname or "any") - policy: list[bool] = [] - if ips: - # This will perform one policy lookup per IP for the host. This should probably be optimized server side. - for ip in ips: - policy.append( - self.pp( - decision=legacy, - action=resolved_action, - request=request, - view=view, - resource_kind=resource_kind, - resource_id=resolved_resource_id, - resource_attrs={"hostname": str(hostname), "ip": str(ip)}, - ) - ) - else: - policy.append( - self.pp( - decision=legacy, - action=resolved_action, - request=request, - view=view, - resource_kind=resource_kind, - resource_id=resolved_resource_id, - resource_attrs={"hostname": str(hostname)}, - ) - ) - - return any(policy) + root = self._target_policy_node( + hostname=str(hostname), + ips=tuple(str(ip) for ip in ips), + action=resolved_action, + resource_kind=resource_kind, + resource_id=resolved_resource_id, + ) + return authorize_policy_stack( + legacy, + request=request, + root=root, + view=view, + permission_class=self.__class__.__name__, + ) def has_obj_perm( self, @@ -519,6 +796,10 @@ def _flatten_policy_attrs(self, data: Mapping[str, Any], *, resource_kind: str) """Adapt serializer data through the resource's registered adapter.""" return adapter_for_kind(resource_kind).attributes(data) + @staticmethod + def _legacy_target_permission(user: User, hostname: str, ips: Sequence[str], *, require_ip: bool = True) -> bool: + return bool(NetGroupRegexPermission.find_perm(user.group_list, hostname, ips, require_ip)) + def _has_create_target_permission( self, *, @@ -529,71 +810,96 @@ def _has_create_target_permission( action: str, resource_kind: str, resource_id: str, + restriction_denied: bool, ) -> bool: - """Apply the view-specific legacy create rules after common checks.""" + """Build and authorize the complete create stack in one call.""" import mreg.api.v1.views as v1_views ip_value = data.get("ipaddress") host = data.get("host") - host_ip_views = ( - v1_views.HostList, - v1_views.IpaddressList, - v1_views.PtrOverrideList, + standalone_name = str(data.get("name") or "") + standalone_target = not host and not isinstance( + view, + ( + v1_views.CnameList, + v1_views.HostList, + v1_views.IpaddressList, + v1_views.PtrOverrideList, + ), ) - - if isinstance(view, (v1_views.IpaddressList, v1_views.PtrOverrideList)): - if host and not self.has_obj_perm( - user, - host, - request=request, - view=view, - action=action, - resource_kind=resource_kind, - resource_id=resource_id, - ): - return False - if isinstance(view, v1_views.CnameList): name = self._stringify_attr_value(data["name"]) - return self.has_perm( - user, - name, - (), - require_ip=False, - request=request, - view=view, + node = self._target_policy_node( + hostname=name, + ips=(), action=action, resource_kind=resource_kind, resource_id=name, ) - - if isinstance(view, host_ip_views): - if not (ip_value and host): - return False - hostname = host.name - ips = [ip_value] - elif host: - hostname, ips = self._get_hostname_and_ips(host) + target_legacy = self._legacy_target_permission(user, name, (), require_ip=False) + nodes = [node] else: - raise exceptions.PermissionDenied(f"Unhandled view: {view}") + if isinstance(view, v1_views.HostList): + hostname = str(getattr(host, "name", None) or data.get("name") or "") + if not hostname: + return False + ips = [ip_value] if ip_value else [] + elif isinstance(view, (v1_views.IpaddressList, v1_views.PtrOverrideList)): + if not (ip_value and host): + return False + hostname = host.name + ips = [ip_value] + elif host: + hostname, ips = self._get_hostname_and_ips(host) + elif standalone_name: + hostname, ips = standalone_name, [] + else: + raise exceptions.PermissionDenied(f"Unhandled view: {view}") + + if not hostname: + return False + nodes = [ + self._target_policy_node( + hostname=str(hostname), + ips=tuple(str(ip) for ip in ips), + action=action, + resource_kind=resource_kind, + resource_id=self._stringify_attr_value(hostname), + policy_name=self._stringify_attr_value(data.get("name") or hostname), + ) + ] + target_legacy = self._legacy_target_permission(user, hostname, ips) + if isinstance(view, (v1_views.IpaddressList, v1_views.PtrOverrideList)): + old_hostname, old_ips = self._get_hostname_and_ips(host) + nodes.insert( + 0, + self._target_policy_node( + hostname=str(old_hostname), + ips=tuple(str(ip) for ip in old_ips), + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + ), + ) + target_legacy = target_legacy and self._legacy_target_permission( + user, + old_hostname, + old_ips, + ) - if not (ips and hostname): - return False - return self.has_perm( - user, - hostname, - ips, + role_legacy = user.is_mreg_superuser or (user.is_mreg_admin and not standalone_target) + legacy = user.is_mreg_superuser or (not restriction_denied and (role_legacy or target_legacy)) + root = nodes[0] if len(nodes) == 1 else policy_all(*nodes) + return authorize_policy_stack( + legacy, request=request, + root=root, view=view, - action=action, - resource_kind=resource_kind, - resource_id=self._stringify_attr_value(hostname), + permission_class=self.__class__.__name__, ) def has_create_permission(self, request, view, validated_serializer): """Authorize create operations using CRUD parity actions and legacy rules.""" - import mreg.api.v1.views as v1_views - user = User.from_request(request) data: dict[str, Any] = validated_serializer.validated_data # type: ignore logger.debug( @@ -603,22 +909,7 @@ def has_create_permission(self, request, view, validated_serializer): fields=sorted(data), ) - if self.user_is_superuser(request=request, view=view): - return True - - handled_by_view = isinstance( - view, - ( - v1_views.CnameList, - v1_views.HostList, - v1_views.IpaddressList, - v1_views.PtrOverrideList, - ), - ) - if not handled_by_view and "host" not in data: - raise exceptions.PermissionDenied(f"Unhandled view: {view}") - - resource_kind = self._resource_kind_from_view( + resource_kind = self._required_resource_kind( view=view, validated_serializer=validated_serializer, ) @@ -632,31 +923,20 @@ def has_create_permission(self, request, view, validated_serializer): validated_serializer=validated_serializer, data=data, ) - attrs = self._flatten_policy_attrs(data, resource_kind=resource_kind) ip_value = data.get("ipaddress") - # First check if we are asking for a restricted name. - if self.deny_superuser_only_names(data=data, view=view, request=request): - return False - # Then check if we are asking for an IP address *and* it is reserved. - if ip_value and self.deny_reserved_ipaddress( - ip=ip_value, + restriction_denied = self.deny_superuser_only_names( + data=data, view=view, request=request, - ): - return False - - # If the user is an admin, they are now free to create (minus the above checks). - if self.pp_generic_action( - decision=user.is_mreg_admin, - action=action, - kind=resource_kind, - resource_id=resource_id, - attrs=attrs, - request=request, - view=view, - ): - return True + ) or bool( + ip_value + and self.deny_restricted_ipaddress( + ip=ip_value, + view=view, + request=request, + ) + ) return self._has_create_target_permission( user=user, request=request, @@ -665,6 +945,7 @@ def has_create_permission(self, request, view, validated_serializer): action=action, resource_kind=resource_kind, resource_id=resource_id, + restriction_denied=restriction_denied, ) def has_destroy_permission(self, request, view, validated_serializer): @@ -673,117 +954,121 @@ def has_destroy_permission(self, request, view, validated_serializer): user = User.from_request(request) - if self.user_is_superuser(request=request, view=view): - return True - target_obj = view.get_object() host_obj = target_obj + standalone_target = False if not isinstance(view, v1_views.HostDetail) and hasattr(target_obj, "host"): host_obj = target_obj.host elif not isinstance(view, v1_views.HostDetail): - raise exceptions.PermissionDenied(f"Unhandled view: {view}") + standalone_target = True - resource_kind = self._resource_kind_from_view(view=view, obj=target_obj) + resource_kind = self._required_resource_kind(view=view, obj=target_obj) action = self._policy_action_from_view( view=view, resource_kind=resource_kind, operation="delete", ) resource_id = self._resource_id_from_view(view=view, obj=target_obj) - if self.deny_superuser_only_names(name=host_obj.name, view=view, request=request): - return False - if hasattr(host_obj, "ipaddress"): - if self.deny_reserved_ipaddress(ip=host_obj.ipaddress, view=view, request=request): - return False - - if self.pp_generic_action( - decision=user.is_mreg_admin, - action=action, - kind=resource_kind, - resource_id=resource_id, - attrs={"id": resource_id}, - request=request, + if standalone_target: + hostname = str(getattr(target_obj, "name", resource_id)) + ips = [] + else: + hostname, ips = self._get_hostname_and_ips(host_obj) + restriction_denied = self.deny_superuser_only_names( + name=host_obj.name, view=view, - ): - return True - return self.has_obj_perm( - user, - host_obj, request=request, + ) or bool( + hasattr(host_obj, "ipaddress") + and self.deny_reserved_ipaddress( + ip=host_obj.ipaddress, + view=view, + request=request, + ) + ) + target_legacy = self._legacy_target_permission(user, hostname, ips) + legacy = user.is_mreg_superuser or (not restriction_denied and ((user.is_mreg_admin and not standalone_target) or target_legacy)) + return authorize_policy_stack( + legacy, + request=request, + root=self._target_policy_node( + hostname=hostname, + ips=tuple(str(ip) for ip in ips), + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + policy_name=self._stringify_attr_value(getattr(target_obj, "name", None) or hostname), + ), view=view, - action=action, - resource_kind=resource_kind, - resource_id=resource_id, + permission_class=self.__class__.__name__, ) - def _has_host_detail_update_permission( + def _host_detail_update_stack( self, *, user: User, - request: Request, - view: GenericAPIView, target_obj: Any, data: Mapping[str, Any], action: str, resource_kind: str, - ) -> bool: + ): hostname, ips = self._get_hostname_and_ips(target_obj) - if "name" in data: - new_name = self._stringify_attr_value(data["name"]) - if not self.has_perm( - user, - new_name, - ips, - request=request, - view=view, + nodes = [ + self._target_policy_node( + hostname=hostname, + ips=tuple(str(ip) for ip in ips), action=action, resource_kind=resource_kind, - resource_id=new_name, - ): - return False - return self.has_perm( - user, - hostname, - ips, - request=request, - view=view, - action=action, - resource_kind=resource_kind, - resource_id=self._stringify_attr_value(hostname), - ) + resource_id=self._stringify_attr_value(hostname), + policy_name=self._stringify_attr_value(getattr(target_obj, "name", None) or hostname), + ) + ] + legacy = self._legacy_target_permission(user, hostname, ips) + if "name" in data: + new_name = self._stringify_attr_value(data["name"]) + nodes.insert( + 0, + self._target_policy_node( + hostname=new_name, + ips=tuple(str(ip) for ip in ips), + action=action, + resource_kind=resource_kind, + resource_id=new_name, + policy_name=new_name, + ), + ) + legacy = legacy and self._legacy_target_permission(user, new_name, ips) + return (nodes[0] if len(nodes) == 1 else policy_all(*nodes), legacy) - def _has_related_host_update_permission( + def _related_host_update_stack( self, *, user: User, - request: Request, - view: GenericAPIView, target_obj: Any, data: Mapping[str, Any], action: str, resource_kind: str, resource_id: str, - ) -> bool: + ): + hosts = [target_obj.host] if "host" in data and data["host"] != target_obj.host: - if not self.has_obj_perm( - user, - data["host"], - request=request, - view=view, - action=action, - resource_kind=resource_kind, - resource_id=resource_id, - ): - return False - return self.has_obj_perm( - user, - target_obj.host, - request=request, - view=view, - action=action, - resource_kind=resource_kind, - resource_id=resource_id, - ) + hosts.insert(0, data["host"]) + nodes = [] + legacy_values = [] + for host in hosts: + hostname, ips = self._get_hostname_and_ips(host) + nodes.append( + self._target_policy_node( + hostname=hostname, + ips=tuple(str(ip) for ip in ips), + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + policy_name=self._stringify_attr_value(data.get("name") or getattr(target_obj, "name", None) or hostname), + ) + ) + legacy_values.append(self._legacy_target_permission(user, hostname, ips)) + return (nodes[0] if len(nodes) == 1 else policy_all(*nodes), all(legacy_values)) def has_update_permission(self, request, view, validated_serializer): """Authorize update operations using CRUD parity actions and legacy rules.""" @@ -791,15 +1076,11 @@ def has_update_permission(self, request, view, validated_serializer): user = User.from_request(request) - if self.user_is_superuser(request=request, view=view): - return True - data: dict[str, Any] = validated_serializer.validated_data # type: ignore target_obj = view.get_object() - if not isinstance(view, v1_views.HostDetail) and not hasattr(target_obj, "host"): - raise exceptions.PermissionDenied(f"Unhandled view: {view}") + standalone_target = not isinstance(view, v1_views.HostDetail) and not hasattr(target_obj, "host") - resource_kind = self._resource_kind_from_view( + resource_kind = self._required_resource_kind( view=view, validated_serializer=validated_serializer, obj=target_obj, @@ -816,48 +1097,72 @@ def has_update_permission(self, request, view, validated_serializer): data=data, ) - if self.deny_superuser_only_names(data=data, view=view, request=request): - return False - if "ipaddress" in data: - if self.deny_reserved_ipaddress(ip=data["ipaddress"], view=view, request=request): - return False - - admin_attrs = {str(key): self._stringify_attr_value(value) for key, value in data.items()} - if self.pp_generic_action( - decision=user.is_mreg_admin, - action=action, - kind=resource_kind, - resource_id=resource_id, - attrs=admin_attrs, - request=request, + restriction_denied = self.deny_superuser_only_names( + data=data, view=view, - ): - return True + request=request, + ) or bool( + "ipaddress" in data + and self.deny_restricted_ipaddress( + ip=data["ipaddress"], + view=view, + request=request, + ) + ) if isinstance(view, v1_views.HostDetail): - return self._has_host_detail_update_permission( + root, target_legacy = self._host_detail_update_stack( user=user, target_obj=target_obj, data=data, - request=request, - view=view, action=action, resource_kind=resource_kind, ) - if hasattr(target_obj, "host"): - return self._has_related_host_update_permission( + elif hasattr(target_obj, "host"): + root, target_legacy = self._related_host_update_stack( user=user, - request=request, - view=view, target_obj=target_obj, data=data, action=action, resource_kind=resource_kind, resource_id=resource_id, ) - # Testing these kinds of should-never-happen codepaths is hard. - # We have to basically mock a complete API call and then break it. - raise exceptions.PermissionDenied(f"Unhandled view: {view}") # pragma: no cover + else: + current_name = str(getattr(target_obj, "name", resource_id)) + nodes = [ + self._target_policy_node( + hostname=current_name, + ips=(), + action=action, + resource_kind=resource_kind, + resource_id=resource_id, + policy_name=str(data.get("name") or current_name), + ) + ] + if data.get("name") and data["name"] != current_name: + new_name = str(data["name"]) + nodes.insert( + 0, + self._target_policy_node( + hostname=new_name, + ips=(), + action=action, + resource_kind=resource_kind, + resource_id=new_name, + policy_name=new_name, + ), + ) + root = nodes[0] if len(nodes) == 1 else policy_all(*nodes) + target_legacy = False + + legacy = user.is_mreg_superuser or (not restriction_denied and ((user.is_mreg_admin and not standalone_target) or target_legacy)) + return authorize_policy_stack( + legacy, + request=request, + root=root, + view=view, + permission_class=self.__class__.__name__, + ) def _get_hostname_and_ips(self, hostobject): """Extract a host's canonical name and all attached IP addresses.""" @@ -868,6 +1173,134 @@ def _get_hostname_and_ips(self, hostobject): return host.data["name"], ips +class IsGrantedNetGroupRegexOrNetworkAdmin(IsGrantedNetGroupRegexPermission): + """Combine the former DRF OR expression into one endpoint decision.""" + + def has_permission(self, request, view): + if not DRFIsAuthenticated.has_permission(self, request, view): + return False + user = User.from_request(request) + legacy = request.method in SAFE_METHODS or user.is_mreg_superuser_or_admin + if not legacy: + qs = NetGroupRegexPermission.objects.filter(group__in=user.group_list) + if network_in_url := view.kwargs.get("network"): + qs = qs.filter(range=network_in_url) + legacy = qs.exists() or user.is_mreg_network_admin + resource_kind = self._resource_kind_from_view(view=view) + operation = self._crud_operation_from_method(request.method) + action = self._policy_action_from_view( + view=view, + resource_kind=resource_kind, + operation=operation, + ) + network = str(view.kwargs.get("network") or "") + attrs = self._normalize_resource_attrs( + resource_kind=resource_kind, + attrs=request.data if isinstance(request.data, Mapping) else None, + ) + if network: + attrs["network"] = network + return authorize_policy_stack( + legacy, + request=request, + root=policy_leaf( + action=action, + resource_kind=resource_kind, + resource_id=self._resource_id_from_view( + view=view, + data=request.data if isinstance(request.data, Mapping) else None, + ), + resource_attrs=attrs, + ), + view=view, + permission_class=self.__class__.__name__, + ) + + +class HostContactsPermission(IsGrantedNetGroupRegexPermission): + """Authorize a host-contact endpoint against its complete host target.""" + + def has_permission(self, request, view): + if not DRFIsAuthenticated.has_permission(self, request, view): + return False + user = User.from_request(request) + hostname = str(view.kwargs.get("name") or "") + host = Host.objects.filter(name=hostname).first() + ips = self._get_hostname_and_ips(host)[1] if host is not None else [] + action = { + "GET": "host_contacts_read", + "HEAD": "host_contacts_read", + "OPTIONS": "host_contacts_read", + "POST": "host_contacts_create", + "DELETE": "host_contacts_delete", + }.get(request.method, "host_contacts_read") + restriction_denied = request.method not in SAFE_METHODS and self.deny_superuser_only_names( + name=hostname, + view=view, + request=request, + ) + target_legacy = self._legacy_target_permission(user, hostname, ips) + legacy = ( + request.method in SAFE_METHODS or user.is_mreg_superuser or (not restriction_denied and (user.is_mreg_admin or target_legacy)) + ) + return authorize_policy_stack( + legacy, + request=request, + root=self._target_policy_node( + hostname=hostname, + ips=tuple(str(ip) for ip in ips), + action=action, + resource_kind="Host", + resource_id=hostname or "any", + ), + view=view, + permission_class=self.__class__.__name__, + ) + + +class BACnetPermission(IsGrantedNetGroupRegexPermission): + """Authorize BACnet reads and mutations against the attached host.""" + + def has_permission(self, request, view): + if not DRFIsAuthenticated.has_permission(self, request, view): + return False + user = User.from_request(request) + host = None + if request.method == "POST": + host_id = request.data.get("host") + hostname = request.data.get("hostname") + if host_id is not None: + host = Host.objects.filter(pk=host_id).first() + elif hostname: + host = Host.objects.filter(name=hostname).first() + elif view.kwargs.get("id") is not None: + try: + obj = view.get_queryset().filter(pk=view.kwargs["id"]).first() + except (TypeError, ValueError): + obj = None + host = getattr(obj, "host", None) + + hostname = str(getattr(host, "name", "any")) + ips = self._get_hostname_and_ips(host)[1] if host is not None else [] + operation = self._crud_operation_from_method(request.method) + action = self._crud_action("BACnetID", operation) + target_legacy = bool(host is not None and self._legacy_target_permission(user, hostname, ips)) + legacy = request.method in SAFE_METHODS or user.is_mreg_superuser_or_admin or target_legacy + return authorize_policy_stack( + legacy, + request=request, + root=self._target_policy_node( + hostname=hostname, + ips=tuple(str(ip) for ip in ips), + action=action, + resource_kind="BACnetID", + resource_id=str(request.data.get("id") or view.kwargs.get("id") or "any"), + ), + view=view, + permission_class=self.__class__.__name__, + ) + + class HostGroupPermission(IsAuthenticated): def has_permission(self, request, view): # This method is called before the view is executed, so @@ -876,6 +1309,8 @@ def has_permission(self, request, view): return False user = User.from_request(request) if request.method in SAFE_METHODS: + return self.authorize_endpoint(legacy_decision=True, request=request, view=view) + if policy_enforcement_enabled() or policy_shadow_enabled(): return True if user.is_mreg_superuser or user.is_mreg_hostgroup_admin: return True @@ -890,30 +1325,90 @@ def _request_user_is_owner(hostgroup, request): owners = list(set(hostgroup.owners.values_list("name", flat=True))) return User.from_request(request).is_member_of_any(owners) + def _authorize_hostgroup( + self, + *, + legacy: bool, + request: Request, + view: GenericAPIView, + hostgroup: HostGroup, + action: str, + requester_is_owner: bool, + owner_mutation: bool = False, + description_update: bool = False, + ) -> bool: + return authorize_policy_stack( + legacy, + request=request, + root=policy_leaf( + action=action, + resource_kind="HostGroup", + resource_id=str(hostgroup.name), + resource_attrs={ + "kind": "host_group", + "name": str(hostgroup.name), + "requesterIsOwner": str(requester_is_owner).lower(), + "ownerMutation": str(owner_mutation).lower(), + "descriptionUpdate": str(description_update).lower(), + }, + ), + view=view, + permission_class=self.__class__.__name__, + ) + def has_m2m_change_permission(self, request, view): user = User.from_request(request) - if user.is_mreg_superuser or user.is_mreg_hostgroup_admin: - return True - return self._request_user_is_owner(view.object, request) + requester_is_owner = self._request_user_is_owner(view.object, request) + owner_mutation = getattr(view, "m2m_field", None) == "owners" + legacy = user.is_mreg_superuser or user.is_mreg_hostgroup_admin + if not owner_mutation: + legacy = legacy or requester_is_owner + return self._authorize_hostgroup( + legacy=legacy, + request=request, + view=view, + hostgroup=view.object, + action="hostgroup_membership_update", + requester_is_owner=requester_is_owner, + owner_mutation=owner_mutation, + ) # patch will only happen on HostGroupDetail def has_update_permission(self, request, view, validated_serializer): user = User.from_request(request) - if user.is_mreg_superuser or user.is_mreg_hostgroup_admin: - return True - if "description" in validated_serializer.validated_data: - return self._request_user_is_owner(view.get_object(), request) - return False + obj = view.get_object() + requester_is_owner = self._request_user_is_owner(obj, request) + legacy = user.is_mreg_superuser or user.is_mreg_hostgroup_admin + if not legacy and "description" in validated_serializer.validated_data: + legacy = requester_is_owner + return self._authorize_hostgroup( + legacy=legacy, + request=request, + view=view, + hostgroup=obj, + action="host_group_update", + requester_is_owner=requester_is_owner, + description_update="description" in validated_serializer.validated_data, + ) def has_destroy_permission(self, request, view, validated_serializer): user = User.from_request(request) - if user.is_mreg_superuser or user.is_mreg_hostgroup_admin: - return True - return False + legacy = user.is_mreg_superuser or user.is_mreg_hostgroup_admin + hostgroup = view.get_object() + return self._authorize_hostgroup( + legacy=legacy, + request=request, + view=view, + hostgroup=hostgroup, + action="host_group_delete", + requester_is_owner=self._request_user_is_owner(hostgroup, request), + ) class IsGrantedReservedAddressPermission(IsAuthenticated): def has_ipaddress_permission(self, request: Request, view: GenericAPIView, validated_serializer: Serializer): + if policy_enforcement_enabled(): + return True user = User.from_request(request) if user.is_mreg_superuser_or_admin or user.is_mreg_network_admin: return True diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index e22ee8b9..a3744616 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -1,3 +1,5 @@ +"""Synchronous TreeTop authorization for endpoint permission stacks.""" + from __future__ import annotations import atexit @@ -6,17 +8,14 @@ import logging import os import threading -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping from contextlib import contextmanager, suppress from contextvars import ContextVar -from dataclasses import dataclass, field -from datetime import timedelta +from dataclasses import dataclass from time import monotonic +from typing import TypeAlias from django.conf import settings -from django.db import close_old_connections, transaction -from django.db.models import Q -from django.utils import timezone from django.views import View from prometheus_client import Counter, Gauge, Histogram from rest_framework.request import Request @@ -33,72 +32,51 @@ ) from mreg.models.auth import User as MregUser -from mreg.models.policy import PolicyParityOutbox -from mreg.policy.config import EnforcementFailureMode, PolicyMode +from mreg.policy.config import PolicyMode + logger = structlog.get_logger("mreg.policy.parity") POLICY_MODE = PolicyMode(getattr(settings, "POLICY_MODE", "shadow")) POLICY_PARITY_ENABLED = getattr(settings, "POLICY_PARITY_ENABLED", POLICY_MODE == PolicyMode.SHADOW) -POLICY_ENFORCEMENT_FAILURE_MODE = EnforcementFailureMode( - getattr(settings, "POLICY_ENFORCEMENT_FAILURE_MODE", "deny") -) POLICY_BASE_URL = (getattr(settings, "POLICY_BASE_URL", "") or "").strip() POLICY_NAMESPACE = getattr(settings, "POLICY_NAMESPACE", ["MREG"]) -POLICY_PARITY_BATCH_ENABLED = getattr(settings, "POLICY_PARITY_BATCH_ENABLED", True) POLICY_PARITY_LOG_DETAILS = getattr(settings, "POLICY_PARITY_LOG_DETAILS", False) POLICY_TIMEOUT_SECONDS = getattr(settings, "POLICY_TIMEOUT_SECONDS", 5.0) -POLICY_PARITY_MAX_ATTEMPTS = getattr(settings, "POLICY_PARITY_MAX_ATTEMPTS", 8) -POLICY_PARITY_RETRY_BASE_SECONDS = getattr(settings, "POLICY_PARITY_RETRY_BASE_SECONDS", 2.0) -POLICY_PARITY_RETRY_MAX_SECONDS = getattr(settings, "POLICY_PARITY_RETRY_MAX_SECONDS", 300.0) -POLICY_PARITY_LEASE_SECONDS = getattr(settings, "POLICY_PARITY_LEASE_SECONDS", 60.0) -POLICY_PARITY_POLL_SECONDS = getattr(settings, "POLICY_PARITY_POLL_SECONDS", 1.0) -POLICY_PARITY_CIRCUIT_FAILURES = getattr(settings, "POLICY_PARITY_CIRCUIT_FAILURES", 5) -POLICY_PARITY_CIRCUIT_RESET_SECONDS = getattr(settings, "POLICY_PARITY_CIRCUIT_RESET_SECONDS", 30.0) +POLICY_CIRCUIT_FAILURES = getattr(settings, "POLICY_CIRCUIT_FAILURES", 5) +POLICY_CIRCUIT_RESET_SECONDS = getattr(settings, "POLICY_CIRCUIT_RESET_SECONDS", 30.0) POLICY_DECISIONS_TOTAL = Counter( "mreg_policy_decisions_total", - "Total policy decisions from the external policy engine.", + "Composite decisions returned by TreeTop.", ["decision"], ) - POLICY_LEGACY_DECISIONS_TOTAL = Counter( "mreg_policy_legacy_decisions_total", - "Total legacy permission decisions used for parity comparison.", + "Composite legacy decisions evaluated for policy comparison.", ["decision"], ) - POLICY_PARITY_RESULTS_TOTAL = Counter( "mreg_policy_parity_results_total", - "Parity comparison outcomes between legacy and external policy decisions.", + "Composite comparison outcomes between legacy and TreeTop.", ["result"], ) - POLICY_AUTHORIZE_CALLS_TOTAL = Counter( "mreg_policy_authorize_calls_total", - "Total calls to the policy authorize endpoint.", - ["status"], -) - -POLICY_PARITY_BATCHES_TOTAL = Counter( - "mreg_policy_parity_batches_total", - "Durable policy parity batch lifecycle events.", + "Synchronous calls to the TreeTop authorize endpoint.", ["status"], ) - -POLICY_PARITY_FAILURES_TOTAL = Counter( - "mreg_policy_parity_failures_total", - "Policy integration failures by processing stage.", +POLICY_FAILURES_TOTAL = Counter( + "mreg_policy_failures_total", + "Policy integration failures by stage.", ["stage"], ) - POLICY_ENFORCEMENT_RESULTS_TOTAL = Counter( "mreg_policy_enforcement_results_total", "Synchronous authoritative policy outcomes.", ["result"], ) - POLICY_MODE_INFO = Gauge( "mreg_policy_mode_info", "Configured MREG policy decision mode.", @@ -106,49 +84,32 @@ multiprocess_mode="livemax", ) POLICY_MODE_INFO.labels(mode=POLICY_MODE.value).set(1) - POLICY_AUTHORIZE_DURATION_SECONDS = Histogram( "mreg_policy_authorize_duration_seconds", - "Duration of policy authorize endpoint calls in seconds.", + "Duration of synchronous policy authorize calls.", ["status"], buckets=[0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5], ) - -POLICY_REQUESTS_PER_AUTHORIZE = Histogram( - "mreg_policy_requests_per_authorize", - "Number of policy requests sent in each authorize call.", - buckets=[0, 1, 2, 3, 5, 8], -) - -POLICY_QUERIES_PER_REQUEST = Histogram( - "mreg_policy_queries_per_request", - "Number of policy authorize batches submitted per HTTP request.", - buckets=[0, 1, 2, 3, 5, 8], -) - -POLICY_PARITY_OUTBOX_ENTRIES = Gauge( - "mreg_policy_parity_outbox_entries", - "Current durable policy parity outbox entries.", - ["status"], - multiprocess_mode="livemax", +POLICY_STACK_SIZE = Histogram( + "mreg_policy_stack_size", + "Number of Cedar checks in one endpoint policy stack.", + buckets=[1, 2, 3, 5, 8, 13, 21], ) - -POLICY_PARITY_OUTBOX_OLDEST_SECONDS = Gauge( - "mreg_policy_parity_outbox_oldest_seconds", - "Age of the oldest pending durable policy parity batch.", - multiprocess_mode="livemax", +POLICY_CALLS_PER_REQUEST = Histogram( + "mreg_policy_authorize_calls_per_request", + "Number of TreeTop authorize HTTP calls made by one MREG request.", + buckets=[0, 1, 2], ) - -POLICY_PARITY_CIRCUIT_OPEN = Gauge( - "mreg_policy_parity_circuit_open", - "Whether this worker's TreeTop delivery circuit breaker is open.", +POLICY_CIRCUIT_OPEN = Gauge( + "mreg_policy_circuit_open", + "Whether this worker's synchronous TreeTop circuit is open.", multiprocess_mode="livemax", ) @dataclass(frozen=True, slots=True) class PolicyResource: - """Explicit resource contract for a policy parity check.""" + """One typed Cedar resource.""" kind: str id: str @@ -165,7 +126,7 @@ def __post_init__(self) -> None: @dataclass(frozen=True, slots=True) class PolicyCheck: - """Typed action/resource pair evaluated by TreeTop.""" + """One action/resource leaf evaluated by Cedar.""" action: str resource: PolicyResource @@ -175,122 +136,145 @@ def __post_init__(self) -> None: raise ValueError("Policy action cannot be empty") -@dataclass(slots=True) -class _ParityBatchItem: - decision: bool - policy_request: TreeTopRequest - context: dict[str, object] +@dataclass(frozen=True, slots=True) +class PolicyLeaf: + """Leaf in an endpoint permission tree.""" + check: PolicyCheck -def _serialize_policy_batch(items: Sequence[_ParityBatchItem]) -> dict[str, object]: - """Convert a batch into the versioned JSON outbox representation.""" - return { - "version": 1, - "items": [ - { - "decision": item.decision, - "policy_request": item.policy_request.to_api(), - "context": item.context, - } - for item in items - ], - } +@dataclass(frozen=True, slots=True) +class PolicyAll: + """Require every child policy node to allow.""" -def _deserialize_policy_request(payload: Mapping[str, object]) -> TreeTopRequest: - principal_payload = payload["principal"] - if not isinstance(principal_payload, Mapping): - raise ValueError("Invalid durable policy principal") - user_payload = principal_payload["User"] - if not isinstance(user_payload, Mapping): - raise ValueError("Invalid durable policy user") - namespace = [str(part) for part in user_payload.get("namespace", [])] - groups_payload = user_payload.get("groups", []) - groups = [str(group["id"]) for group in groups_payload if isinstance(group, Mapping)] - - action_payload = payload["action"] - resource_payload = payload["resource"] - if not isinstance(action_payload, Mapping) or not isinstance(resource_payload, Mapping): - raise ValueError("Invalid durable policy action or resource") - action = Action.new( - str(action_payload["id"]), - [str(part) for part in action_payload.get("namespace", [])], - ) - attrs_payload = resource_payload.get("attrs", {}) - if not isinstance(attrs_payload, Mapping): - raise ValueError("Invalid durable policy resource attributes") - attrs: dict[str, ResourceAttribute] = {} - for key, raw_attribute in attrs_payload.items(): - if not isinstance(raw_attribute, Mapping): - raise ValueError("Invalid durable policy resource attribute") - attrs[str(key)] = ResourceAttribute.new( - str(raw_attribute["value"]), - ResourceAttributeType(str(raw_attribute["type"])), + children: tuple[PolicyNode, ...] + + def __post_init__(self) -> None: + if not self.children: + raise ValueError("PolicyAll requires at least one child") + + +@dataclass(frozen=True, slots=True) +class PolicyAny: + """Require at least one child policy node to allow.""" + + children: tuple[PolicyNode, ...] + + def __post_init__(self) -> None: + if not self.children: + raise ValueError("PolicyAny requires at least one child") + + +PolicyNode: TypeAlias = PolicyLeaf | PolicyAll | PolicyAny + + +def policy_leaf( + *, + action: str, + resource_kind: str, + resource_id: str, + resource_attrs: Mapping[str, str], +) -> PolicyLeaf: + """Build one validated leaf without exposing wire-model details.""" + return PolicyLeaf( + PolicyCheck( + action=action, + resource=PolicyResource( + kind=resource_kind, + id=str(resource_id), + attrs=resource_attrs, + ), ) - return TreeTopRequest( - principal=TreeTopUser.new(str(user_payload["id"]), namespace, groups=groups), - action=action, - resource=TreeTopResource.new( - kind=str(resource_payload["kind"]), - id=str(resource_payload["id"]), - attrs=attrs, - ), ) -def _deserialize_policy_batch(payload: Mapping[str, object]) -> list[_ParityBatchItem]: - if payload.get("version") != 1: - raise ValueError(f"Unsupported policy outbox payload version: {payload.get('version')}") - raw_items = payload.get("items") - if not isinstance(raw_items, list): - raise ValueError("Invalid durable policy batch") - items: list[_ParityBatchItem] = [] - for raw_item in raw_items: - if not isinstance(raw_item, Mapping): - raise ValueError("Invalid durable policy batch item") - request_payload = raw_item.get("policy_request") - context = raw_item.get("context") - if not isinstance(request_payload, Mapping) or not isinstance(context, Mapping): - raise ValueError("Invalid durable policy batch request or context") - items.append( - _ParityBatchItem( - decision=bool(raw_item.get("decision")), - policy_request=_deserialize_policy_request(request_payload), - context={str(key): value for key, value in context.items()}, - ) - ) - return items +def policy_all(*nodes: PolicyNode) -> PolicyAll: + return PolicyAll(tuple(nodes)) + + +def policy_any(*nodes: PolicyNode) -> PolicyAny: + return PolicyAny(tuple(nodes)) @dataclass(slots=True) -class _RequestParityState: - items: list[_ParityBatchItem] = field(default_factory=list) - submitted_queries: int = 0 +class _RequestPolicyState: + calls: int = 0 + fingerprint: tuple[object, ...] | None = None + policy_decision: bool | None = None + error: str | None = None -_request_state: ContextVar[_RequestParityState | None] = ContextVar( - "policy_parity_request_state", +_request_state: ContextVar[_RequestPolicyState | None] = ContextVar( + "mreg_policy_request_state", default=None, ) -_parity_disabled_depth: ContextVar[int] = ContextVar( - "policy_parity_disabled_depth", +_shadow_disabled_depth: ContextVar[int] = ContextVar( + "mreg_policy_shadow_disabled_depth", default=0, ) + +class _SynchronousCircuitBreaker: + """Thread-safe closed/open/half-open circuit for request-path calls.""" + + def __init__(self, failure_threshold: int, reset_seconds: float) -> None: + self.failure_threshold = max(1, int(failure_threshold)) + self.reset_seconds = max(0.1, float(reset_seconds)) + self._lock = threading.Lock() + self._failures = 0 + self._open_until = 0.0 + self._probe_in_flight = False + + def allow_call(self) -> bool: + now = monotonic() + with self._lock: + if self._open_until == 0.0: + POLICY_CIRCUIT_OPEN.set(0) + return True + if now < self._open_until or self._probe_in_flight: + POLICY_CIRCUIT_OPEN.set(1) + return False + self._probe_in_flight = True + POLICY_CIRCUIT_OPEN.set(1) + return True + + def success(self) -> None: + with self._lock: + self._failures = 0 + self._open_until = 0.0 + self._probe_in_flight = False + POLICY_CIRCUIT_OPEN.set(0) + + def failure(self) -> None: + now = monotonic() + with self._lock: + self._probe_in_flight = False + self._failures += 1 + if self._open_until or self._failures >= self.failure_threshold: + self._open_until = now + self.reset_seconds + POLICY_CIRCUIT_OPEN.set(1) + _safe_log( + logging.ERROR, + "policy_circuit_open", + reset_seconds=self.reset_seconds, + consecutive_failures=self._failures, + ) + + +_circuit = _SynchronousCircuitBreaker(POLICY_CIRCUIT_FAILURES, POLICY_CIRCUIT_RESET_SECONDS) _client: TreeTopClient | None = None _client_pid: int | None = None _client_lock = threading.Lock() def _get_treetop_client() -> TreeTopClient: - """Return a process-local client, creating it only on first use.""" global _client, _client_pid - pid = os.getpid() with _client_lock: if _client is None or _client_pid != pid: if _client is not None: - _client.close() + with suppress(Exception): + _client.close() _client = TreeTopClient( base_url=POLICY_BASE_URL, timeout=float(POLICY_TIMEOUT_SECONDS), @@ -299,10 +283,9 @@ def _get_treetop_client() -> TreeTopClient: return _client -def _close_treetop_client() -> None: - """Close both transports owned by the process-local TreeTop client.""" +def close_policy_client() -> None: + """Close transports owned by this process.""" global _client, _client_pid - with _client_lock: client = _client _client = None @@ -312,264 +295,11 @@ def _close_treetop_client() -> None: try: asyncio.run(client.aclose()) except Exception: - client.close() - - -@dataclass(frozen=True, slots=True) -class _ClaimedPolicyBatch: - id: int - attempts: int - payload: Mapping[str, object] - - -class _CircuitBreaker: - """Small process-local breaker protecting the shared TreeTop service.""" - - def __init__(self, failure_threshold: int, reset_seconds: float) -> None: - self.failure_threshold = max(1, failure_threshold) - self.reset_seconds = max(0.1, reset_seconds) - self.consecutive_failures = 0 - self.open_until = 0.0 - - def wait_seconds(self) -> float: - remaining = self.open_until - monotonic() - if remaining <= 0: - POLICY_PARITY_CIRCUIT_OPEN.set(0) - return 0.0 - POLICY_PARITY_CIRCUIT_OPEN.set(1) - return remaining - - def success(self) -> None: - self.consecutive_failures = 0 - self.open_until = 0.0 - POLICY_PARITY_CIRCUIT_OPEN.set(0) - - def failure(self) -> None: - self.consecutive_failures += 1 - if self.consecutive_failures >= self.failure_threshold: - self.open_until = monotonic() + self.reset_seconds - POLICY_PARITY_CIRCUIT_OPEN.set(1) - _safe_log( - logging.ERROR, - "policy_parity_circuit_open", - reset_seconds=self.reset_seconds, - consecutive_failures=self.consecutive_failures, - ) - - -def _refresh_outbox_metrics() -> None: - """Refresh low-cardinality gauges from the durable shared queue.""" - try: - now = timezone.now() - pending = PolicyParityOutbox.objects.filter(failed_at__isnull=True) - POLICY_PARITY_OUTBOX_ENTRIES.labels(status="pending").set(pending.count()) - POLICY_PARITY_OUTBOX_ENTRIES.labels(status="dead_letter").set( - PolicyParityOutbox.objects.filter(failed_at__isnull=False).count() - ) - oldest = pending.order_by("created_at").values_list("created_at", flat=True).first() - POLICY_PARITY_OUTBOX_OLDEST_SECONDS.set(max(0.0, (now - oldest).total_seconds()) if oldest else 0.0) - except Exception as exc: - _record_instrumentation_failure(exc, stage="outbox_metrics") - - -class _ParityDispatcher: - """Database-outbox worker shared safely by all application processes.""" - - def __init__(self) -> None: - self._thread: threading.Thread | None = None - self._start_lock = threading.Lock() - self._wake = threading.Event() - self._stop = threading.Event() - self._circuit = _CircuitBreaker( - int(POLICY_PARITY_CIRCUIT_FAILURES), - float(POLICY_PARITY_CIRCUIT_RESET_SECONDS), - ) - - def submit(self, items: Sequence[_ParityBatchItem]) -> bool: - """Persist a batch before waking a worker; no policy I/O occurs here.""" - if not items: - return False - self._ensure_started() - PolicyParityOutbox.objects.create(payload=_serialize_policy_batch(items)) - POLICY_PARITY_BATCHES_TOTAL.labels(status="persisted").inc() - transaction.on_commit(self.wake) - _refresh_outbox_metrics() - return True - - def wake(self) -> None: - self._wake.set() - - def _ensure_started(self) -> None: - if self._thread is not None and self._thread.is_alive(): - return - with self._start_lock: - if self._thread is None or not self._thread.is_alive(): - self._stop.clear() - self._thread = threading.Thread( - target=self._run, - name="mreg-policy-parity-outbox", - daemon=True, - ) - self._thread.start() - - def _claim(self) -> _ClaimedPolicyBatch | None: - now = timezone.now() - stale_before = now - timedelta(seconds=float(POLICY_PARITY_LEASE_SECONDS)) - with transaction.atomic(): - row = ( - PolicyParityOutbox.objects.select_for_update(skip_locked=True) - .filter(failed_at__isnull=True, available_at__lte=now) - .filter(Q(locked_at__isnull=True) | Q(locked_at__lt=stale_before)) - .order_by("available_at", "id") - .first() - ) - if row is None: - return None - row.attempts += 1 - row.locked_at = now - row.save(update_fields=("attempts", "locked_at")) - return _ClaimedPolicyBatch(id=row.id, attempts=row.attempts, payload=row.payload) - - def _complete(self, claimed: _ClaimedPolicyBatch) -> None: - PolicyParityOutbox.objects.filter(id=claimed.id).delete() - POLICY_PARITY_BATCHES_TOTAL.labels(status="processed").inc() - self._circuit.success() - - def _fail( - self, - claimed: _ClaimedPolicyBatch, - exc: Exception, - items: Sequence[_ParityBatchItem], - ) -> None: - error = f"{type(exc).__name__}: {exc}" - self._circuit.failure() - now = timezone.now() - if claimed.attempts >= int(POLICY_PARITY_MAX_ATTEMPTS): - PolicyParityOutbox.objects.filter(id=claimed.id).update( - locked_at=None, - failed_at=now, - last_error=error, - ) - POLICY_PARITY_BATCHES_TOTAL.labels(status="dead_letter").inc() - for item in items: - _log_parity_payload( - _compute_parity_payload( - decision=item.decision, - policy_allowed=None, - error=error, - context=item.context, - ) - ) - _safe_log( - logging.ERROR, - "policy_parity_dead_letter", - outbox_id=claimed.id, - attempts=claimed.attempts, - error_type=type(exc).__name__, - ) - return - delay = min( - float(POLICY_PARITY_RETRY_MAX_SECONDS), - float(POLICY_PARITY_RETRY_BASE_SECONDS) * (2 ** (claimed.attempts - 1)), - ) - PolicyParityOutbox.objects.filter(id=claimed.id).update( - locked_at=None, - available_at=now + timedelta(seconds=delay), - last_error=error, - ) - POLICY_PARITY_BATCHES_TOTAL.labels(status="retried").inc() - _safe_log( - logging.WARNING, - "policy_parity_retry_scheduled", - outbox_id=claimed.id, - attempts=claimed.attempts, - delay_seconds=delay, - error_type=type(exc).__name__, - ) - - def _run(self) -> None: - close_old_connections() - try: - while not self._stop.is_set(): - circuit_wait = self._circuit.wait_seconds() - if circuit_wait > 0: - self._wake.wait(timeout=min(circuit_wait, float(POLICY_PARITY_POLL_SECONDS))) - self._wake.clear() - continue - close_old_connections() - try: - claimed = self._claim() - except Exception as exc: - _record_instrumentation_failure(exc, stage="outbox_claim") - close_old_connections() - self._wake.wait(timeout=float(POLICY_PARITY_POLL_SECONDS)) - self._wake.clear() - continue - if claimed is None: - _refresh_outbox_metrics() - self._wake.wait(timeout=float(POLICY_PARITY_POLL_SECONDS)) - self._wake.clear() - continue - items: list[_ParityBatchItem] = [] - try: - items = _deserialize_policy_batch(claimed.payload) - _process_policy_parity_batch(items) - self._complete(claimed) - except Exception as exc: - _record_instrumentation_failure(exc, stage="worker") - try: - self._fail(claimed, exc, items) - except Exception as fail_exc: - _record_instrumentation_failure(fail_exc, stage="outbox_retry") - close_old_connections() - finally: - _refresh_outbox_metrics() - finally: - close_old_connections() - - def shutdown(self) -> None: - thread = self._thread - if thread is None or not thread.is_alive(): - return - self._stop.set() - self._wake.set() - thread.join(timeout=max(1.0, float(POLICY_TIMEOUT_SECONDS) + 1.0)) - - -_dispatcher: _ParityDispatcher | None = None -_dispatcher_pid: int | None = None -_dispatcher_lock = threading.Lock() - - -def _get_dispatcher() -> _ParityDispatcher: - global _dispatcher, _dispatcher_pid - - pid = os.getpid() - with _dispatcher_lock: - if _dispatcher is None or _dispatcher_pid != pid: - _dispatcher = _ParityDispatcher() - _dispatcher_pid = pid - return _dispatcher - - -def start_policy_parity_dispatcher() -> None: - """Start the shadow-mode outbox worker after Gunicorn forks.""" - if _is_shadow_enabled(): - _get_dispatcher()._ensure_started() - - -def stop_policy_parity_dispatcher() -> None: - """Stop this process's outbox worker without affecting persisted work.""" - if _dispatcher is not None and _dispatcher_pid == os.getpid(): - _dispatcher.shutdown() - - -def _shutdown_policy_runtime() -> None: - stop_policy_parity_dispatcher() - _close_treetop_client() + with suppress(Exception): + client.close() -atexit.register(_shutdown_policy_runtime) +atexit.register(close_policy_client) def _safe_log(level: int, event: str, **context: object) -> None: @@ -577,423 +307,290 @@ def _safe_log(level: int, event: str, **context: object) -> None: logger.log(level, event, **context) -def _record_instrumentation_failure(exc: Exception, *, stage: str) -> None: +def _record_failure(stage: str, error: str, **context: object) -> None: with suppress(Exception): - POLICY_PARITY_FAILURES_TOTAL.labels(stage=stage).inc() - _safe_log( - logging.ERROR, - "policy_parity_instrumentation_error", - stage=stage, - error_type=type(exc).__name__, - error_msg=str(exc), - ) - - -def _submit_policy_batch(items: Sequence[_ParityBatchItem]) -> bool: - try: - return _get_dispatcher().submit(items) - except Exception as exc: - with suppress(Exception): - POLICY_PARITY_BATCHES_TOTAL.labels(status="persist_failed").inc() - _record_instrumentation_failure(exc, stage="persist") - return False + POLICY_FAILURES_TOTAL.labels(stage=stage).inc() + _safe_log(logging.ERROR, "policy_integration_error", stage=stage, error=error, **context) @contextmanager -def batch_policy_parity(): - """Track one request's policy work and batch shadow checks.""" - if not _is_policy_enabled(): - yield - return +def policy_request_scope(): + """Record and enforce the one-authorize-call-per-request invariant.""" if _request_state.get() is not None: yield return - - state = _RequestParityState() + state = _RequestPolicyState() token = _request_state.set(state) try: yield finally: - try: - if _current_policy_mode() == PolicyMode.SHADOW and POLICY_PARITY_BATCH_ENABLED and state.items: - if _submit_policy_batch(state.items): - state.submitted_queries += 1 - with suppress(Exception): - POLICY_QUERIES_PER_REQUEST.observe(float(state.submitted_queries)) - except Exception as exc: - _record_instrumentation_failure(exc, stage="request_exit") - finally: - _request_state.reset(token) + with suppress(Exception): + POLICY_CALLS_PER_REQUEST.observe(float(state.calls)) + _request_state.reset(token) @contextmanager def disable_policy_parity(): - """Temporarily disable shadow checks in the current execution context. - - Enforcement deliberately ignores this test helper so production code cannot - turn an authoritative decision back into a legacy decision accidentally. - """ - token = _parity_disabled_depth.set(_parity_disabled_depth.get() + 1) + """Disable synchronous shadow comparisons in a narrow test scope.""" + token = _shadow_disabled_depth.set(_shadow_disabled_depth.get() + 1) try: yield finally: - _parity_disabled_depth.reset(token) + _shadow_disabled_depth.reset(token) def _current_policy_mode() -> PolicyMode: - value = POLICY_MODE - return value if isinstance(value, PolicyMode) else PolicyMode(value) + return POLICY_MODE if isinstance(POLICY_MODE, PolicyMode) else PolicyMode(POLICY_MODE) -def _current_enforcement_failure_mode() -> EnforcementFailureMode: - value = POLICY_ENFORCEMENT_FAILURE_MODE - return value if isinstance(value, EnforcementFailureMode) else EnforcementFailureMode(value) +def policy_enforcement_enabled() -> bool: + """Return whether TreeTop decisions are authoritative.""" + return _current_policy_mode() == PolicyMode.ENFORCE -def _is_shadow_enabled() -> bool: +def policy_shadow_enabled() -> bool: + """Return whether synchronous shadow evaluation is active in this scope.""" return bool( - _current_policy_mode() == PolicyMode.SHADOW - and POLICY_PARITY_ENABLED - and POLICY_BASE_URL - and _parity_disabled_depth.get() == 0 + _current_policy_mode() == PolicyMode.SHADOW and POLICY_PARITY_ENABLED and POLICY_BASE_URL and _shadow_disabled_depth.get() == 0 ) -def _is_enforcement_enabled() -> bool: - return _current_policy_mode() == PolicyMode.ENFORCE - - -def _is_policy_enabled() -> bool: - if _is_enforcement_enabled(): - return True - return _is_shadow_enabled() - - -def _is_parity_enabled() -> bool: - """Compatibility alias for callers that mean shadow parity.""" - return _is_shadow_enabled() +def _policy_is_configured() -> bool: + mode = _current_policy_mode() + if mode == PolicyMode.OFF: + return False + if mode == PolicyMode.SHADOW: + return policy_shadow_enabled() + return True def _corr_id(request: Request) -> str | None: return request.headers.get("X-Correlation-ID") or request.META.get("HTTP_X_CORRELATION_ID") -def _model_name_from_view(view: View | None) -> str | None: - if view is None: - return None - try: - serializer_class = view.get_serializer_class() # type: ignore[attr-defined] - return serializer_class.Meta.model.__name__ - except (AttributeError, TypeError): - return None +def _qualified_resource_kind(kind: str) -> str: + return "::".join([*POLICY_NAMESPACE, kind]) if POLICY_NAMESPACE else kind -def _build_resource_attrs( - resource_attrs: Mapping[str, str], -) -> dict[str, ResourceAttribute]: +def _build_resource_attrs(resource_attrs: Mapping[str, str]) -> dict[str, ResourceAttribute]: attrs: dict[str, ResourceAttribute] = {} for key, value in resource_attrs.items(): + normalized = str(value) + if normalized.lower() in {"true", "false"}: + attrs[key] = ResourceAttribute.new(normalized.lower(), ResourceAttributeType.BOOLEAN) + continue try: - ip = ipaddress.ip_address(value) + ip = ipaddress.ip_address(normalized) attrs[key] = ResourceAttribute.new(str(ip), ResourceAttributeType.IP) except ValueError: - attrs[key] = ResourceAttribute.new(value, ResourceAttributeType.STRING) + attrs[key] = ResourceAttribute.new(normalized, ResourceAttributeType.STRING) return attrs -def _fully_qualified_action(action: Action) -> str: - return str(action) - - -def _qualified_resource_kind(kind: str) -> str: - return "::".join([*POLICY_NAMESPACE, kind]) if POLICY_NAMESPACE else kind - - -def _build_policy_request(muser: MregUser, check: PolicyCheck) -> TreeTopRequest: - """Build a typed request using the bundle's qualified resource kind.""" - principal = TreeTopUser.new( - str(muser.username), - POLICY_NAMESPACE, - groups=list(muser.group_list), - ) - action = Action.new(check.action, POLICY_NAMESPACE) - attrs = _build_resource_attrs(check.resource.attrs) +def _build_policy_request( + user: MregUser, + check: PolicyCheck, + *, + request_id: str, +) -> TreeTopRequest: return TreeTopRequest( - principal=principal, - action=action, + id=request_id, + principal=TreeTopUser.new( + str(user.username), + POLICY_NAMESPACE, + groups=list(user.group_list), + ), + action=Action.new(check.action, POLICY_NAMESPACE), resource=TreeTopResource.new( kind=_qualified_resource_kind(check.resource.kind), id=check.resource.id, - attrs=attrs, + attrs=_build_resource_attrs(check.resource.attrs), ), ) -def _compute_parity_payload( - *, - decision: bool, - policy_allowed: bool | None, - error: str | None, - context: dict[str, object], -) -> dict[str, object]: - parity = policy_allowed is not None and bool(decision) is policy_allowed - return { - "parity": parity, - "legacy_decision": bool(decision), - "policy_decision": policy_allowed, - "error": error, - "context": context, - } - - -def _log_parity_payload(payload: dict[str, object]) -> None: - try: - legacy_decision = payload["legacy_decision"] - POLICY_LEGACY_DECISIONS_TOTAL.labels(decision="allow" if legacy_decision is True else "deny").inc() - - policy_decision = payload["policy_decision"] - if policy_decision is True: - policy_label = "allow" - elif policy_decision is False: - policy_label = "deny" - else: - policy_label = "error" - POLICY_DECISIONS_TOTAL.labels(decision=policy_label).inc() - - if payload["error"] is not None or policy_decision is None: - result = "error" - elif payload["parity"] is True: - result = "match" - else: - result = "mismatch" - POLICY_PARITY_RESULTS_TOTAL.labels(result=result).inc() +def _iter_leaves(node: PolicyNode) -> Iterator[PolicyLeaf]: + if isinstance(node, PolicyLeaf): + yield node + return + for child in node.children: + yield from _iter_leaves(child) + + +def _evaluate_tree(node: PolicyNode, decisions: Iterator[bool]) -> bool: + if isinstance(node, PolicyLeaf): + return next(decisions) + values = tuple(_evaluate_tree(child, decisions) for child in node.children) + if isinstance(node, PolicyAll): + return all(values) + return any(values) + + +def _node_fingerprint(node: PolicyNode) -> tuple[object, ...]: + if isinstance(node, PolicyLeaf): + resource = node.check.resource + return ( + "leaf", + node.check.action, + resource.kind, + resource.id, + tuple(sorted((str(key), str(value)) for key, value in resource.attrs.items())), + ) + return ( + "all" if isinstance(node, PolicyAll) else "any", + tuple(_node_fingerprint(child) for child in node.children), + ) - level = logging.INFO if payload["parity"] is True else logging.WARNING - event = "policy_parity_ok" if payload["parity"] is True else "policy_parity_mismatch" - _safe_log(level, event, **payload) - except Exception as exc: - _record_instrumentation_failure(exc, stage="result_logging") +def _result_decision(result: AuthorizeResultBrief, index: int) -> bool: + if result.index != index: + raise RuntimeError(f"Authorization result index {result.index} does not match {index}") + if result.id != f"mreg-{index}": + raise RuntimeError(f"Authorization result {index} has unexpected id={result.id!r}") + if not result.is_success(): + raise RuntimeError(result.error or f"Authorization result {index} failed with status={result.status}") + return result.is_allowed() -def _result_to_decision_and_error( - results: Sequence[AuthorizeResultBrief], - index: int, -) -> tuple[bool | None, str | None]: - if index >= len(results): - return None, f"Missing policy result at index {index}" - result = results[index] - if result.is_success(): - return result.is_allowed(), None - return None, result.error or f"Authorization failed with status={result.status}" +def _authorize_stack( + *, + request: Request, + root: PolicyNode, + context: dict[str, object], +) -> bool: + leaves = tuple(_iter_leaves(root)) + if not leaves: + raise RuntimeError("Endpoint policy stack is empty") + fingerprint = _node_fingerprint(root) + state = _request_state.get() + if state is not None and state.fingerprint is not None: + if state.fingerprint != fingerprint: + raise RuntimeError("A second different endpoint policy stack was evaluated in one request") + if state.error is not None: + raise RuntimeError(state.error) + if state.policy_decision is None: + raise RuntimeError("Cached endpoint policy stack has no decision") + return state.policy_decision + if not POLICY_BASE_URL: + raise RuntimeError("MREG_POLICY_BASE_URL is not configured") + if not _circuit.allow_call(): + raise RuntimeError("TreeTop circuit breaker is open") -def _authorize_with_metrics( - *, - policy_requests: Sequence[TreeTopRequest], - correlation_id: str | None, - path: str | None, -) -> tuple[list[AuthorizeResultBrief], str | None]: - if not policy_requests: - return [], None - - request_count = len(policy_requests) - POLICY_REQUESTS_PER_AUTHORIZE.observe(float(request_count)) + user = MregUser.from_request(request) + policy_requests = [_build_policy_request(user, leaf.check, request_id=f"mreg-{index}") for index, leaf in enumerate(leaves)] + POLICY_STACK_SIZE.observe(float(len(policy_requests))) started = monotonic() + if state is not None: + state.calls += 1 + state.fingerprint = fingerprint try: response = _get_treetop_client().authorize( policy_requests, - correlation_id=correlation_id, + correlation_id=_corr_id(request), ) + if len(response.results) != len(leaves): + raise RuntimeError(f"TreeTop returned {len(response.results)} results for {len(leaves)} checks") + ordered_results = sorted(response.results, key=lambda result: result.index) + decisions = tuple(_result_decision(result, index) for index, result in enumerate(ordered_results)) except Exception as exc: + _circuit.failure() POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="exception").inc() POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="exception").observe(monotonic() - started) - _safe_log( - logging.ERROR, - "policy_server_error", - error_type=type(exc).__name__, - error_msg=str(exc), - path=path, - correlation_id=correlation_id, - batch_size=request_count, - ) - return [], repr(exc) + error = f"{type(exc).__name__}: {exc}" + if state is not None: + state.error = error + raise RuntimeError(error) from exc + _circuit.success() POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="success").inc() POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="success").observe(monotonic() - started) - return response.results, None - - -def _process_policy_parity_batch(items: Sequence[_ParityBatchItem]) -> None: - """Evaluate and record one durable batch outside request threads. - - Delivery-level errors raise so the outbox can retry them. Successful - responses are recorded only after a successful authorize call and before - deleting the durable row. Delivery is at-least-once across process crashes. - """ - if not items: - return - - correlation_id = items[0].context.get("correlation_id") - path = items[0].context.get("path") - results, authorize_error = _authorize_with_metrics( - policy_requests=[item.policy_request for item in items], - correlation_id=correlation_id if isinstance(correlation_id, str) else None, - path=path if isinstance(path, str) else None, - ) - if authorize_error is not None: - raise RuntimeError(authorize_error) - parsed_results = [_result_to_decision_and_error(results, index) for index in range(len(items))] - result_error = next((error for _, error in parsed_results if error is not None), None) - if result_error is not None: - raise RuntimeError(result_error) - for index, item in enumerate(items): - policy_allowed, error = parsed_results[index] - _log_parity_payload( - _compute_parity_payload( - decision=item.decision, - policy_allowed=policy_allowed, - error=error, - context=item.context, - ) - ) - - -def flush_policy_parity_batch() -> bool: - """Persist and clear the current request batch, if one exists.""" - state = _request_state.get() - if state is None or not state.items: - return False - items = list(state.items) - state.items.clear() - submitted = _submit_policy_batch(items) - if submitted: - state.submitted_queries += 1 - return submitted + policy_decision = _evaluate_tree(root, iter(decisions)) + if state is not None: + state.policy_decision = policy_decision + if POLICY_PARITY_LOG_DETAILS: + context["checks"] = [ + { + "action": leaf.check.action, + "resource_kind": leaf.check.resource.kind, + "resource_id": leaf.check.resource.id, + "resource_attrs": dict(leaf.check.resource.attrs), + "decision": decisions[index], + } + for index, leaf in enumerate(leaves) + ] + return policy_decision -def _build_policy_context( +def authorize_policy_stack( + legacy_decision: bool, *, request: Request, - check: PolicyCheck, - view: View | None, - permission_class: str | None, -) -> dict[str, object]: - policy_action = Action.new(check.action, POLICY_NAMESPACE) - return { + root: PolicyNode, + view: View | None = None, + permission_class: str | None = None, +) -> bool: + """Synchronously evaluate one endpoint stack and apply the configured mode.""" + mode = _current_policy_mode() + legacy_decision = bool(legacy_decision) + if not _policy_is_configured(): + return legacy_decision + + context: dict[str, object] = { "path": request.path, "method": request.method, "permission": permission_class or (view and view.__class__.__name__), "view": view and view.__class__.__name__, - "model": _model_name_from_view(view), - "action": _fully_qualified_action(policy_action), - "resource_kind": _qualified_resource_kind(check.resource.kind), "correlation_id": _corr_id(request), - "mode": _current_policy_mode().value, + "mode": mode.value, } + try: + policy_decision = _authorize_stack(request=request, root=root, context=context) + except Exception as exc: + error = str(exc) + _record_failure("authorize", error, **context) + _record_parity(legacy_decision, None, error, context) + if mode == PolicyMode.ENFORCE: + with suppress(Exception): + POLICY_ENFORCEMENT_RESULTS_TOTAL.labels(result="error_deny").inc() + _safe_log(logging.CRITICAL, "policy_enforcement_failure", enforced_decision=False, error=error, **context) + return False + return legacy_decision - -def _record_enforcement_result(result: str) -> None: - with suppress(Exception): - POLICY_ENFORCEMENT_RESULTS_TOTAL.labels(result=result).inc() + _record_parity(legacy_decision, policy_decision, None, context) + if mode == PolicyMode.ENFORCE: + with suppress(Exception): + POLICY_ENFORCEMENT_RESULTS_TOTAL.labels(result="allow" if policy_decision else "deny").inc() + return policy_decision + return legacy_decision -def _enforcement_failure( - *, - decision: bool, - error: str, +def _record_parity( + legacy_decision: bool, + policy_decision: bool | None, + error: str | None, context: dict[str, object], - stage: str, -) -> bool: - """Apply the configured fail-closed or transitional legacy fallback.""" - _record_instrumentation_failure(RuntimeError(error), stage=stage) - _log_parity_payload( - _compute_parity_payload( - decision=decision, - policy_allowed=None, +) -> None: + with suppress(Exception): + POLICY_LEGACY_DECISIONS_TOTAL.labels(decision="allow" if legacy_decision else "deny").inc() + policy_label = "error" if policy_decision is None else "allow" if policy_decision else "deny" + POLICY_DECISIONS_TOTAL.labels(decision=policy_label).inc() + if error is not None or policy_decision is None: + result = "error" + elif legacy_decision == policy_decision: + result = "match" + else: + result = "mismatch" + POLICY_PARITY_RESULTS_TOTAL.labels(result=result).inc() + _safe_log( + logging.INFO if result == "match" else logging.WARNING, + "policy_stack_result", + parity=result == "match", + legacy_decision=legacy_decision, + policy_decision=policy_decision, error=error, context=context, ) - ) - failure_mode = _current_enforcement_failure_mode() - if failure_mode == EnforcementFailureMode.LEGACY: - result = "error_legacy" - enforced_decision = bool(decision) - else: - result = "error_deny" - enforced_decision = False - _record_enforcement_result(result) - _safe_log( - logging.CRITICAL, - "policy_enforcement_failure", - failure_mode=failure_mode.value, - enforced_decision=enforced_decision, - error=error, - **context, - ) - return enforced_decision - - -def _enforce_policy_decision( - *, - decision: bool, - policy_request: TreeTopRequest, - context: dict[str, object], -) -> bool: - """Synchronously return the authoritative TreeTop decision.""" - if not POLICY_BASE_URL: - return _enforcement_failure( - decision=decision, - error="MREG_POLICY_BASE_URL is not configured", - context=context, - stage="enforce_configuration", - ) - - state = _request_state.get() - if state is not None: - state.submitted_queries += 1 - try: - results, authorize_error = _authorize_with_metrics( - policy_requests=[policy_request], - correlation_id=context.get("correlation_id") - if isinstance(context.get("correlation_id"), str) - else None, - path=context.get("path") if isinstance(context.get("path"), str) else None, - ) - except Exception as exc: - return _enforcement_failure( - decision=decision, - error=f"{type(exc).__name__}: {exc}", - context=context, - stage="enforce_instrumentation", - ) - if authorize_error is not None: - return _enforcement_failure( - decision=decision, - error=authorize_error, - context=context, - stage="enforce_authorize", - ) - - policy_allowed, result_error = _result_to_decision_and_error(results, 0) - if result_error is not None or policy_allowed is None: - return _enforcement_failure( - decision=decision, - error=result_error or "TreeTop returned no decision", - context=context, - stage="enforce_result", - ) - - _log_parity_payload( - _compute_parity_payload( - decision=decision, - policy_allowed=policy_allowed, - error=None, - context=context, - ) - ) - _record_enforcement_result("allow" if policy_allowed else "deny") - return policy_allowed def policy_parity( @@ -1004,64 +601,11 @@ def policy_parity( view: View | None = None, permission_class: str | None = None, ) -> bool: - """Apply the configured off, shadow, or enforce policy behavior.""" - mode = _current_policy_mode() - if mode == PolicyMode.OFF: - return decision - if mode == PolicyMode.SHADOW and not _is_shadow_enabled(): - return decision - - context: dict[str, object] = { - "path": request.path, - "method": request.method, - "action": check.action, - "resource_kind": check.resource.kind, - "mode": mode.value, - } - try: - context = _build_policy_context( - request=request, - check=check, - view=view, - permission_class=permission_class, - ) - muser = MregUser.from_request(request) - policy_request = _build_policy_request(muser, check) - if POLICY_PARITY_LOG_DETAILS: - context.update( - { - "principal": muser.username, - "groups": list(muser.group_list), - "resource_id": check.resource.id, - "resource_attrs": dict(check.resource.attrs), - } - ) - except Exception as exc: - if mode == PolicyMode.ENFORCE: - return _enforcement_failure( - decision=decision, - error=f"{type(exc).__name__}: {exc}", - context=context, - stage="enforce_build", - ) - _record_instrumentation_failure(exc, stage="shadow_build") - return decision - - if mode == PolicyMode.ENFORCE: - return _enforce_policy_decision( - decision=decision, - policy_request=policy_request, - context=context, - ) - - item = _ParityBatchItem( - decision=bool(decision), - policy_request=policy_request, - context=context, + """Compatibility wrapper for a single-leaf endpoint stack.""" + return authorize_policy_stack( + decision, + request=request, + root=PolicyLeaf(check), + view=view, + permission_class=permission_class, ) - state = _request_state.get() - if state is not None and POLICY_PARITY_BATCH_ENABLED: - state.items.append(item) - elif _submit_policy_batch([item]) and state is not None: - state.submitted_queries += 1 - return decision diff --git a/mreg/api/v1/tests/test_logging.py b/mreg/api/v1/tests/test_logging.py index c82fac12..48646af3 100644 --- a/mreg/api/v1/tests/test_logging.py +++ b/mreg/api/v1/tests/test_logging.py @@ -143,8 +143,8 @@ def mock_get_response(_): # Check that the body was logged as '' self.assertEqual(cap_logs[0]["content"], "") - def test_middleware_uses_policy_parity_batching_context(self) -> None: - """Ensure request handling is wrapped in the parity batching context.""" + def test_middleware_uses_policy_request_scope(self) -> None: + """Ensure request handling is wrapped in one policy request scope.""" middleware = LoggingMiddleware(MagicMock()) def mock_get_response(_): @@ -156,11 +156,11 @@ def mock_get_response(_): request._body = b"Some request body" request.user = get_user_model().objects.get(username="superuser") - with patch("mreg.middleware.logging_http.batch_policy_parity") as mock_batch: + with patch("mreg.middleware.logging_http.policy_request_scope") as mock_scope: middleware(request) - mock_batch.assert_called_once() - mock_batch.return_value.__enter__.assert_called_once() - mock_batch.return_value.__exit__.assert_called_once() + mock_scope.assert_called_once() + mock_scope.return_value.__enter__.assert_called_once() + mock_scope.return_value.__exit__.assert_called_once() class TestLoggingMiddleware(MregAPITestCase): diff --git a/mreg/api/v1/views.py b/mreg/api/v1/views.py index e1e6f5a5..71b38a37 100644 --- a/mreg/api/v1/views.py +++ b/mreg/api/v1/views.py @@ -23,11 +23,12 @@ from mreg.types import IPAllocationMethod from mreg.api.responses import created_response, error_response +from mreg.api.treetop import policy_enforcement_enabled from mreg.api.permissions import ( - IsAuthenticatedAndReadOnly, + HostContactsPermission, + IsNetworkAdminOrReadOnly, IsGrantedNetGroupRegexPermission, IsSuperOrAdminOrReadOnly, - IsSuperOrNetworkAdminMember, IsGrantedReservedAddressPermission, ) @@ -557,7 +558,12 @@ class HostContactsView(HostPermissionsUpdateDestroy, APIView): """ policy_resource_kind = "Host" - policy_actions = {"read": "host_contacts_read"} + policy_actions = { + "read": "host_contacts_read", + "create": "host_contacts_create", + "delete": "host_contacts_delete", + } + permission_classes = (HostContactsPermission,) def get_host(self, name): """Get the host object by name.""" @@ -952,7 +958,7 @@ class NetworkList(MregListCreateAPIView): queryset = Network.objects.all().prefetch_related("excluded_ranges") serializer_class = NetworkSerializer - permission_classes = (IsSuperOrNetworkAdminMember | IsAuthenticatedAndReadOnly,) + permission_classes = (IsNetworkAdminOrReadOnly,) lookup_field = "network" location_lookup_safe = "/:" filterset_class = NetworkFilterSet @@ -978,7 +984,7 @@ class NetworkDetail(MregRetrieveUpdateDestroyAPIView): queryset = Network.objects.all() serializer_class = NetworkSerializer - permission_classes = (IsSuperOrNetworkAdminMember | IsAuthenticatedAndReadOnly,) + permission_classes = (IsNetworkAdminOrReadOnly,) lookup_field = "network" location_lookup_safe = "/:" @@ -1025,7 +1031,7 @@ class NetworkExcludedRangeList(MregListCreateAPIView): """ serializer_class = NetworkExcludedRangeSerializer - permission_classes = (IsSuperOrNetworkAdminMember | IsAuthenticatedAndReadOnly,) + permission_classes = (IsNetworkAdminOrReadOnly,) def get_queryset(self): """ @@ -1053,7 +1059,7 @@ class NetworkExcludedRangeDetail(MregRetrieveUpdateDestroyAPIView): """ serializer_class = NetworkExcludedRangeSerializer - permission_classes = (IsSuperOrNetworkAdminMember | IsAuthenticatedAndReadOnly,) + permission_classes = (IsNetworkAdminOrReadOnly,) lookup_field = "pk" def get_queryset(self): @@ -1244,6 +1250,14 @@ class NetGroupRegexPermissionList(MregListCreateAPIView): permission_classes = (IsSuperOrAdminOrReadOnly,) filterset_class = NetGroupRegexPermissionFilterSet + def post(self, request, *args, **kwargs): + if policy_enforcement_enabled(): + return error_response( + "NetGroupRegexPermission is bundle-managed while TreeTop enforcement is enabled.", + status.HTTP_409_CONFLICT, + ) + return super().post(request, *args, **kwargs) + class NetGroupRegexPermissionDetail(MregRetrieveUpdateDestroyAPIView): """ """ @@ -1252,6 +1266,27 @@ class NetGroupRegexPermissionDetail(MregRetrieveUpdateDestroyAPIView): serializer_class = NetGroupRegexPermissionSerializer permission_classes = (IsSuperOrAdminOrReadOnly,) + def _reject_bundle_managed_write(self): + return error_response( + "NetGroupRegexPermission is bundle-managed while TreeTop enforcement is enabled.", + status.HTTP_409_CONFLICT, + ) + + def put(self, request, *args, **kwargs): + if policy_enforcement_enabled(): + return self._reject_bundle_managed_write() + return super().put(request, *args, **kwargs) + + def patch(self, request, *args, **kwargs): + if policy_enforcement_enabled(): + return self._reject_bundle_managed_write() + return super().patch(request, *args, **kwargs) + + def delete(self, request, *args, **kwargs): + if policy_enforcement_enabled(): + return self._reject_bundle_managed_write() + return super().delete(request, *args, **kwargs) + def _get_iprange(kwargs): """ diff --git a/mreg/api/v1/views_bacnet.py b/mreg/api/v1/views_bacnet.py index 0f5a0e88..da554964 100644 --- a/mreg/api/v1/views_bacnet.py +++ b/mreg/api/v1/views_bacnet.py @@ -6,7 +6,7 @@ MregListCreateAPIView, MregRetrieveUpdateDestroyAPIView, ) -from mreg.api.permissions import IsGrantedNetGroupRegexPermission +from mreg.api.permissions import BACnetPermission from mreg.models.host import Host, BACnetID from . import serializers @@ -16,7 +16,7 @@ class BACnetIDList(MregListCreateAPIView): queryset = BACnetID.objects.order_by("id") serializer_class = serializers.BACnetIDSerializer - permission_classes = (IsGrantedNetGroupRegexPermission,) + permission_classes = (BACnetPermission,) lookup_field = "id" filterset_fields = "id" filterset_class = BACnetIDFilterSet @@ -65,7 +65,7 @@ def post(self, request, *args, **kwargs): class BACnetIDDetail(MregRetrieveUpdateDestroyAPIView): queryset = BACnetID.objects.all() serializer_class = serializers.BACnetIDSerializer - permission_classes = (IsGrantedNetGroupRegexPermission,) + permission_classes = (BACnetPermission,) lookup_field = "id" # Don't allow patch or put requests diff --git a/mreg/api/v1/views_hostgroups.py b/mreg/api/v1/views_hostgroups.py index 6846263c..4f69640c 100644 --- a/mreg/api/v1/views_hostgroups.py +++ b/mreg/api/v1/views_hostgroups.py @@ -8,7 +8,6 @@ from mreg.api.permissions import (HostGroupPermission, IsSuperOrGroupAdminOrReadOnly) from mreg.models.host import Host, HostGroup -from mreg.models.auth import User from mreg.mixins import LowerCaseLookupMixin @@ -28,13 +27,8 @@ class HostGroupM2MPermissions(M2MPermissions): def check_m2m_update_permission(self, request): for permission in self.get_permissions(): - if isinstance(self, (HostGroupOwnersList, HostGroupOwnersDetail)): - user = User.from_request(request) - if not (user.is_mreg_superuser or user.is_mreg_hostgroup_admin): - self.permission_denied(request) - else: - if not permission.has_m2m_change_permission(request, self): - self.permission_denied(request) + if not permission.has_m2m_change_permission(request, self): + self.permission_denied(request) class HostGroupLogMixin(HistoryLog): diff --git a/mreg/api/v1/views_network_policy.py b/mreg/api/v1/views_network_policy.py index 660b54da..ff47e1d8 100644 --- a/mreg/api/v1/views_network_policy.py +++ b/mreg/api/v1/views_network_policy.py @@ -23,7 +23,10 @@ from mreg.api.responses import created_response_at_url from mreg.api.v1.views import JSONContentTypeMixin, HistoryLog -from mreg.api.permissions import IsGrantedNetGroupRegexPermission, IsSuperOrNetworkAdminMember +from mreg.api.permissions import ( + IsGrantedNetGroupRegexOrNetworkAdmin, + IsSuperOrNetworkAdminMember, +) from mreg.api.v1.endpoints import URL class CommunityLogMixin(HistoryLog): @@ -131,7 +134,7 @@ class NetworkPolicyAttributeDetail(JSONContentTypeMixin, generics.RetrieveUpdate class NetworkCommunityList(JSONContentTypeMixin, CommunityLogMixin, generics.ListCreateAPIView): serializer_class = CommunitySerializer - permission_classes = (IsGrantedNetGroupRegexPermission | IsSuperOrNetworkAdminMember,) + permission_classes = (IsGrantedNetGroupRegexOrNetworkAdmin,) filterset_class = CommunityFilterSet def get_queryset(self): @@ -176,7 +179,7 @@ def create(self, request, *args, **kwargs): # Retrieve, update, or delete a specific Community under a specific Network class NetworkCommunityDetail(JSONContentTypeMixin, CommunityLogMixin, generics.RetrieveUpdateDestroyAPIView): serializer_class = CommunitySerializer - permission_classes = (IsGrantedNetGroupRegexPermission | IsSuperOrNetworkAdminMember,) + permission_classes = (IsGrantedNetGroupRegexOrNetworkAdmin,) def get_queryset(self): network = self.kwargs.get("network") @@ -213,7 +216,7 @@ def get_policy_and_community(self): # List all hosts in a specific community, or add a host to a community class NetworkCommunityHostList(HostInCommunityMixin, generics.ListCreateAPIView): serializer_class = HostSerializer - permission_classes = (IsGrantedNetGroupRegexPermission | IsSuperOrNetworkAdminMember,) + permission_classes = (IsGrantedNetGroupRegexOrNetworkAdmin,) def get_queryset(self): if "network" not in self.kwargs or "cpk" not in self.kwargs: @@ -269,7 +272,7 @@ def create(self, request, *args, **kwargs): # Retrieve or delete a specific host in a specific community class NetworkCommunityHostDetail(HostInCommunityMixin, generics.RetrieveDestroyAPIView): serializer_class = HostSerializer - permission_classes = (IsGrantedNetGroupRegexPermission | IsSuperOrNetworkAdminMember,) + permission_classes = (IsGrantedNetGroupRegexOrNetworkAdmin,) def get_queryset(self): if "network" not in self.kwargs or "cpk" not in self.kwargs: diff --git a/mreg/api/v1/views_zones.py b/mreg/api/v1/views_zones.py index 42db4121..97c725f9 100644 --- a/mreg/api/v1/views_zones.py +++ b/mreg/api/v1/views_zones.py @@ -19,7 +19,7 @@ from mreg.mixins import LowerCaseLookupMixin from mreg.api.responses import created_response, error_response -from mreg.api.permissions import (IsSuperGroupMember, IsAuthenticatedAndReadOnly) +from mreg.api.permissions import IsSuperOrReadOnly from .serializers import (ForwardZoneByHostnameSerializer, ForwardZoneDelegationSerializer, ForwardZoneSerializer, ReverseZoneDelegationSerializer, ReverseZoneSerializer) @@ -90,7 +90,7 @@ class ZoneList(generics.ListCreateAPIView): """ lookup_field = 'name' - permission_classes = (IsSuperGroupMember | IsAuthenticatedAndReadOnly, ) + permission_classes = (IsSuperOrReadOnly,) def get_queryset(self): qs = super().get_queryset() @@ -142,7 +142,7 @@ class ZoneDelegationList(generics.ListCreateAPIView): """ lookup_field = 'name' - permission_classes = (IsSuperGroupMember | IsAuthenticatedAndReadOnly, ) + permission_classes = (IsSuperOrReadOnly,) def get_queryset(self): if self.lookup_field not in self.kwargs: @@ -201,7 +201,7 @@ class ZoneDetail(LowerCaseLookupMixin, MregRetrieveUpdateDestroyAPIView): """ lookup_field = 'name' - permission_classes = (IsSuperGroupMember | IsAuthenticatedAndReadOnly, ) + permission_classes = (IsSuperOrReadOnly,) def patch(self, request, *args, **kwargs): query = self.kwargs[self.lookup_field] @@ -258,7 +258,7 @@ class ReverseZoneDetail(ZoneDetail): class ZoneDelegationDetail(LowerCaseLookupMixin, MregRetrieveUpdateDestroyAPIView): lookup_field = 'delegation' - permission_classes = (IsSuperGroupMember | IsAuthenticatedAndReadOnly, ) + permission_classes = (IsSuperOrReadOnly,) def get_queryset(self): parentname = self.kwargs['name'] @@ -321,7 +321,7 @@ class ZoneNameServerDetail(MregRetrieveUpdateDestroyAPIView): """ lookup_field = 'name' - permission_classes = (IsSuperGroupMember | IsAuthenticatedAndReadOnly, ) + permission_classes = (IsSuperOrReadOnly,) def get(self, request, *args, **kwargs): zone = self.get_object() diff --git a/mreg/api/views.py b/mreg/api/views.py index 7b3fb3de..a6e64c96 100644 --- a/mreg/api/views.py +++ b/mreg/api/views.py @@ -15,8 +15,7 @@ from django.contrib.auth.models import update_last_login from rest_framework import serializers, status from rest_framework.authtoken.views import ObtainAuthToken -from rest_framework.exceptions import AuthenticationFailed, NotFound, PermissionDenied -from rest_framework.permissions import IsAuthenticated +from rest_framework.exceptions import AuthenticationFailed, NotFound from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView @@ -32,7 +31,11 @@ ) from mreg.__about__ import __version__ as mreg_version -from mreg.api.permissions import IsSuperOrNetworkAdminMember +from mreg.api.permissions import ( + IsAuthenticatedWithPolicy, + IsSuperOrNetworkAdminMember, + UserInfoPermission, +) from mreg.api.serializers import ( HealthHeartbeatSerializer, MetaVersionsSerializer, @@ -149,7 +152,7 @@ def post(self, request: Request, *args: Any, **kwargs: Any): class TokenLogout(APIView): - permission_classes = (IsAuthenticated,) + permission_classes = (IsAuthenticatedWithPolicy,) @extend_schema(request=None, responses={status.HTTP_200_OK: None}) def post(self, request: Request): @@ -160,7 +163,7 @@ def post(self, request: Request): class TokenIsValid(APIView): - permission_classes = (IsAuthenticated,) + permission_classes = (IsAuthenticatedWithPolicy,) @extend_schema(responses={status.HTTP_200_OK: None}) def get(self, request: Request): @@ -172,7 +175,7 @@ def get(self, request: Request): class UserInfo(APIView): - permission_classes = (IsAuthenticated,) + permission_classes = (UserInfoPermission,) @extend_schema( parameters=[ @@ -195,8 +198,6 @@ def get(self, request: Request): target_user = req_user if username and username != req_user.username: - if not (req_user.is_mreg_superuser_or_admin or req_user.is_mreg_hostgroup_admin): - raise PermissionDenied("You do not have permission to view other users' details.") try: target_user = User.objects.get(username=username) except User.DoesNotExist: @@ -256,7 +257,7 @@ def get(self, request: Request): ### class MregVersion(APIView): - permission_classes = (IsAuthenticated,) + permission_classes = (IsAuthenticatedWithPolicy,) @extend_schema(responses={status.HTTP_200_OK: MregVersionSerializer}) def get(self, request: Request): @@ -288,6 +289,8 @@ def get(self, request: Request): class HealthHeartbeat(APIView): + permission_classes = () + @extend_schema(responses={status.HTTP_200_OK: HealthHeartbeatSerializer}) def get(self, request: Request): uptime = int(time.time() - start_time) @@ -299,6 +302,8 @@ def get(self, request: Request): class HealthLDAP(APIView): + permission_classes = () + @extend_schema( responses={ status.HTTP_200_OK: None, diff --git a/mreg/management/commands/check_policy_rollout.py b/mreg/management/commands/check_policy_rollout.py index cccc34e6..f598e578 100644 --- a/mreg/management/commands/check_policy_rollout.py +++ b/mreg/management/commands/check_policy_rollout.py @@ -19,10 +19,6 @@ def handle(self, *args, **options): # type: ignore[no-untyped-def] min_comparisons=settings.POLICY_ROLLOUT_MIN_COMPARISONS, max_mismatch_rate=settings.POLICY_ROLLOUT_MAX_MISMATCH_RATE, max_error_rate=settings.POLICY_ROLLOUT_MAX_ERROR_RATE, - max_persist_failures=settings.POLICY_ROLLOUT_MAX_PERSIST_FAILURES, - max_dead_letters=settings.POLICY_ROLLOUT_MAX_DEAD_LETTERS, - max_pending_batches=settings.POLICY_ROLLOUT_MAX_PENDING_BATCHES, - max_backlog_age_seconds=settings.POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS, ) try: snapshot = fetch_rollout_snapshot( @@ -36,11 +32,7 @@ def handle(self, *args, **options): # type: ignore[no-untyped-def] summary = ( f"comparisons={snapshot.comparisons:g} " f"mismatch_rate={snapshot.mismatch_rate:.6f} " - f"error_rate={snapshot.error_rate:.6f} " - f"persist_failures={snapshot.persist_failures:g} " - f"dead_letters={snapshot.dead_letters:g} " - f"pending_batches={snapshot.pending_batches:g} " - f"backlog_age_seconds={snapshot.backlog_age_seconds:g}" + f"error_rate={snapshot.error_rate:.6f}" ) if not evaluation.ready: raise CommandError(f"TreeTop rollout gate failed: {'; '.join(evaluation.reasons)} ({summary})") diff --git a/mreg/middleware/logging_http.py b/mreg/middleware/logging_http.py index 46dd5edb..2f5b827f 100644 --- a/mreg/middleware/logging_http.py +++ b/mreg/middleware/logging_http.py @@ -10,7 +10,7 @@ import traceback from django.conf import settings from django.http import HttpRequest, HttpResponse -from mreg.api.treetop import batch_policy_parity +from mreg.api.treetop import policy_request_scope mreg_logger = structlog.getLogger("mreg.http") @@ -48,7 +48,7 @@ def __call__(self, request: HttpRequest) -> HttpResponse: self.log_request(request) - with batch_policy_parity(): + with policy_request_scope(): try: response = self.get_response(request) except Exception as e: # pragma: no cover (this is somewhat tricky to properly test) diff --git a/mreg/migrations/0017_policyparityoutbox.py b/mreg/migrations/0017_policyparityoutbox.py deleted file mode 100644 index 2878427f..00000000 --- a/mreg/migrations/0017_policyparityoutbox.py +++ /dev/null @@ -1,35 +0,0 @@ -# Generated by Django 5.2 for MREG's durable policy parity outbox. - -import django.utils.timezone -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("mreg", "0016_host_contacts"), - ] - - operations = [ - migrations.CreateModel( - name="PolicyParityOutbox", - fields=[ - ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("payload", models.JSONField()), - ("attempts", models.PositiveIntegerField(default=0)), - ("available_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)), - ("locked_at", models.DateTimeField(blank=True, db_index=True, null=True)), - ("failed_at", models.DateTimeField(blank=True, db_index=True, null=True)), - ("last_error", models.TextField(blank=True, default="")), - ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)), - ], - options={ - "verbose_name_plural": "policy parity outbox entries", - "indexes": [ - models.Index( - fields=["failed_at", "available_at", "id"], - name="policy_outbox_ready_idx", - ) - ], - }, - ), - ] diff --git a/mreg/models/__init__.py b/mreg/models/__init__.py index 3f132229..b099b24b 100644 --- a/mreg/models/__init__.py +++ b/mreg/models/__init__.py @@ -1,4 +1,3 @@ """Models for mreg.""" from .auth import User # noqa: F401, needed by mreg.settings for now -from .policy import PolicyParityOutbox # noqa: F401 diff --git a/mreg/models/policy.py b/mreg/models/policy.py deleted file mode 100644 index db6c7525..00000000 --- a/mreg/models/policy.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Durable policy parity delivery models.""" - -from django.db import models -from django.utils import timezone - - -class PolicyParityOutbox(models.Model): - """One durable, shared TreeTop parity batch. - - Successful rows are deleted. Rows that exhaust their retries remain as - dead letters so operators can inspect and explicitly resolve them. - """ - - payload = models.JSONField() - attempts = models.PositiveIntegerField(default=0) - available_at = models.DateTimeField(default=timezone.now, db_index=True) - locked_at = models.DateTimeField(null=True, blank=True, db_index=True) - failed_at = models.DateTimeField(null=True, blank=True, db_index=True) - last_error = models.TextField(blank=True, default="") - created_at = models.DateTimeField(auto_now_add=True, db_index=True) - - class Meta: - indexes = [ - models.Index(fields=("failed_at", "available_at", "id"), name="policy_outbox_ready_idx"), - ] - verbose_name_plural = "policy parity outbox entries" diff --git a/mreg/policy/config.py b/mreg/policy/config.py index 7d495b67..44393373 100644 --- a/mreg/policy/config.py +++ b/mreg/policy/config.py @@ -13,13 +13,6 @@ class PolicyMode(StrEnum): ENFORCE = "enforce" -class EnforcementFailureMode(StrEnum): - """Decision used when authoritative TreeTop evaluation fails.""" - - DENY = "deny" - LEGACY = "legacy" - - def resolve_policy_mode(raw: str | None, *, legacy_parity_enabled: bool) -> PolicyMode: """Resolve the explicit mode, falling back to the deprecated boolean.""" candidate = (raw or "").strip().lower() @@ -31,17 +24,6 @@ def resolve_policy_mode(raw: str | None, *, legacy_parity_enabled: bool) -> Poli raise ValueError("MREG_POLICY_MODE must be one of: off, shadow, enforce") from exc -def resolve_enforcement_failure_mode(raw: str | None) -> EnforcementFailureMode: - """Parse the explicit behavior used when TreeTop cannot decide.""" - candidate = (raw or EnforcementFailureMode.DENY).strip().lower() - try: - return EnforcementFailureMode(candidate) - except ValueError as exc: - raise ValueError( - "MREG_POLICY_ENFORCEMENT_FAILURE_MODE must be one of: deny, legacy" - ) from exc - - def validate_policy_configuration(mode: PolicyMode, base_url: str) -> None: """Reject configurations that cannot provide authoritative decisions.""" if mode == PolicyMode.ENFORCE and not base_url.strip(): diff --git a/mreg/policy/contracts.py b/mreg/policy/contracts.py index 57624509..c0400328 100644 --- a/mreg/policy/contracts.py +++ b/mreg/policy/contracts.py @@ -48,7 +48,7 @@ def snake_case(value: str) -> str: return re.sub(r"[^a-zA-Z0-9_]+", "_", value).strip("_").lower() or "generic" -HOST_ATTRIBUTES = tuple( +ENDPOINT_ATTRIBUTES = tuple( ResourceAttributeContract(name, cedar_type) for name, cedar_type in ( ("kind", "String"), @@ -58,25 +58,32 @@ def snake_case(value: str) -> str: ("hostname", "String"), ("ip", "ipaddr"), ("nameLabels", "Set"), + ("dnsWildcard", "Bool"), + ("dnsWildcardValidDepth", "Bool"), + ("dnsUnderscore", "Bool"), + ("ipReserved", "Bool"), + ("ipRestricted", "Bool"), + ("selfAccess", "Bool"), + ("requesterIsOwner", "Bool"), + ("ownerMutation", "Bool"), + ("descriptionUpdate", "Bool"), + ("roleLabel", "String"), + ("network", "String"), ) ) RESOURCE_CONTRACTS = ( - ResourceContract("Generic"), - ResourceContract("Host", CRUD_OPERATIONS, HOST_ATTRIBUTES), - ResourceContract("HostContact", identifier_fields=("pk", "id", "email")), + ResourceContract("Generic", attributes=ENDPOINT_ATTRIBUTES), + ResourceContract("Host", CRUD_OPERATIONS, ENDPOINT_ATTRIBUTES), + ResourceContract("HostContact", attributes=ENDPOINT_ATTRIBUTES, identifier_fields=("pk", "id", "email")), ResourceContract( "Ipaddress", CRUD_OPERATIONS, - ( - ResourceAttributeContract("kind"), - ResourceAttributeContract("id"), - ResourceAttributeContract("ip", "ipaddr"), - ), + ENDPOINT_ATTRIBUTES, identifier_fields=("pk", "id", "ipaddress"), ), - *(ResourceContract(kind, CRUD_OPERATIONS) for kind in ( + *(ResourceContract(kind, CRUD_OPERATIONS, ENDPOINT_ATTRIBUTES) for kind in ( "Cname", "Hinfo", "Loc", @@ -95,6 +102,15 @@ def snake_case(value: str) -> str: "NetworkPolicy", "NetworkPolicyAttribute", "NetworkPolicyAttributeValue", + "HostGroup", + "NetworkExcludedRange", + "ForwardZone", + "ForwardZoneDelegation", + "ReverseZone", + "ReverseZoneDelegation", + "HostPolicyAtom", + "HostPolicyRole", + "NetGroupRegexPermission", )), ) @@ -114,16 +130,23 @@ def snake_case(value: str) -> str: CUSTOM_ACTIONS = frozenset( { *MEMBERSHIP_ACTIONS.values(), + "authenticated_access", "create_label", "delete_label", "edit_label", "host_contacts_read", + "host_contacts_create", + "host_contacts_delete", + "hostgroup_membership_update", + "hostpolicy_role_atom_membership_update", + "hostpolicy_role_host_membership_update", "ip_broadcast_management", "ip_gw_management", "ip_network_management", "ip_reserved_management", "ip_restricted_management", "is_superuser", + "user_info_read", "view_label", } ) diff --git a/mreg/policy/rollout.py b/mreg/policy/rollout.py index e0b86543..b954259d 100644 --- a/mreg/policy/rollout.py +++ b/mreg/policy/rollout.py @@ -13,10 +13,6 @@ class RolloutThresholds: min_comparisons: int = 10_000 max_mismatch_rate: float = 0.001 max_error_rate: float = 0.001 - max_persist_failures: int = 0 - max_dead_letters: int = 0 - max_pending_batches: int = 0 - max_backlog_age_seconds: float = 300.0 @dataclass(frozen=True, slots=True) @@ -24,10 +20,6 @@ class RolloutSnapshot: comparisons: float mismatches: float errors: float - persist_failures: float - dead_letters: float - pending_batches: float - backlog_age_seconds: float @property def mismatch_rate(self) -> float: @@ -54,18 +46,6 @@ def evaluate_rollout(snapshot: RolloutSnapshot, thresholds: RolloutThresholds) - reasons.append(f"mismatch rate {snapshot.mismatch_rate:.6f} > {thresholds.max_mismatch_rate:.6f}") if snapshot.error_rate > thresholds.max_error_rate: reasons.append(f"error rate {snapshot.error_rate:.6f} > {thresholds.max_error_rate:.6f}") - if snapshot.persist_failures > thresholds.max_persist_failures: - reasons.append(f"persist failures {snapshot.persist_failures:g} > {thresholds.max_persist_failures}") - if snapshot.dead_letters > thresholds.max_dead_letters: - reasons.append(f"dead letters {snapshot.dead_letters:g} > {thresholds.max_dead_letters}") - if snapshot.pending_batches > thresholds.max_pending_batches: - reasons.append( - f"pending batches {snapshot.pending_batches:g} > {thresholds.max_pending_batches}" - ) - if snapshot.backlog_age_seconds > thresholds.max_backlog_age_seconds: - reasons.append( - f"oldest backlog age {snapshot.backlog_age_seconds:g}s > {thresholds.max_backlog_age_seconds:g}s" - ) return RolloutEvaluation(ready=not reasons, reasons=tuple(reasons)) @@ -87,15 +67,11 @@ def fetch_rollout_snapshot( window: str = "24h", timeout: float = 10.0, ) -> RolloutSnapshot: - """Read the six low-cardinality signals required by the rollout gate.""" + """Read the composite parity signals required by the rollout gate.""" queries = { "comparisons": f'sum(increase(mreg_policy_parity_results_total{{result=~"match|mismatch"}}[{window}]))', "mismatches": f'sum(increase(mreg_policy_parity_results_total{{result="mismatch"}}[{window}]))', "errors": f'sum(increase(mreg_policy_parity_results_total{{result="error"}}[{window}]))', - "persist_failures": f'sum(increase(mreg_policy_parity_batches_total{{status="persist_failed"}}[{window}]))', - "dead_letters": 'max(mreg_policy_parity_outbox_entries{status="dead_letter"})', - "pending_batches": 'max(mreg_policy_parity_outbox_entries{status="pending"})', - "backlog_age_seconds": "max(mreg_policy_parity_outbox_oldest_seconds)", } values = { name: _prometheus_value(prometheus_url, query, timeout) diff --git a/mreg/tests/test_gunicorn_conf.py b/mreg/tests/test_gunicorn_conf.py index e44a1f3a..e49d5fef 100644 --- a/mreg/tests/test_gunicorn_conf.py +++ b/mreg/tests/test_gunicorn_conf.py @@ -10,43 +10,28 @@ class GunicornLifecycleHookTests(SimpleTestCase): - """Ensure parity dispatchers follow each Gunicorn worker lifecycle.""" + """Ensure worker-local policy clients follow Gunicorn lifecycle.""" - @patch("django.setup") - @patch("django.apps.apps") - def test_setup_initializes_django_when_apps_are_not_ready(self, apps, django_setup): - apps.ready = False - - gunicorn_conf._setup_django() - - django_setup.assert_called_once_with() - - @patch("mreg.api.treetop.start_policy_parity_dispatcher") - def test_post_fork_starts_dispatcher(self, start_dispatcher): - gunicorn_conf.post_fork(None, None) - - start_dispatcher.assert_called_once_with() - - @patch("mreg.api.treetop.stop_policy_parity_dispatcher") - def test_worker_exit_stops_dispatcher(self, stop_dispatcher): + @patch("mreg.api.treetop.close_policy_client") + def test_worker_exit_closes_policy_client(self, close_client): gunicorn_conf.worker_exit(None, None) - stop_dispatcher.assert_called_once_with() + close_client.assert_called_once_with() - @patch("mreg.api.treetop.stop_policy_parity_dispatcher") + @patch("mreg.api.treetop.close_policy_client") @patch("django.apps.apps") - def test_worker_exit_is_safe_before_django_setup(self, apps, stop_dispatcher): + def test_worker_exit_is_safe_before_django_setup(self, apps, close_client): apps.ready = False gunicorn_conf.worker_exit(None, SimpleNamespace(pid=42)) - stop_dispatcher.assert_not_called() + close_client.assert_not_called() @patch("prometheus_client.multiprocess.mark_process_dead") - @patch("mreg.api.treetop.stop_policy_parity_dispatcher") - def test_worker_exit_marks_prometheus_process_dead(self, stop_dispatcher, mark_process_dead): + @patch("mreg.api.treetop.close_policy_client") + def test_worker_exit_marks_prometheus_process_dead(self, close_client, mark_process_dead): with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": "/tmp/prometheus"}): gunicorn_conf.worker_exit(None, SimpleNamespace(pid=42)) - stop_dispatcher.assert_called_once_with() + close_client.assert_called_once_with() mark_process_dead.assert_called_once_with(42) diff --git a/mreg/tests/test_policy_config.py b/mreg/tests/test_policy_config.py index 0070dd10..7f7fd75e 100644 --- a/mreg/tests/test_policy_config.py +++ b/mreg/tests/test_policy_config.py @@ -1,9 +1,7 @@ from django.test import SimpleTestCase from mreg.policy.config import ( - EnforcementFailureMode, PolicyMode, - resolve_enforcement_failure_mode, resolve_policy_mode, validate_policy_configuration, ) @@ -30,20 +28,6 @@ def test_invalid_policy_mode_is_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "off, shadow, enforce"): resolve_policy_mode("invalid", legacy_parity_enabled=True) - def test_enforcement_failure_mode_defaults_to_deny(self) -> None: - self.assertEqual( - resolve_enforcement_failure_mode(None), - EnforcementFailureMode.DENY, - ) - self.assertEqual( - resolve_enforcement_failure_mode(" legacy "), - EnforcementFailureMode.LEGACY, - ) - - def test_invalid_enforcement_failure_mode_is_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "deny, legacy"): - resolve_enforcement_failure_mode("allow") - def test_enforcement_requires_a_base_url(self) -> None: with self.assertRaisesRegex(ValueError, "MREG_POLICY_BASE_URL"): validate_policy_configuration(PolicyMode.ENFORCE, "") diff --git a/mreg/tests/test_policy_rollout.py b/mreg/tests/test_policy_rollout.py index 0463d0a5..d7b500be 100644 --- a/mreg/tests/test_policy_rollout.py +++ b/mreg/tests/test_policy_rollout.py @@ -35,12 +35,12 @@ def test_prometheus_value_handles_value_empty_and_error_responses(self) -> None: ): _prometheus_value("http://prometheus", "invalid", 2) - @patch("mreg.policy.rollout._prometheus_value", side_effect=[100, 1, 2, 0, 0, 0, 3]) + @patch("mreg.policy.rollout._prometheus_value", side_effect=[100, 1, 2]) def test_fetch_rollout_snapshot_queries_every_gate(self, prometheus_value) -> None: snapshot = fetch_rollout_snapshot("http://prometheus", window="6h", timeout=4) - self.assertEqual(snapshot, RolloutSnapshot(100, 1, 2, 0, 0, 0, 3)) - self.assertEqual(prometheus_value.call_count, 7) + self.assertEqual(snapshot, RolloutSnapshot(100, 1, 2)) + self.assertEqual(prometheus_value.call_count, 3) self.assertTrue(all(call.args[0] == "http://prometheus" for call in prometheus_value.call_args_list)) self.assertTrue(all(call.args[2] == 4 for call in prometheus_value.call_args_list)) self.assertIn("[6h]", prometheus_value.call_args_list[0].args[1]) @@ -51,10 +51,6 @@ def test_ready_snapshot_passes_every_gate(self) -> None: comparisons=20_000, mismatches=1, errors=1, - persist_failures=0, - dead_letters=0, - pending_batches=0, - backlog_age_seconds=10, ), RolloutThresholds(), ) @@ -68,14 +64,10 @@ def test_failed_snapshot_reports_every_broken_gate(self) -> None: comparisons=100, mismatches=5, errors=5, - persist_failures=2, - dead_letters=3, - pending_batches=4, - backlog_age_seconds=600, ), RolloutThresholds(), ) self.assertFalse(result.ready) - self.assertEqual(len(result.reasons), 7) + self.assertEqual(len(result.reasons), 3) self.assertIn("comparisons", result.reasons[0]) diff --git a/mreg/tests/test_treetop.py b/mreg/tests/test_treetop.py new file mode 100644 index 00000000..fe14e848 --- /dev/null +++ b/mreg/tests/test_treetop.py @@ -0,0 +1,283 @@ +"""Tests for synchronous endpoint policy stacks.""" + +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from django.test import SimpleTestCase +from rest_framework.test import APIRequestFactory + +from mreg.api.treetop import ( + PolicyAll, + PolicyAny, + PolicyCheck, + PolicyLeaf, + PolicyResource, + _SynchronousCircuitBreaker, + _build_resource_attrs, + authorize_policy_stack, + close_policy_client, + disable_policy_parity, + policy_all, + policy_any, + policy_leaf, + policy_request_scope, + policy_shadow_enabled, +) +from mreg.policy.config import PolicyMode + + +class SynchronousPolicyStackTests(SimpleTestCase): + def setUp(self) -> None: + self.factory = APIRequestFactory() + self.user = SimpleNamespace(username="alice", group_list=("users",)) + + def _request(self): + return self.factory.get( + "/api/v1/hosts/", + HTTP_X_CORRELATION_ID="test-correlation", + ) + + @staticmethod + def _leaf(name: str = "host.example.org") -> PolicyLeaf: + return policy_leaf( + action="host_read", + resource_kind="Host", + resource_id=name, + resource_attrs={"kind": "host", "name": name}, + ) + + @staticmethod + def _result(allowed: bool, index: int): + result = Mock() + result.index = index + result.id = f"mreg-{index}" + result.is_success.return_value = True + result.is_allowed.return_value = allowed + return result + + def _client(self, *decisions: bool): + client = Mock() + client.authorize.return_value = SimpleNamespace(results=[self._result(decision, index) for index, decision in enumerate(decisions)]) + return client + + def test_policy_contracts_reject_empty_values(self) -> None: + with self.assertRaisesRegex(ValueError, "kind"): + PolicyResource("", "id", {"kind": "host"}) + with self.assertRaisesRegex(ValueError, "ID"): + PolicyResource("Host", "", {"kind": "host"}) + with self.assertRaisesRegex(ValueError, "attributes"): + PolicyResource("Host", "id", {}) + with self.assertRaisesRegex(ValueError, "action"): + PolicyCheck("", PolicyResource("Host", "id", {"kind": "host"})) + with self.assertRaisesRegex(ValueError, "at least one"): + PolicyAll(()) + with self.assertRaisesRegex(ValueError, "at least one"): + PolicyAny(()) + + def test_resource_attributes_detect_bool_ip_and_string(self) -> None: + attrs = _build_resource_attrs({"restricted": "true", "ip": "192.0.2.1", "name": "host"}) + self.assertEqual(attrs["restricted"].type.value, "Bool") + self.assertEqual(attrs["ip"].type.value, "Ip") + self.assertEqual(attrs["name"].type.value, "String") + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop._get_treetop_client") + def test_nested_stack_uses_one_batched_authorize_call(self, get_client, from_request) -> None: + from_request.return_value = self.user + client = self._client(True, False, True) + get_client.return_value = client + root = policy_all( + policy_any(self._leaf("one.example.org"), self._leaf("two.example.org")), + self._leaf("three.example.org"), + ) + + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + policy_request_scope(), + ): + self.assertTrue(authorize_policy_stack(False, request=self._request(), root=root)) + + client.authorize.assert_called_once() + requests = client.authorize.call_args.args[0] + self.assertEqual(len(requests), 3) + self.assertEqual([request.id for request in requests], ["mreg-0", "mreg-1", "mreg-2"]) + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop._get_treetop_client") + def test_identical_stack_is_cached_inside_request(self, get_client, from_request) -> None: + from_request.return_value = self.user + client = self._client(True) + get_client.return_value = client + root = self._leaf() + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + policy_request_scope(), + ): + self.assertTrue(authorize_policy_stack(False, request=self._request(), root=root)) + self.assertTrue(authorize_policy_stack(False, request=self._request(), root=root)) + client.authorize.assert_called_once() + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop._get_treetop_client") + def test_second_different_stack_denies_without_second_call(self, get_client, from_request) -> None: + from_request.return_value = self.user + client = self._client(True) + get_client.return_value = client + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + policy_request_scope(), + ): + self.assertTrue(authorize_policy_stack(False, request=self._request(), root=self._leaf("one"))) + self.assertFalse(authorize_policy_stack(True, request=self._request(), root=self._leaf("two"))) + client.authorize.assert_called_once() + + @patch("mreg.api.treetop._get_treetop_client") + def test_off_and_unconfigured_shadow_do_not_call_treetop(self, get_client) -> None: + for mode in (PolicyMode.OFF, PolicyMode.SHADOW): + with ( + patch("mreg.api.treetop.POLICY_MODE", mode), + patch("mreg.api.treetop.POLICY_BASE_URL", ""), + ): + self.assertTrue(authorize_policy_stack(True, request=self._request(), root=self._leaf())) + get_client.assert_not_called() + + def test_shadow_enabled_reflects_configuration_and_disable_scope(self) -> None: + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.SHADOW), + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + ): + self.assertTrue(policy_shadow_enabled()) + with disable_policy_parity(): + self.assertFalse(policy_shadow_enabled()) + + with patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE): + self.assertFalse(policy_shadow_enabled()) + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop._get_treetop_client") + def test_shadow_calls_synchronously_but_returns_legacy(self, get_client, from_request) -> None: + from_request.return_value = self.user + get_client.return_value = self._client(True) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.SHADOW), + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + policy_request_scope(), + ): + self.assertFalse(authorize_policy_stack(False, request=self._request(), root=self._leaf())) + get_client.return_value.authorize.assert_called_once() + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop._get_treetop_client") + def test_enforce_returns_allow_and_deny(self, get_client, from_request) -> None: + from_request.return_value = self.user + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + ): + get_client.return_value = self._client(True) + self.assertTrue(authorize_policy_stack(False, request=self._request(), root=self._leaf())) + get_client.return_value = self._client(False) + self.assertFalse(authorize_policy_stack(True, request=self._request(), root=self._leaf())) + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop._get_treetop_client") + def test_results_are_composed_by_response_index(self, get_client, from_request) -> None: + from_request.return_value = self.user + client = self._client(True, False) + client.authorize.return_value.results.reverse() + get_client.return_value = client + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + ): + self.assertFalse( + authorize_policy_stack( + True, + request=self._request(), + root=policy_all(self._leaf("one"), self._leaf("two")), + ) + ) + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop._get_treetop_client") + def test_invalid_result_is_a_circuit_failure(self, get_client, from_request) -> None: + from_request.return_value = self.user + client = self._client(True) + client.authorize.return_value.results[0].id = "wrong" + get_client.return_value = client + circuit = _SynchronousCircuitBreaker(2, 30) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch("mreg.api.treetop._circuit", circuit), + ): + self.assertFalse(authorize_policy_stack(True, request=self._request(), root=self._leaf())) + self.assertEqual(circuit._failures, 1) + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop._get_treetop_client") + def test_enforce_errors_deny_and_shadow_errors_use_legacy(self, get_client, from_request) -> None: + from_request.return_value = self.user + get_client.return_value.authorize.side_effect = RuntimeError("offline") + with ( + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + patch("mreg.api.treetop._circuit", _SynchronousCircuitBreaker(5, 30)), + ): + with patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE): + self.assertFalse(authorize_policy_stack(True, request=self._request(), root=self._leaf())) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.SHADOW), + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + ): + self.assertTrue(authorize_policy_stack(True, request=self._request(), root=self._leaf())) + + @patch("mreg.api.treetop._get_treetop_client") + def test_disable_helper_only_disables_shadow(self, get_client) -> None: + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.SHADOW), + patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + disable_policy_parity(), + ): + self.assertTrue(authorize_policy_stack(True, request=self._request(), root=self._leaf())) + get_client.assert_not_called() + + @patch("mreg.api.treetop.MregUser.from_request") + @patch("mreg.api.treetop._get_treetop_client") + def test_disable_helper_cannot_bypass_enforcement(self, get_client, from_request) -> None: + from_request.return_value = self.user + get_client.return_value = self._client(False) + with ( + patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), + patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), + disable_policy_parity(), + ): + self.assertFalse(authorize_policy_stack(True, request=self._request(), root=self._leaf())) + get_client.return_value.authorize.assert_called_once() + + def test_circuit_opens_and_allows_one_half_open_probe(self) -> None: + circuit = _SynchronousCircuitBreaker(2, 30) + self.assertTrue(circuit.allow_call()) + circuit.failure() + self.assertTrue(circuit.allow_call()) + circuit.failure() + self.assertFalse(circuit.allow_call()) + + circuit._open_until = 0.1 + with patch("mreg.api.treetop.monotonic", return_value=1.0): + self.assertTrue(circuit.allow_call()) + self.assertFalse(circuit.allow_call()) + circuit.success() + self.assertTrue(circuit.allow_call()) + + @patch("mreg.api.treetop._client_lock") + def test_close_policy_client_is_safe_without_client(self, client_lock) -> None: + client_lock.__enter__ = Mock() + client_lock.__exit__ = Mock(return_value=False) + with patch("mreg.api.treetop._client", None): + close_policy_client() diff --git a/mreg/tests/test_treetop_batching.py b/mreg/tests/test_treetop_batching.py deleted file mode 100644 index 2e65bf66..00000000 --- a/mreg/tests/test_treetop_batching.py +++ /dev/null @@ -1,808 +0,0 @@ -from __future__ import annotations - -from copy import deepcopy -import logging -from types import SimpleNamespace -from unittest.mock import Mock, patch - -from django.http import HttpRequest, HttpResponse -from django.test import TestCase - -from mreg.api.treetop import ( - EnforcementFailureMode, - PolicyCheck, - PolicyMode, - PolicyResource, - _CircuitBreaker, - _ParityBatchItem, - _ParityDispatcher, - _build_policy_request, - _build_resource_attrs, - _compute_parity_payload, - _deserialize_policy_batch, - _fully_qualified_action, - _get_treetop_client, - _is_enforcement_enabled, - _is_parity_enabled, - _is_policy_enabled, - _process_policy_parity_batch, - _qualified_resource_kind, - _request_state, - _result_to_decision_and_error, - _safe_log, - _serialize_policy_batch, - _close_treetop_client, - batch_policy_parity, - disable_policy_parity, - flush_policy_parity_batch, - policy_parity, - start_policy_parity_dispatcher, - stop_policy_parity_dispatcher, -) -from mreg.middleware.logging_http import LoggingMiddleware - - -class _DummyAuthorizeResult: - def __init__( - self, - allowed: bool = False, - *, - status: str = "success", - error: str | None = None, - ) -> None: - self._allowed = allowed - self.status = status - self.error = error - - def is_success(self) -> bool: - return self.status == "success" - - def is_allowed(self) -> bool: - return self._allowed - - -class _DummyAuthorizeResponse: - def __init__(self, decisions: list[bool]) -> None: - self.results = [_DummyAuthorizeResult(decision) for decision in decisions] - - -class TreeTopParityBatchingTests(TestCase): - @staticmethod - def _request() -> HttpRequest: - request = HttpRequest() - request.method = "GET" - request.path = "/api/v1/hosts/" - request.META["HTTP_X_CORRELATION_ID"] = "test-correlation-id" - request.user = SimpleNamespace(is_authenticated=True) - return request - - @staticmethod - def _middleware_request() -> HttpRequest: - request = TreeTopParityBatchingTests._request() - request.path_info = request.path - request._body = b"" - request.user = SimpleNamespace(username="tester") - return request - - @staticmethod - def _check(hostname: str = "host.example.org") -> PolicyCheck: - return PolicyCheck( - action="host_read", - resource=PolicyResource( - kind="Host", - id=hostname, - attrs={"kind": "host", "hostname": hostname}, - ), - ) - - def _run_parity_check( - self, - request: HttpRequest, - *, - decision: bool, - hostname: str, - ) -> bool: - return policy_parity( - decision, - request=request, - check=self._check(hostname), - ) - - def test_policy_contract_rejects_empty_values(self) -> None: - with self.assertRaisesRegex(ValueError, "kind"): - PolicyResource(kind="", id="id", attrs={"kind": "host"}) - with self.assertRaisesRegex(ValueError, "ID"): - PolicyResource(kind="Host", id="", attrs={"kind": "host"}) - with self.assertRaisesRegex(ValueError, "attributes"): - PolicyResource(kind="Host", id="id", attrs={}) - with self.assertRaisesRegex(ValueError, "action"): - PolicyCheck(action="", resource=self._check().resource) - - def test_is_parity_enabled_requires_configuration_and_context(self) -> None: - with patch("mreg.api.treetop.POLICY_PARITY_ENABLED", False): - self.assertFalse(_is_parity_enabled()) - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", ""), - ): - self.assertFalse(_is_parity_enabled()) - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - disable_policy_parity(), - ): - self.assertFalse(_is_parity_enabled()) - - def test_disable_policy_parity_supports_nesting(self) -> None: - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - ): - self.assertTrue(_is_parity_enabled()) - with disable_policy_parity(): - with disable_policy_parity(): - self.assertFalse(_is_parity_enabled()) - self.assertFalse(_is_parity_enabled()) - self.assertTrue(_is_parity_enabled()) - - def test_policy_modes_distinguish_shadow_and_enforcement(self) -> None: - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.OFF), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - ): - self.assertFalse(_is_parity_enabled()) - self.assertFalse(_is_enforcement_enabled()) - self.assertFalse(_is_policy_enabled()) - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - disable_policy_parity(), - ): - self.assertFalse(_is_parity_enabled()) - self.assertTrue(_is_enforcement_enabled()) - self.assertTrue(_is_policy_enabled()) - - def test_off_mode_does_not_build_submit_or_authorize(self) -> None: - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.OFF), - patch("mreg.api.treetop.MregUser.from_request") as from_request, - patch("mreg.api.treetop._submit_policy_batch") as submit, - patch("mreg.api.treetop._get_treetop_client") as get_client, - ): - decision = self._run_parity_check( - self._request(), - decision=True, - hostname="host.example", - ) - - self.assertTrue(decision) - from_request.assert_not_called() - submit.assert_not_called() - get_client.assert_not_called() - - def test_build_resource_attrs_detects_ip_values(self) -> None: - attrs = _build_resource_attrs({"ip": "192.0.2.1", "name": "host"}) - self.assertEqual(attrs["ip"].type.value, "Ip") - self.assertEqual(attrs["name"].type.value, "String") - - def test_build_policy_request_uses_namespaced_resource_and_groups(self) -> None: - muser = SimpleNamespace(username="tester", group_list=["admins"]) - with patch("mreg.api.treetop.POLICY_NAMESPACE", ["UiO", "MREG"]): - payload = _build_policy_request(muser, self._check()).to_api() - - self.assertEqual(payload["resource"]["kind"], "UiO::MREG::Host") - self.assertEqual(payload["action"]["namespace"], ["UiO", "MREG"]) - self.assertEqual( - payload["principal"]["User"]["groups"][0], - {"id": "admins", "namespace": ["UiO", "MREG"]}, - ) - - def test_qualified_names_without_namespace(self) -> None: - action = SimpleNamespace(__str__=lambda _self: "host_read") - self.assertEqual(_fully_qualified_action(action), str(action)) - with patch("mreg.api.treetop.POLICY_NAMESPACE", []): - self.assertEqual(_qualified_resource_kind("Host"), "Host") - - def test_result_parsing_covers_missing_failed_and_success(self) -> None: - allowed, error = _result_to_decision_and_error([], 0) - self.assertIsNone(allowed) - self.assertEqual(error, "Missing policy result at index 0") - - failed = _DummyAuthorizeResult(status="failed") - allowed, error = _result_to_decision_and_error([failed], 0) # type: ignore[arg-type] - self.assertIsNone(allowed) - self.assertEqual(error, "Authorization failed with status=failed") - - failed_with_error = _DummyAuthorizeResult(status="failed", error="bad request") - allowed, error = _result_to_decision_and_error([failed_with_error], 0) # type: ignore[arg-type] - self.assertIsNone(allowed) - self.assertEqual(error, "bad request") - - allowed, error = _result_to_decision_and_error([_DummyAuthorizeResult(True)], 0) # type: ignore[arg-type] - self.assertTrue(allowed) - self.assertIsNone(error) - - def test_compute_payload_distinguishes_match_mismatch_and_error(self) -> None: - matching = _compute_parity_payload( - decision=True, - policy_allowed=True, - error=None, - context={}, - ) - mismatch = _compute_parity_payload( - decision=False, - policy_allowed=True, - error=None, - context={}, - ) - unavailable = _compute_parity_payload( - decision=True, - policy_allowed=None, - error="offline", - context={}, - ) - self.assertTrue(matching["parity"]) - self.assertFalse(mismatch["parity"]) - self.assertFalse(unavailable["parity"]) - - @patch("mreg.api.treetop.logger") - def test_safe_log_preserves_structured_context(self, logger: Mock) -> None: - _safe_log(logging.WARNING, "policy_event", result="mismatch") - - logger.log.assert_called_once_with( - logging.WARNING, - "policy_event", - result="mismatch", - ) - - @patch("mreg.api.treetop.MregUser.from_request") - def test_request_batch_is_submitted_after_response_without_authorize_io( - self, - mock_from_request: Mock, - ) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - submitted: list[list[_ParityBatchItem]] = [] - - def capture(items): # type: ignore[no-untyped-def] - submitted.append(list(items)) - return True - - request = self._request() - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), - patch("mreg.api.treetop._submit_policy_batch", side_effect=capture), - patch("mreg.api.treetop._get_treetop_client") as get_client, - batch_policy_parity(), - ): - self.assertTrue(self._run_parity_check(request, decision=True, hostname="one.example")) - self.assertFalse(self._run_parity_check(request, decision=False, hostname="two.example")) - self.assertEqual(submitted, []) - - get_client.assert_not_called() - self.assertEqual(len(submitted), 1) - self.assertEqual(len(submitted[0]), 2) - self.assertIsNone(_request_state.get()) - - @patch("mreg.api.treetop.MregUser.from_request") - def test_batching_disabled_submits_each_check(self, mock_from_request: Mock) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - submissions: list[int] = [] - - def capture(items): # type: ignore[no-untyped-def] - submissions.append(len(items)) - return True - - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", False), - patch("mreg.api.treetop._submit_policy_batch", side_effect=capture), - batch_policy_parity(), - ): - self._run_parity_check(self._request(), decision=True, hostname="one.example") - self._run_parity_check(self._request(), decision=True, hostname="two.example") - - self.assertEqual(submissions, [1, 1]) - - @patch("mreg.api.treetop.MregUser.from_request") - def test_flush_submits_and_clears_active_batch(self, mock_from_request: Mock) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), - patch("mreg.api.treetop._submit_policy_batch", return_value=True) as submit, - batch_policy_parity(), - ): - self._run_parity_check(self._request(), decision=True, hostname="one.example") - self.assertTrue(flush_policy_parity_batch()) - self.assertFalse(flush_policy_parity_batch()) - - submit.assert_called_once() - - @patch("mreg.api.treetop.MregUser.from_request", side_effect=RuntimeError("broken user")) - @patch("mreg.api.treetop._record_instrumentation_failure") - def test_policy_parity_is_fail_open_for_build_errors( - self, - record_failure: Mock, - _from_request: Mock, - ) -> None: - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - ): - self.assertTrue(policy_parity(True, request=self._request(), check=self._check())) - record_failure.assert_called_once() - - @patch("mreg.api.treetop.MregUser.from_request") - def test_enforcement_is_synchronous_and_does_not_use_outbox( - self, - mock_from_request: Mock, - ) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - client = Mock() - client.authorize.return_value = _DummyAuthorizeResponse([True]) - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop._get_treetop_client", return_value=client), - patch("mreg.api.treetop._submit_policy_batch") as submit, - batch_policy_parity(), - ): - enforced = self._run_parity_check( - self._request(), - decision=False, - hostname="host.example", - ) - - self.assertTrue(enforced) - client.authorize.assert_called_once() - submit.assert_not_called() - - @patch("mreg.api.treetop.MregUser.from_request") - def test_enforcement_policy_deny_overrides_legacy_allow( - self, - mock_from_request: Mock, - ) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - client = Mock() - client.authorize.return_value = _DummyAuthorizeResponse([False]) - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop._get_treetop_client", return_value=client), - ): - enforced = self._run_parity_check( - self._request(), - decision=True, - hostname="host.example", - ) - - self.assertFalse(enforced) - - @patch("mreg.api.treetop.MregUser.from_request") - def test_enforcement_errors_deny_by_default( - self, - mock_from_request: Mock, - ) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - client = Mock() - client.authorize.side_effect = RuntimeError("offline") - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch( - "mreg.api.treetop.POLICY_ENFORCEMENT_FAILURE_MODE", - EnforcementFailureMode.DENY, - ), - patch("mreg.api.treetop._get_treetop_client", return_value=client), - patch("mreg.api.treetop._submit_policy_batch") as submit, - ): - enforced = self._run_parity_check( - self._request(), - decision=True, - hostname="host.example", - ) - - self.assertFalse(enforced) - submit.assert_not_called() - - @patch("mreg.api.treetop.MregUser.from_request") - def test_enforcement_can_use_explicit_legacy_failure_fallback( - self, - mock_from_request: Mock, - ) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - client = Mock() - client.authorize.side_effect = RuntimeError("offline") - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch( - "mreg.api.treetop.POLICY_ENFORCEMENT_FAILURE_MODE", - EnforcementFailureMode.LEGACY, - ), - patch("mreg.api.treetop._get_treetop_client", return_value=client), - ): - enforced = self._run_parity_check( - self._request(), - decision=True, - hostname="host.example", - ) - - self.assertTrue(enforced) - - @patch("mreg.api.treetop.MregUser.from_request") - def test_disable_shadow_helper_cannot_bypass_enforcement( - self, - mock_from_request: Mock, - ) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - client = Mock() - client.authorize.return_value = _DummyAuthorizeResponse([False]) - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop._get_treetop_client", return_value=client), - disable_policy_parity(), - ): - enforced = self._run_parity_check( - self._request(), - decision=True, - hostname="host.example", - ) - - self.assertFalse(enforced) - - @patch("mreg.api.treetop.MregUser.from_request", side_effect=RuntimeError("bad principal")) - def test_enforcement_build_errors_fail_closed(self, _from_request: Mock) -> None: - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch( - "mreg.api.treetop.POLICY_ENFORCEMENT_FAILURE_MODE", - EnforcementFailureMode.DENY, - ), - ): - enforced = self._run_parity_check( - self._request(), - decision=True, - hostname="host.example", - ) - - self.assertFalse(enforced) - - @patch("mreg.api.treetop.MregUser.from_request") - def test_enforcement_missing_result_fails_closed( - self, - mock_from_request: Mock, - ) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - client = Mock() - client.authorize.return_value = _DummyAuthorizeResponse([]) - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop._get_treetop_client", return_value=client), - ): - enforced = self._run_parity_check( - self._request(), - decision=True, - hostname="host.example", - ) - - self.assertFalse(enforced) - - @patch("mreg.api.treetop.MregUser.from_request") - def test_enforcement_without_runtime_url_fails_closed( - self, - mock_from_request: Mock, - ) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", ""), - patch("mreg.api.treetop._get_treetop_client") as get_client, - ): - enforced = self._run_parity_check( - self._request(), - decision=True, - hostname="host.example", - ) - - self.assertFalse(enforced) - get_client.assert_not_called() - - @patch("mreg.api.treetop.MregUser.from_request") - def test_sensitive_log_details_are_disabled_by_default(self, mock_from_request: Mock) -> None: - mock_from_request.return_value = SimpleNamespace( - username="tester", - group_list=["secret-group"], - ) - captured: list[_ParityBatchItem] = [] - - def capture(items): # type: ignore[no-untyped-def] - captured.extend(items) - return True - - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop.POLICY_PARITY_LOG_DETAILS", False), - patch("mreg.api.treetop._submit_policy_batch", side_effect=capture), - ): - self._run_parity_check(self._request(), decision=True, hostname="secret.example") - - self.assertNotIn("principal", captured[0].context) - self.assertNotIn("groups", captured[0].context) - self.assertNotIn("resource_attrs", captured[0].context) - - @patch("mreg.api.treetop.MregUser.from_request") - def test_sensitive_log_details_can_be_enabled(self, mock_from_request: Mock) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=["admins"]) - captured: list[_ParityBatchItem] = [] - - def capture(items): # type: ignore[no-untyped-def] - captured.extend(items) - return True - - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop.POLICY_PARITY_LOG_DETAILS", True), - patch("mreg.api.treetop._submit_policy_batch", side_effect=capture), - ): - self._run_parity_check(self._request(), decision=True, hostname="host.example") - - self.assertEqual(captured[0].context["principal"], "tester") - self.assertEqual(captured[0].context["groups"], ["admins"]) - - @patch("mreg.api.treetop._log_parity_payload") - @patch("mreg.api.treetop._get_treetop_client") - def test_worker_processes_a_batch_in_one_authorize_call( - self, - get_client: Mock, - log_payload: Mock, - ) -> None: - client = get_client.return_value - client.authorize.return_value = _DummyAuthorizeResponse([True, False]) - items = [ - _ParityBatchItem(True, {"request": "one"}, {"correlation_id": "cid", "path": "/one"}), - _ParityBatchItem(False, {"request": "two"}, {"correlation_id": "cid", "path": "/one"}), - ] - - _process_policy_parity_batch(items) - - client.authorize.assert_called_once_with( - [{"request": "one"}, {"request": "two"}], - correlation_id="cid", - ) - self.assertEqual(log_payload.call_count, 2) - self.assertTrue(log_payload.call_args_list[0].args[0]["parity"]) - self.assertTrue(log_payload.call_args_list[1].args[0]["parity"]) - - @patch("mreg.api.treetop._log_parity_payload") - @patch("mreg.api.treetop._get_treetop_client") - def test_worker_records_authorize_exceptions_for_every_item( - self, - get_client: Mock, - log_payload: Mock, - ) -> None: - get_client.return_value.authorize.side_effect = RuntimeError("offline") - items = [ - _ParityBatchItem(True, {"request": "one"}, {}), - _ParityBatchItem(False, {"request": "two"}, {}), - ] - - with self.assertRaisesRegex(RuntimeError, "offline"): - _process_policy_parity_batch(items) - - log_payload.assert_not_called() - - def test_dispatcher_persists_batches_in_the_shared_outbox(self) -> None: - dispatcher = _ParityDispatcher() - request = _build_policy_request( - SimpleNamespace(username="tester", group_list=[]), - self._check(), - ) - item = _ParityBatchItem(True, request, {"path": "/one"}) - with patch.object(dispatcher, "_ensure_started"): - self.assertTrue(dispatcher.submit([item])) - - from mreg.models.policy import PolicyParityOutbox - - row = PolicyParityOutbox.objects.get() - self.assertEqual(row.payload["version"], 1) - self.assertEqual(row.payload["items"][0]["context"]["path"], "/one") - - def test_durable_batch_round_trip_preserves_typed_requests(self) -> None: - request = _build_policy_request( - SimpleNamespace(username="tester", group_list=["admins"]), - self._check(), - ) - items = [_ParityBatchItem(False, request, {"correlation_id": "cid"})] - - restored = _deserialize_policy_batch(_serialize_policy_batch(items)) - - self.assertEqual(len(restored), 1) - self.assertFalse(restored[0].decision) - self.assertEqual(restored[0].context, {"correlation_id": "cid"}) - self.assertEqual(restored[0].policy_request.to_api(), request.to_api()) - - def test_durable_batch_rejects_malformed_payloads(self) -> None: - request = _build_policy_request( - SimpleNamespace(username="tester", group_list=[]), - self._check(), - ).to_api() - - def batch(policy_request): # type: ignore[no-untyped-def] - return { - "version": 1, - "items": [{"decision": True, "policy_request": policy_request, "context": {}}], - } - - invalid_payloads: list[dict[str, object]] = [ - {"version": 2, "items": []}, - {"version": 1, "items": {}}, - {"version": 1, "items": ["invalid"]}, - {"version": 1, "items": [{"policy_request": request, "context": []}]}, - batch({"principal": []}), - batch({"principal": {"User": []}}), - ] - - invalid_action = deepcopy(request) - invalid_action["action"] = [] - invalid_payloads.append(batch(invalid_action)) - invalid_attrs = deepcopy(request) - invalid_attrs["resource"]["attrs"] = [] - invalid_payloads.append(batch(invalid_attrs)) - invalid_attribute = deepcopy(request) - invalid_attribute["resource"]["attrs"]["kind"] = [] - invalid_payloads.append(batch(invalid_attribute)) - - for payload in invalid_payloads: - with self.subTest(payload=payload), self.assertRaises((ValueError, KeyError)): - _deserialize_policy_batch(payload) - - def test_treetop_client_is_process_local_and_closed_safely(self) -> None: - old_client = Mock() - new_client = Mock() - with ( - patch("mreg.api.treetop._client", old_client), - patch("mreg.api.treetop._client_pid", -1), - patch("mreg.api.treetop.os.getpid", return_value=42), - patch("mreg.api.treetop.TreeTopClient", return_value=new_client) as client_class, - ): - self.assertIs(_get_treetop_client(), new_client) - - old_client.close.assert_called_once_with() - client_class.assert_called_once() - - with ( - patch("mreg.api.treetop._client", new_client), - patch("mreg.api.treetop._client_pid", 42), - patch("mreg.api.treetop.asyncio.run") as async_run, - ): - _close_treetop_client() - async_run.assert_called_once_with(new_client.aclose.return_value) - - fallback_client = Mock() - with ( - patch("mreg.api.treetop._client", fallback_client), - patch("mreg.api.treetop._client_pid", 42), - patch("mreg.api.treetop.asyncio.run", side_effect=RuntimeError("no event loop")), - ): - _close_treetop_client() - fallback_client.close.assert_called_once_with() - - def test_dispatcher_lifecycle_respects_configuration_and_process(self) -> None: - dispatcher = Mock() - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop._get_dispatcher", return_value=dispatcher), - ): - start_policy_parity_dispatcher() - dispatcher._ensure_started.assert_called_once_with() - - with ( - patch("mreg.api.treetop._dispatcher", dispatcher), - patch("mreg.api.treetop._dispatcher_pid", 42), - patch("mreg.api.treetop.os.getpid", return_value=42), - ): - stop_policy_parity_dispatcher() - dispatcher.shutdown.assert_called_once_with() - - def test_dispatcher_does_not_start_in_enforcement_mode(self) -> None: - with ( - patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop._get_dispatcher") as get_dispatcher, - ): - start_policy_parity_dispatcher() - - get_dispatcher.assert_not_called() - - def test_dispatcher_claims_and_completes_a_durable_batch(self) -> None: - from mreg.models.policy import PolicyParityOutbox - - request = _build_policy_request( - SimpleNamespace(username="tester", group_list=[]), - self._check(), - ) - row = PolicyParityOutbox.objects.create( - payload=_serialize_policy_batch([_ParityBatchItem(True, request, {})]), - ) - dispatcher = _ParityDispatcher() - - claimed = dispatcher._claim() - - self.assertIsNotNone(claimed) - assert claimed is not None - self.assertEqual(claimed.id, row.id) - self.assertEqual(claimed.attempts, 1) - dispatcher._complete(claimed) - self.assertFalse(PolicyParityOutbox.objects.filter(id=row.id).exists()) - - def test_dispatcher_retries_then_retains_a_dead_letter(self) -> None: - from mreg.models.policy import PolicyParityOutbox - - request = _build_policy_request( - SimpleNamespace(username="tester", group_list=[]), - self._check(), - ) - item = _ParityBatchItem(True, request, {}) - row = PolicyParityOutbox.objects.create(payload=_serialize_policy_batch([item])) - dispatcher = _ParityDispatcher() - claimed = dispatcher._claim() - assert claimed is not None - - with patch("mreg.api.treetop.POLICY_PARITY_MAX_ATTEMPTS", 2): - dispatcher._fail(claimed, RuntimeError("offline"), [item]) - row.refresh_from_db() - self.assertIsNone(row.failed_at) - self.assertIsNone(row.locked_at) - - row.available_at = row.created_at - row.save(update_fields=("available_at",)) - claimed = dispatcher._claim() - assert claimed is not None - dispatcher._fail(claimed, RuntimeError("still offline"), [item]) - - row.refresh_from_db() - self.assertIsNotNone(row.failed_at) - self.assertIn("still offline", row.last_error) - - def test_circuit_breaker_opens_and_recovers(self) -> None: - circuit = _CircuitBreaker(failure_threshold=2, reset_seconds=30) - circuit.failure() - self.assertEqual(circuit.wait_seconds(), 0) - circuit.failure() - self.assertGreater(circuit.wait_seconds(), 0) - circuit.success() - self.assertEqual(circuit.wait_seconds(), 0) - - @patch("mreg.api.treetop.MregUser.from_request") - def test_logging_middleware_only_enqueues_policy_work(self, mock_from_request: Mock) -> None: - mock_from_request.return_value = SimpleNamespace(username="tester", group_list=[]) - - def get_response(request: HttpRequest) -> HttpResponse: - self._run_parity_check(request, decision=True, hostname="one.example") - self._run_parity_check(request, decision=True, hostname="two.example") - return HttpResponse(status=200) - - middleware = LoggingMiddleware(get_response) - with ( - patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), - patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - patch("mreg.api.treetop.POLICY_PARITY_BATCH_ENABLED", True), - patch("mreg.api.treetop._submit_policy_batch", return_value=True) as submit, - patch("mreg.api.treetop._get_treetop_client") as get_client, - ): - response = middleware(self._middleware_request()) - - self.assertEqual(response.status_code, 200) - get_client.assert_not_called() - self.assertEqual(len(submit.call_args.args[0]), 2) diff --git a/mregsite/gunicorn_conf.py b/mregsite/gunicorn_conf.py index 58b20868..0bb59696 100644 --- a/mregsite/gunicorn_conf.py +++ b/mregsite/gunicorn_conf.py @@ -3,36 +3,16 @@ import os -def _setup_django() -> None: - """Initialize Django before Gunicorn loads worker-scoped integrations.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mregsite.settings") - - import django - from django.apps import apps - - if not apps.ready: - django.setup() - - -def post_fork(server, worker): # noqa: ARG001 - """Start the shadow-mode outbox consumer only after worker fork.""" - _setup_django() - - from mreg.api.treetop import start_policy_parity_dispatcher - - start_policy_parity_dispatcher() - - def worker_exit(server, worker): # noqa: ARG001 - """Stop the worker thread; unprocessed rows remain durable in PostgreSQL.""" + """Close worker-local clients and mark its multiprocess metrics dead.""" from django.apps import apps if not apps.ready: return - from mreg.api.treetop import stop_policy_parity_dispatcher + from mreg.api.treetop import close_policy_client - stop_policy_parity_dispatcher() + close_policy_client() if os.environ.get("PROMETHEUS_MULTIPROC_DIR"): from prometheus_client import multiprocess diff --git a/mregsite/settings.py b/mregsite/settings.py index b5d09854..de375510 100644 --- a/mregsite/settings.py +++ b/mregsite/settings.py @@ -23,7 +23,6 @@ import mreg.__about__ from mreg.policy.config import ( PolicyMode, - resolve_enforcement_failure_mode, resolve_policy_mode, validate_policy_configuration, ) @@ -98,14 +97,10 @@ def parse_protected_attrs(raw: str) -> list[dict[str, str]]: _raw_policy_mode, legacy_parity_enabled=_legacy_policy_parity_enabled, ) - _policy_enforcement_failure_mode = resolve_enforcement_failure_mode( - envvar("MREG_POLICY_ENFORCEMENT_FAILURE_MODE", "deny") - ) validate_policy_configuration(_policy_mode, POLICY_BASE_URL) except ValueError as exc: raise ImproperlyConfigured(str(exc)) from exc POLICY_MODE = _policy_mode.value -POLICY_ENFORCEMENT_FAILURE_MODE = _policy_enforcement_failure_mode.value # Compatibility for local settings and integrations that still inspect the old # boolean. Explicit MREG_POLICY_MODE takes precedence over the deprecated flag. POLICY_PARITY_ENABLED = _policy_mode == PolicyMode.SHADOW @@ -113,23 +108,13 @@ def parse_protected_attrs(raw: str) -> list[dict[str, str]]: # Accept both Cedar-style `org::MREG` and comma-separated `org,MREG`. raw = raw.replace("::", ",") POLICY_NAMESPACE = [ns.strip() for ns in raw.split(",") if ns.strip()] or ["MREG"] -POLICY_PARITY_BATCH_ENABLED = envvar("MREG_POLICY_PARITY_BATCH_ENABLED", True) POLICY_PARITY_LOG_DETAILS = envvar("MREG_POLICY_PARITY_LOG_DETAILS", False) POLICY_TIMEOUT_SECONDS = envvar("MREG_POLICY_TIMEOUT_SECONDS", 5.0) -POLICY_PARITY_MAX_ATTEMPTS = envvar("MREG_POLICY_PARITY_MAX_ATTEMPTS", 8) -POLICY_PARITY_RETRY_BASE_SECONDS = envvar("MREG_POLICY_PARITY_RETRY_BASE_SECONDS", 2.0) -POLICY_PARITY_RETRY_MAX_SECONDS = envvar("MREG_POLICY_PARITY_RETRY_MAX_SECONDS", 300.0) -POLICY_PARITY_LEASE_SECONDS = envvar("MREG_POLICY_PARITY_LEASE_SECONDS", 60.0) -POLICY_PARITY_POLL_SECONDS = envvar("MREG_POLICY_PARITY_POLL_SECONDS", 1.0) -POLICY_PARITY_CIRCUIT_FAILURES = envvar("MREG_POLICY_PARITY_CIRCUIT_FAILURES", 5) -POLICY_PARITY_CIRCUIT_RESET_SECONDS = envvar("MREG_POLICY_PARITY_CIRCUIT_RESET_SECONDS", 30.0) +POLICY_CIRCUIT_FAILURES = envvar("MREG_POLICY_CIRCUIT_FAILURES", 5) +POLICY_CIRCUIT_RESET_SECONDS = envvar("MREG_POLICY_CIRCUIT_RESET_SECONDS", 30.0) POLICY_ROLLOUT_MIN_COMPARISONS = envvar("MREG_POLICY_ROLLOUT_MIN_COMPARISONS", 10_000) POLICY_ROLLOUT_MAX_MISMATCH_RATE = envvar("MREG_POLICY_ROLLOUT_MAX_MISMATCH_RATE", 0.001) POLICY_ROLLOUT_MAX_ERROR_RATE = envvar("MREG_POLICY_ROLLOUT_MAX_ERROR_RATE", 0.001) -POLICY_ROLLOUT_MAX_PERSIST_FAILURES = envvar("MREG_POLICY_ROLLOUT_MAX_PERSIST_FAILURES", 0) -POLICY_ROLLOUT_MAX_DEAD_LETTERS = envvar("MREG_POLICY_ROLLOUT_MAX_DEAD_LETTERS", 0) -POLICY_ROLLOUT_MAX_PENDING_BATCHES = envvar("MREG_POLICY_ROLLOUT_MAX_PENDING_BATCHES", 0) -POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS = envvar("MREG_POLICY_ROLLOUT_MAX_BACKLOG_AGE_SECONDS", 300.0) REQUESTS_THRESHOLD_SLOW = envvar("MREG_REQUESTS_THRESHOLD_SLOW", 1000) REQUESTS_LOG_LEVEL_SLOW = envvar("MREG_REQUESTS_LOG_LEVEL_SLOW", "WARNING") @@ -543,14 +528,10 @@ def parse_protected_attrs(raw: str) -> list[dict[str, str]]: _post_local_policy_mode, legacy_parity_enabled=POLICY_PARITY_ENABLED, ) - _policy_enforcement_failure_mode = resolve_enforcement_failure_mode( - POLICY_ENFORCEMENT_FAILURE_MODE - ) validate_policy_configuration(_policy_mode, POLICY_BASE_URL) except ValueError as exc: raise ImproperlyConfigured(str(exc)) from exc POLICY_MODE = _policy_mode.value -POLICY_ENFORCEMENT_FAILURE_MODE = _policy_enforcement_failure_mode.value POLICY_PARITY_ENABLED = _policy_mode == PolicyMode.SHADOW if TESTING or "CI" in os.environ: diff --git a/treetop/data/labels.json b/treetop/data/labels.json index 031957aa..d479858b 100644 --- a/treetop/data/labels.json +++ b/treetop/data/labels.json @@ -1,25 +1,93 @@ [ - { - "kind": "MREG::Host", - "field": "name", - "output": "nameLabels", - "patterns": [ - { - "name": "in_domain", - "regex": "example\\.com$" - }, - { - "name": "valid_webserver_name", - "regex": "^web-\\d+" - }, - { - "name": "admin_subdomain", - "regex": "^admin\\." - }, - { - "name": "staging_environment", - "regex": "^staging\\." - } - ] - } + { + "kind": "MREG::Host", + "field": "hostname", + "output": "nameLabels", + "patterns": [ + {"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}, + {"name": "netgroup_host_example_org", "regex": "^ho.*\\.example\\.org$"}, + {"name": "safelabel", "regex": ".*\\.example\\.org$"}, + {"name": "webserver", "regex": "^web-\\d+"}, + {"name": "admin_subdomain", "regex": "^admin\\."}, + {"name": "staging_environment", "regex": "^staging\\."} + ] + }, + { + "kind": "MREG::Ipaddress", + "field": "hostname", + "output": "nameLabels", + "patterns": [ + {"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}, + {"name": "netgroup_host_example_org", "regex": "^ho.*\\.example\\.org$"} + ] + }, + { + "kind": "MREG::Cname", + "field": "hostname", + "output": "nameLabels", + "patterns": [ + {"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}, + {"name": "netgroup_host_example_org", "regex": "^ho.*\\.example\\.org$"} + ] + }, + { + "kind": "MREG::Hinfo", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + }, + { + "kind": "MREG::Loc", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + }, + { + "kind": "MREG::Mx", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + }, + { + "kind": "MREG::Naptr", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + }, + { + "kind": "MREG::NameServer", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + }, + { + "kind": "MREG::PtrOverride", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + }, + { + "kind": "MREG::Sshfp", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + }, + { + "kind": "MREG::Srv", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + }, + { + "kind": "MREG::Txt", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + }, + { + "kind": "MREG::BACnetID", + "field": "hostname", + "output": "nameLabels", + "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + } ] diff --git a/treetop/data/mreg-bundle.tar.gz b/treetop/data/mreg-bundle.tar.gz index 02e941077691d9bf39373ad5da8683c664a64c15..437855c84c3b3bda07dea9d867bf0bda7a094ae6 100644 GIT binary patch literal 5102 zcmY+IWmFW7(#98L1r~5gmj(%8Nu@*uVUg~X?rvC+rMtTu=@yV~kdjV;mF`?pQu2EL z=f3ZK-}y9i&Yb7WZ|1|C^Uy?K0se`{L)}~Z1@VT<@ds9obt8fm6tlZ#jVf`IWqig{ z3u=dW-PHy?L(ym(zI9`%p&0l3&f~N-Z9=i|c-_Xc*TQV^=4Q`!JkPSY!Ov{BHMpq* zD^}2vZ(URK<@$40QhfF_9u{cKIyX>RE}en`7Bm9g)mS3i*7JUO{aIso6X~z%anDaV z5iR}QGG1RjjuyGtsOxa;xb%f)17v0e$tJS&5In0{Yfx@8@s(8aCq1cI(ZToBC`4=f z6r5`)P=^e^L*N6(<8m4==;4iI!+X#GUk0T)8XX3qn3$*4m?AOa(kG< zc|Ob1WB!9JSiwB=Ovz_JT-=@?g{#rJ@|H|ap=Enw_w`UE_j}7)!zv+bh2oE6nBplH zHNW58>oumZI*b*0Qhz0O<>HCDJTaNh7!_yoT7DBUa&YE?n8--KCkm){TR1>G6*uQO z_&j2Wuy8PbwM*J2Bq(?oC2r}rWM}T|LT@&>o^C@z2PG?Q{9cCK`=`KTS?w^&LWP z*_^$+Jc}isQ=slY6)#w+_)qKzIJZj`U@loU&Q~NX;`Ll43@=4{`+QxfY)(DH!cRVI z?!dql4DYdd@#o=SR9o;Jj)2)Fht%TYRBbrMZ=(qIZkeON5jL#yRj%L?l~T z(QNrjsKTEF4V`5sCvfW>LH6_|`^WFl^3C11>a3j_U{qAX8&ABmitx>Bt9->vlX9c! z4$*dHW4PB(c&bl1PJtIGU1f;9mU;D}T7cV2yx3U!=6J8M*!_#UO2I1YOvH z5JAc73uKG=uWn_rUuaKlsw?V!V#*lEE$e6~yLEq6*^fS>INytrkyen%WR>HP!KWTV zJv3KSres@GWI9_#*Fw&tHkRk{BBhAhqFrb>!BndCedUbJ_%RslA`tGk+!a8XvGM5L z`SY!2*rP%VRqtbrLz`=EpW*5m^FD<&dj5(vog}q2dI|y0Gb!c8$tGj>%B{fkRXb#( zxA}{{T=qjeZVujY#77Qtg_qAKs)U`WD*xQ+<5=mE&orGLdc>_kSZ*RvmYJc}Fj>+G z&Z8A8r!lkitpQc3*E-x-H{?zulHhgpaXe;hH~gV@8Y4ZkYio~M+qHO=Tf4F83ARD= zwI8FNSzSdNy4iTMDU?D)WAg4#0@w@%yrJX$#$N3i@s=efh&774ligd-J!t<7%Ord= zjo(|7#eE1jj|W~w6lbbaB9M$hfAp158oN#;)}0L5o%fLV=u0X1v3=?V}k`IJuVPD2zK^1-&sJ16F0Q)UyErKgE6t zWEs@jsvr|9@rmkD@l>9}UsbXA1=;q|LMM6T99&pwluf8|1?SBI&SLIJ1&enX>|7|M)Zw*>7TYt*r8{5MomuqK zJht0doW?|1v)PC$RB1u+EI2W}EI84Jglht_L4^h|TpEKDcUEO-Trs4A?& zWyQ)ZxkxHd-pQ>VpYJE&E$;G;yt=%tLt|>Zozm4-!AV9l#H!1jWo8Ax>t-<2zVOYu zIcap5QM%opK*tKCh?5fai{lFnvdH87Kg%{$e$>)VD0HqT(Uw-%9&h5-((csgXwa*^ zYq&AdMcMKWE8CSDC)YGLObQD-W#ylb^2SU(YFczYA4D2u)}OiF9;JV}SXhUaYblTn z&N`{b*;KrX60QByTo8Cxqj#o|C^;))8T9-m2SxO^M8V(UUeY0&>z2TyPMi;*El>cv zwO!@5!SC-U>oljV zZ0uKPYdy&uhwe_i+8fdol&XTZRi-yO?ijj0X$=xqo7|i>OVJ0Z^Dr=PoE+!JeK(nhdzWL+h%w#wwJN0RK+|Z5bZUR+e5m}{OGd8b|%*%riZYnn5lOhHk5_m&^{yJ~8Izfn1DAa$aCW`A zxxPM4#DTXqMlN4tha;I}{X2pkC?XLtZzTH7m&spB6Q_@vAh`d^Tz8mqU5G>W~KahkFoDVin?Tjr^rDlrsQ;ZV(M(=DGfNq8+)gJ;tjru@Fb zu5kv%$kt{ObO5ZoxybUcJOm%5X9`6Ya)DdY=1S=DcHuKb$deE$nq)JE5~sZgf`TL*^%sWc~`==M}ZkzeM8*vr`vqkgkMTBHrHDkcBc_! zKv5$sdYS4atKBS{`a@L9a1hdOwx~Ut+GKL~2Q2@_-F`XT%3~DS+euX`+Rr(3fL9=-ei>6`L#uSp8ttPAp=8{ZdOB#52vsMYahjx95K5!|rv z&;$jNR_AmIwbISqQrsK|>~0Qs@;y>Uwb7umMIMb87jkOH^T2EmH|pn)DHL(HF`W8J zC09rF0|IG@GHRjYD-t+8qHad1`o!@^tfY@Oi3hQo{61-IM(XYx$Llk#O>K>*!GYDA z8P{8~Z{>tj9y#ZEP>;Iidk3FepkQ60h|8t0Lby2)s-%+_W)0!U42u#dBt+>YPmCWQ z#~|&UpJFfv`X-5Sf9r^!jbYL@^kS^wY@GE!Osew|aK%Bfn%s9EFtAl+Zmk`v)e8u; zr@Mrts89SJmRbIHE7-oO=3?!1re3hE!+$oV!<+6+qyLs#jpyc$p&O9B(-2DHn?s5} z%ZY`7OYD#KrUKtvSg;k+(H{b%=|eLxqj7uk+&2gVhx8f7a}bRW2B9woE(to01CI0) zK!>p9Sr1wXrf+u#*S9u!a>%&)%x{NFt9dFtJ=kz-Mn?@UGXNC&1Q0|D!ome$U}K1a zhD36&PZel|H8E&86sgd#FsJK;O7!J=0!~SI@5IjKIO;r-Mq_5H>i!TMKy5rDub8d8 z6~YxrO#zQ!{9mEy0Cb9K0IhGRur2=e`q^M8764ro1Aw*+M#I1-C;_3@c`=SgUse{a zaenGt#l~e0^E)Wdp?xCe7mJJShX+8P!-)+DpnYmU7lKX3flefCoOj@K0r1nJZE6a> zn5pj|f?{Kp;Gl7ZmY5KcDw01r#}daPW{&j##h@XYQ+HAAE^LZ2-dEIJFea9Sp)eB+ z)*O9a$byDhP5v~OJneNK(xP_!QkEe% z_UV|eTy0MytEEem3oz>0c;%P5Ad;7G-DEJNs_A|WHPaxJ*)cR5a-}}`po%=X{i{l? zuQuF+EVo3s{yp1xZN=i=Wv|R}&&88xZv3>=07|%na4g_efr0R54g+Oi@UJZ?Hw;=u zRpezT=cBg6%~_UBl<8PP5WoB%F)$ zW(XEa%@Jg?*hD#_tme0@5&8<-H5>*A(*yb79eK$MLhTcda?u+q9WG8d+90}v5 zy<6U0RcT7>EG0^)rLsgd`_bC)PbR;~k8L!j6gVqi-WB(YDa^1SqusLt}Of{z@^bgtXvLV&!P^awWw;R2NMQy)X zx**KN#2^qb$X}$fXo4pYGs<5amY<_EVey1~6wK=@2`ivg#)pPFzo8qA>hxEJmFB1_ zvM|@96~3$uqE(7zmLz5X5!#C6!ArZYk3MKhqwxVV{N=ie41PYfVM(d*gLUN?D86DQ zt$Ptrwq|bDgH8ub>%-bAO5$baIkD>#kKgKc8Up$GK&8Lq6p#S^%I8W&?@3v(4%|#JbA$I z@fYgKULAjV_s6gBy?KB{SN^Kf3A1Y*K}mcz3(jEI05u;P&v)1YIg!FolX=bGNNIfoR3r0ca1B8dyId*CaRWq{Qu8t z;zZnxtUFnarAf$BhCTdsJ{C?jJQ+eOnHP?HlQG$qs6c~OWjJ^ zmg_sD{^;W%4%g3yOsQ|F>6N2wOq?9{O3Oes5f>P)z}i^SR%`TmNHfE!ZQ8C5LwSwF zWp12Mh?14ybouqV{hb^ydWqy!v5SNxWUsABOY6szOgf zkXTQ$!bg&He|N(0P{&ZF+<|}p?B!!UA%DJ|UC8QpzbE}4VjclP!YpV27IeVB0OEbZ A7XSbN literal 3477 zcmV;G4QlcqiwFP!00000|Lt8_bK5u)_OpKlhP5?i@5(YsiMpnyYO|TCY-*ECD#y-4 zl2Jh`r$)@pO@o1od4~-9vi=NZapRzP-|DHzS$#H@S zq1Ww-UkK~J9jjNq%b#0j*RoH{lSc@PG@}V2C*NqeH)xXiQ|e}^G5T zu4^%CI++d}&zwxDXAXH<{(qSdx4l3OlUpV|I7QOwaVZiOVdfH*qH& zI$@sK8w{<gy&>ysWOUn>-8LaYmN1q@@mS`A zXQFMko!SdGN{Yv}Wh`69Jn|L+ljh*h<0uiP0YwSvoJ+oZ^Yb%9-lZ{hYmbE0+(B7+ z1o5z)FzStI5Wrh;PrbPxj@@M8tvpG`GH&tkhD{ou=MVjO?Rf?iHoi)t#rlhoWj9gs zc`WrRB}CH2lU2Dz3Yc$yEzK#{(~#*#b;ZM3lx8w-8;^5LSDt&2&l^ART$*^BZ=j+f zr@UJ8cTjanBP zZmH7NP8RWLd3u83PZL%<`jexChyA~f+3EB*{6EJ=|L-xv*r~k$NCpzG0A3rN&-L4h!2tOLYLh5*-U7^q_KZl)7g&iMkB%c zW2pp`|K0b_)>!ZSRK&^1*(re#Cw}PqF%4ScHqV|PiaQ=Y@aUt_6Y*j+GB}}+d1^+h zlp#$rZWLzRdsYd9a?b-8l!7kr1(mA(a2lzEg2+{g%x_f^A&s*{Ed*JHegxc7l_*aw z0=dNZl%-75*)&!PCD$sE>{eNq6Y4_$2hN7FdcevTp`YDJ!B0}MumW)`gs9Ol7!>*xNzMV*z_msmVP!T}7*Uyz6=`KgR-~0Vk$kJnf)rEgmSSq9z~5?>Af#3Z zLP~YKFHoxELP~X9NU5&sSCmRCQKjliR85a%f2F3#QdCWorD&dk9s0p?1i5=xDB~TY z_uIv_+pC9M1Au@-E2aMlLIMtOc7H7f)5N7AVVYJN;3dnf6P|1;Jzz>Wl@KwF zK!mI5hkkfP3>cP6elc2vh8}p(j68qMb>tlU z-L2Ko#xQ^PN-CrNKh)Fd&5VVo_p359nu>aOA!c*P_up^6_T%-(?J7n)uetQBR&VDU z5&XFAr_aMTyr$0l_{>HcVGjB0kjCaW>5`N0+k4* zq~z)bY=umch{D7c7)K$~;1Yyjs7QoI3h5<`00m5a7|j6y z1_)<~?`F(1AQv#W+i3pPyxh?k090JvjO8G6&cf_L7js-fv2%=Kj%-)?0>OPE ziEl&vVWE8`YIP^Cis9BJc~y>#U^KF_0w!E*KE!5X%syjDIgXMO_(ZI)6B?wENGf42 z^RF3CQuxqs*cH2dU8YFD-B0*>Y|TG^U{k>Us|tByxVV6xJ9_TuxufTfo;!N(wSb;G zdhTm#&~yL#J@?fW6PQ;oR!hK9=2y8}yv^%w+90dn_x|z6eU=&R#mt5E8@DKYS}XmwUBSB-TBLBa@@-wGt6Fx|qQ9_3?LYv7!V)Wst67>C zK}Q9Y2R@Qq&dtx9)NpRvAilq0i5=H_j^-qJk3|L^=!_)`E5)vTcB2&H3*NcV))g(W zVp`hoD}LN1kmK%jeMM2{p1Mub%=fR0B{ZX9cf-cKKuXR;!WVNkpMdSpz!a;g0^-h? z`}uqu!8{6^tWG36Tr|EMUMwSiN$J)PO6*@g5qpo)t!dn)e6K}=%bV<4=2wn$?4SE8 zi9L%@*Q+SQPp{sRU-M%sn}ujN#Ik=eAd!nAk=@F$sdk4_ye|*pG?!!Jpw5?-q)KM@ zvaGA#UYdrz^zuYZv70_WA5-q9e~c?cU@*=qQG5%Fj*Z{g1?KH(~U=j?61=h1kbCBNiTk8t;v&j#|}*Cbo%KeiJR%s>+KAW-S#lLB=fXR#H zvb8x)hmh)nG@m6xNcBM~_S_<%`T)%*J`vJGAzhhvMmY7ri43Y0PGnDwU?Owsf^&VP z79rIKDKe;bNRdS~1By(l3)1?!8NzueoGa^#2-t`AG>iZr6xaGU^v)d6hWaL56XJrvn(pEN{dH6XkCT}gyg z1EispL0mQC+Vs^?1Xd5Q4PR$PXb*>W?K@8hsXj{ERimCA&G3M3CnQv`|1d)PEe6Qj{qtXsdh+_QMG|u`!^W~rye+wMYY0-{HYO4 zWKLah=KGj%JL6Gc#V$!Rpx7yC1~ksMGZ!LgvPVZ4ZTQ#-qpx*be9`?nHp16j2D$i= za7IRc9GsDt9|L7%=i}gvYKiJg!pFt2@fgZ~^#3z&c=3o}rb!*XBcY;?aY zN-i(Ue2UuwK6%4GjVv!EM)H#53Q49j8cVw`^b1Whc%3J2$Y0|vycZ=-*nbv&!hm|F zG)NgVpgW#~@w3S6FA2{~6dL>+SKfi~<63-QRQg`=Kl|Z0-!1*_jn|!886@b{A@JJ{iJA-#fvPL$-5oVMPw`qdDE^~G#q=; zocf`l!7B#DWw2X1j|2AcW8019e*xkwrLSq=d*d56$b>IsqKSpqeiDVEo2*62UyDeemYR diff --git a/treetop/data/mreg.cedar b/treetop/data/mreg.cedar index 876d7837..f795e755 100644 --- a/treetop/data/mreg.cedar +++ b/treetop/data/mreg.cedar @@ -1,30 +1,75 @@ -// MREG permissions example. +// MREG endpoint authorization. Each protected endpoint sends one request stack; +// TreeTop evaluates every leaf in one authorize call and MREG composes the result. -// Common CRUD action sets for resource-managed APIs. -// Keep this list in sync with mreg/api/permissions.py::_crud_action. +@id("MREG.authenticated_access") +permit ( + principal, + action == MREG::Action::"authenticated_access", + resource +); @id("MREG.read_all") permit ( principal, action in - [MREG::Action::"host_read", - MREG::Action::"host_contacts_read", - MREG::Action::"ipaddress_read", + [MREG::Action::"bacnet_id_read", MREG::Action::"cname_read", + MREG::Action::"community_read", + MREG::Action::"forward_zone_delegation_read", + MREG::Action::"forward_zone_read", MREG::Action::"hinfo_read", + MREG::Action::"host_community_mapping_read", + MREG::Action::"host_contacts_read", + MREG::Action::"host_group_read", + MREG::Action::"host_policy_atom_read", + MREG::Action::"host_policy_role_read", + MREG::Action::"host_read", + MREG::Action::"ipaddress_read", + MREG::Action::"label_read", MREG::Action::"loc_read", MREG::Action::"mx_read", - MREG::Action::"naptr_read", MREG::Action::"name_server_read", + MREG::Action::"naptr_read", + MREG::Action::"net_group_regex_permission_read", + MREG::Action::"network_excluded_range_read", + MREG::Action::"network_policy_attribute_read", + MREG::Action::"network_policy_attribute_value_read", + MREG::Action::"network_policy_read", + MREG::Action::"network_read", MREG::Action::"ptr_override_read", - MREG::Action::"sshfp_read", + MREG::Action::"reverse_zone_delegation_read", + MREG::Action::"reverse_zone_read", MREG::Action::"srv_read", - MREG::Action::"txt_read", - MREG::Action::"bacnet_id_read", - MREG::Action::"community_read"], + MREG::Action::"sshfp_read", + MREG::Action::"txt_read"], resource ); +@id("MREG.user_info_self") +permit ( + principal, + action == MREG::Action::"user_info_read", + resource is MREG::Generic +) +when { + resource has selfAccess && resource.selfAccess +}; + +@id("MREG.user_info_admin") +permit ( + principal in MREG::Group::"default-admin-group", + action == MREG::Action::"user_info_read", + resource is MREG::Generic +); + +@id("MREG.user_info_hostgroup_admin") +permit ( + principal in MREG::Group::"default-groupadmin-group", + action == MREG::Action::"user_info_read", + resource is MREG::Generic +); + +// Resources managed by the ordinary MREG administrator role. @id("MREG.admin_crud") permit ( principal in MREG::Group::"default-admin-group", @@ -32,6 +77,8 @@ permit ( [MREG::Action::"host_create", MREG::Action::"host_update", MREG::Action::"host_delete", + MREG::Action::"host_contacts_create", + MREG::Action::"host_contacts_delete", MREG::Action::"ipaddress_create", MREG::Action::"ipaddress_update", MREG::Action::"ipaddress_delete", @@ -70,70 +117,108 @@ permit ( MREG::Action::"bacnet_id_delete", MREG::Action::"community_create", MREG::Action::"community_update", - MREG::Action::"community_delete"], + MREG::Action::"community_delete", + MREG::Action::"host_community_mapping_create", + MREG::Action::"host_community_mapping_update", + MREG::Action::"host_community_mapping_delete", + MREG::Action::"label_create", + MREG::Action::"label_update", + MREG::Action::"label_delete", + MREG::Action::"net_group_regex_permission_create", + MREG::Action::"net_group_regex_permission_update", + MREG::Action::"net_group_regex_permission_delete"], resource ); -@id("MREG.admins_policy") +@id("MREG.network_admin_crud") permit ( - principal in MREG::Group::"admins", + principal in MREG::Group::"default-networkadmin-group", action in - [MREG::Action::"host_create", - MREG::Action::"host_read", - MREG::Action::"host_update", + [MREG::Action::"network_create", + MREG::Action::"network_update", + MREG::Action::"network_delete", + MREG::Action::"network_excluded_range_create", + MREG::Action::"network_excluded_range_update", + MREG::Action::"network_excluded_range_delete", + MREG::Action::"network_policy_create", + MREG::Action::"network_policy_update", + MREG::Action::"network_policy_delete", + MREG::Action::"network_policy_attribute_create", + MREG::Action::"network_policy_attribute_update", + MREG::Action::"network_policy_attribute_delete", + MREG::Action::"network_policy_attribute_value_create", + MREG::Action::"network_policy_attribute_value_update", + MREG::Action::"network_policy_attribute_value_delete", + MREG::Action::"community_create", + MREG::Action::"community_update", + MREG::Action::"community_delete", + MREG::Action::"host_create", MREG::Action::"host_delete"], - resource is MREG::Host + resource ); -// Webadmins can edit, delete, or create hosts with a name label containing "webserver", and the IP -// address must be in the range 192.168.1.0/24 -@id("MREG.webadmins_policy") +@id("MREG.hostgroup_admin_crud") permit ( - principal in MREG::Group::"webadmins", + principal in MREG::Group::"default-groupadmin-group", action in - [MREG::Action::"host_create", - MREG::Action::"host_read", - MREG::Action::"host_update", - MREG::Action::"host_delete"], - resource is MREG::Host + [MREG::Action::"host_group_create", + MREG::Action::"host_group_update", + MREG::Action::"host_group_delete", + MREG::Action::"hostgroup_membership_update"], + resource is MREG::HostGroup +); + +@id("MREG.hostgroup_owner_update") +permit ( + principal, + action == MREG::Action::"host_group_update", + resource is MREG::HostGroup ) -when -{ - resource has nameLabels && - resource has ip && - resource.nameLabels.contains("webserver") && - resource.ip.isInRange(ip("192.168.1.0/24")) +when { + resource has requesterIsOwner && resource.requesterIsOwner && + resource has descriptionUpdate && resource.descriptionUpdate +}; + +@id("MREG.hostgroup_owner_membership") +permit ( + principal, + action == MREG::Action::"hostgroup_membership_update", + resource is MREG::HostGroup +) +when { + resource has requesterIsOwner && resource.requesterIsOwner && + resource has ownerMutation && !resource.ownerMutation }; -// Admins can manipulate any IP address, even if it is a gw, a broadcast address, -// the network address, reserved. These three groups are unified as "restricted" IPs. -@id("MREG.admins_ip_policy") +@id("MREG.hostpolicy_admin_crud") permit ( - principal in MREG::Group::"admins", + principal in MREG::Group::"default-hostpolicyadmin-group", action in - [MREG::Action::"ip_gw_management", - MREG::Action::"ip_broadcast_management", - MREG::Action::"ip_network_management", - MREG::Action::"ip_reserved_management", - MREG::Action::"ip_restricted_management"], - resource is MREG::Ipaddress + [MREG::Action::"host_policy_atom_create", + MREG::Action::"host_policy_atom_update", + MREG::Action::"host_policy_atom_delete", + MREG::Action::"host_policy_role_create", + MREG::Action::"host_policy_role_update", + MREG::Action::"host_policy_role_delete", + MREG::Action::"hostpolicy_role_atom_membership_update", + MREG::Action::"hostpolicy_role_host_membership_update"], + resource ); - -/// Test group access, used during testing. -@id("MREG.test_group_policy") +// Static bundle representation of the testgroup NetGroupRegexPermission rows. +// The regex portion is a TreeTop-derived label; MREG sends only hostname/IP facts. +@id("MREG.testgroup_netgroup") permit ( principal in MREG::Group::"testgroup", action in [MREG::Action::"host_create", MREG::Action::"host_update", MREG::Action::"host_delete", + MREG::Action::"host_contacts_create", + MREG::Action::"host_contacts_delete", MREG::Action::"ipaddress_create", MREG::Action::"ipaddress_update", MREG::Action::"ipaddress_delete", - MREG::Action::"cname_create", - MREG::Action::"cname_update", - MREG::Action::"cname_delete", MREG::Action::"hinfo_create", MREG::Action::"hinfo_update", MREG::Action::"hinfo_delete", @@ -163,45 +248,38 @@ permit ( MREG::Action::"txt_delete", MREG::Action::"bacnet_id_create", MREG::Action::"bacnet_id_update", - MREG::Action::"bacnet_id_delete", - MREG::Action::"community_create", - MREG::Action::"community_update", - MREG::Action::"community_delete"], + MREG::Action::"bacnet_id_delete"], resource -) when { - resource has hostname && - ( - ( - action in - [MREG::Action::"cname_create", - MREG::Action::"cname_update", - MREG::Action::"cname_delete"] && - !(resource has ip) && - resource.hostname like "ho*.example.org" - ) || - ( - !(action in - [MREG::Action::"cname_create", - MREG::Action::"cname_update", - MREG::Action::"cname_delete"]) && - resource.hostname like "*.example.org" && - resource has ip && - ( - resource.ip.isInRange(ip("10.0.0.0/24")) || - resource.ip.isInRange(ip("10.1.0.0/25")) || - resource.ip.isInRange(ip("192.168.1.0/24")) || - resource.ip.isInRange(ip("192.168.2.1/32")) || - resource.ip.isInRange(ip("2001:db8::/64")) || - resource.ip.isInRange(ip("2001:db8::1/128")) || - resource.ip.isInRange(ip("2002:db9::/64")) - ) - ) - ) +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_example_org") && + resource has ip && + (resource.ip.isInRange(ip("10.0.0.0/24")) || + resource.ip.isInRange(ip("10.1.0.0/25")) || + resource.ip.isInRange(ip("192.168.1.0/24")) || + resource.ip.isInRange(ip("192.168.2.1/32")) || + resource.ip.isInRange(ip("2001:db8::/64")) || + resource.ip.isInRange(ip("2002:db9::/64"))) }; -/// Network admin permissions used in tests where network-admin users also -/// receive NetGroupRegexPermission entries. -@id("MREG.network_admin_group_policy") +@id("MREG.testgroup_cname_netgroup") +permit ( + principal in MREG::Group::"testgroup", + action in + [MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete"], + resource is MREG::Cname +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_host_example_org") +}; + +// Network administrators need an explicit NetGroup mapping for host/DNS CRUD, +// just like the legacy database permission path. +@id("MREG.network_admin_netgroup") permit ( principal in MREG::Group::"default-networkadmin-group", action in @@ -211,9 +289,6 @@ permit ( MREG::Action::"ipaddress_create", MREG::Action::"ipaddress_update", MREG::Action::"ipaddress_delete", - MREG::Action::"cname_create", - MREG::Action::"cname_update", - MREG::Action::"cname_delete", MREG::Action::"hinfo_create", MREG::Action::"hinfo_update", MREG::Action::"hinfo_delete", @@ -243,120 +318,214 @@ permit ( MREG::Action::"txt_delete", MREG::Action::"bacnet_id_create", MREG::Action::"bacnet_id_update", - MREG::Action::"bacnet_id_delete", - MREG::Action::"community_create", - MREG::Action::"community_update", - MREG::Action::"community_delete"], + MREG::Action::"bacnet_id_delete"], resource -) when { - resource has hostname && - ( - ( - action in - [MREG::Action::"cname_create", - MREG::Action::"cname_update", - MREG::Action::"cname_delete"] && - !(resource has ip) && - resource.hostname like "ho*.example.org" - ) || - ( - !(action in - [MREG::Action::"cname_create", - MREG::Action::"cname_update", - MREG::Action::"cname_delete"]) && - resource.hostname like "*.example.org" && - resource has ip && - ( - resource.ip.isInRange(ip("10.0.0.0/24")) || - resource.ip.isInRange(ip("10.1.0.0/25")) || - resource.ip.isInRange(ip("192.168.1.0/24")) || - resource.ip.isInRange(ip("192.168.2.1/32")) || - resource.ip.isInRange(ip("2001:db8::/64")) || - resource.ip.isInRange(ip("2001:db8::1/128")) || - resource.ip.isInRange(ip("2002:db9::/64")) - ) - ) - ) +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_example_org") && + resource has ip && + (resource.ip.isInRange(ip("10.0.0.0/24")) || + resource.ip.isInRange(ip("10.1.0.0/25")) || + resource.ip.isInRange(ip("192.168.1.0/24")) || + resource.ip.isInRange(ip("192.168.2.1/32")) || + resource.ip.isInRange(ip("2001:db8::/64")) || + resource.ip.isInRange(ip("2002:db9::/64"))) }; -/// Network Admins can manage any IP in any network -@id("MREG.network_admins_ip_network_policy") +@id("MREG.network_admin_cname_netgroup") permit ( principal in MREG::Group::"default-networkadmin-group", - action == MREG::Action::"ip_network_management", - resource is MREG::Ipaddress -); + action in + [MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete"], + resource is MREG::Cname +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_host_example_org") +}; -/// Users can only manage IPs in specific networks -@id("MREG.users_ip_network_policy") +@id("MREG.testgroup_community_network") permit ( - principal in MREG::Group::"users", - action == MREG::Action::"ip_network_management", - resource is MREG::Ipaddress + principal in MREG::Group::"testgroup", + action in + [MREG::Action::"community_create", + MREG::Action::"community_update", + MREG::Action::"community_delete", + MREG::Action::"host_create", + MREG::Action::"host_delete"], + resource ) -when -{ - resource has ip && - ( - resource.ip.isInRange(ip("192.168.1.0/24")) || - resource.ip.isInRange(ip("10.0.0.0/8")) - ) +when { + resource has network && + (resource.network == "10.0.0.0/24" || + resource.network == "10.1.0.0/25" || + resource.network == "192.168.1.0/24") +}; + +@id("MREG.dummygroup_hostpolicy_role_host") +permit ( + principal in MREG::Group::"dummygroup", + action == MREG::Action::"hostpolicy_role_host_membership_update", + resource is MREG::Host +) +when { + resource has nameLabels && + resource has roleLabel && + resource.nameLabels.contains(resource.roleLabel) && + resource has ip && resource.ip.isInRange(ip("11.22.33.0/24")) }; -/// Admins can do whatever with labels. -@id("MREG.labels_admin_policy") +@id("MREG.webadmins") permit ( - principal in MREG::Group::"default-super-group", + principal in MREG::Group::"webadmins", action in - [MREG::Action::"create_label", - MREG::Action::"delete_label", - MREG::Action::"view_label", - MREG::Action::"edit_label"], - resource is MREG::Label -); + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"host_delete"], + resource is MREG::Host +) +when { + resource has nameLabels && + resource has ip && + resource.nameLabels.contains("webserver") && + resource.ip.isInRange(ip("192.168.1.0/24")) +}; + +// These rules replace local post-policy denials in authoritative mode. +@id("MREG.invalid_or_unprivileged_dns_wildcard") +forbid ( + principal, + action in + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"hinfo_create", + MREG::Action::"hinfo_update", + MREG::Action::"loc_create", + MREG::Action::"loc_update", + MREG::Action::"mx_create", + MREG::Action::"mx_update", + MREG::Action::"naptr_create", + MREG::Action::"naptr_update", + MREG::Action::"name_server_create", + MREG::Action::"name_server_update", + MREG::Action::"ptr_override_create", + MREG::Action::"ptr_override_update", + MREG::Action::"sshfp_create", + MREG::Action::"sshfp_update", + MREG::Action::"srv_create", + MREG::Action::"srv_update", + MREG::Action::"txt_create", + MREG::Action::"txt_update"], + resource +) +when { + resource has dnsWildcard && resource.dnsWildcard && + principal != MREG::User::"super" && + !(principal in MREG::Group::"default-super-group") && + (resource has dnsWildcardValidDepth && !resource.dnsWildcardValidDepth || + !(principal in MREG::Group::"default-dns-wildcard-group")) +}; -/// Normal admins -@id("MREG.admin") +@id("MREG.unprivileged_dns_underscore") +forbid ( + principal, + action in + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"hinfo_create", + MREG::Action::"hinfo_update", + MREG::Action::"loc_create", + MREG::Action::"loc_update", + MREG::Action::"mx_create", + MREG::Action::"mx_update", + MREG::Action::"naptr_create", + MREG::Action::"naptr_update", + MREG::Action::"name_server_create", + MREG::Action::"name_server_update", + MREG::Action::"ptr_override_create", + MREG::Action::"ptr_override_update", + MREG::Action::"sshfp_create", + MREG::Action::"sshfp_update", + MREG::Action::"txt_create", + MREG::Action::"txt_update"], + resource +) +when { + resource has dnsUnderscore && resource.dnsUnderscore && + principal != MREG::User::"super" && + !(principal in MREG::Group::"default-super-group") && + !(principal in MREG::Group::"default-dns-underscore-group") +}; + +@id("MREG.unprivileged_restricted_ip") +forbid ( + principal, + action in + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"ipaddress_create", + MREG::Action::"ipaddress_update", + MREG::Action::"ptr_override_create", + MREG::Action::"ptr_override_update"], + resource +) +when { + resource has ipRestricted && resource.ipRestricted && + principal != MREG::User::"super" && + !(principal in MREG::Group::"default-super-group") && + !(principal in MREG::Group::"default-networkadmin-group") +}; + +// Explicit membership actions retained for custom/admin endpoints. +@id("MREG.admin_membership") permit ( principal in MREG::Group::"default-admin-group", action == MREG::Action::"admin_access", resource ); -/// Network admins (group-membership permission check) -@id("MREG.network_admin") +@id("MREG.network_admin_membership") permit ( principal in MREG::Group::"default-networkadmin-group", - action == MREG::Action::"network_admin_access", + action in + [MREG::Action::"network_admin_access", + MREG::Action::"ip_network_management", + MREG::Action::"ip_reserved_management", + MREG::Action::"ip_restricted_management", + MREG::Action::"ip_gw_management", + MREG::Action::"ip_broadcast_management"], resource ); -/// Host group admins (group-membership permission check) -@id("MREG.hostgroup_admin") +@id("MREG.hostgroup_admin_membership") permit ( principal in MREG::Group::"default-groupadmin-group", action == MREG::Action::"hostgroup_admin_access", resource ); -/// Host Policy Admins -@id("MREG.hostpolicy_admin") +@id("MREG.hostpolicy_admin_membership") permit ( principal in MREG::Group::"default-hostpolicyadmin-group", action == MREG::Action::"hostpolicy_admin_access", resource ); -/// DNS Wildcard Admins -@id("MREG.dns_wildcard_admin") +@id("MREG.dns_wildcard_admin_membership") permit ( principal in MREG::Group::"default-dns-wildcard-group", action == MREG::Action::"dns_wildcard_admin_access", resource ); -/// DNS Underscore Admins -@id("MREG.dns_underscore_admin") +@id("MREG.dns_underscore_admin_membership") permit ( principal in MREG::Group::"default-dns-underscore-group", action == MREG::Action::"dns_underscore_admin_access", diff --git a/treetop/data/mreg.cedarschema b/treetop/data/mreg.cedarschema index 09d6b9ad..b6db6422 100644 --- a/treetop/data/mreg.cedarschema +++ b/treetop/data/mreg.cedarschema @@ -2,7 +2,26 @@ namespace MREG { entity Group; entity User in [Group]; - entity Generic; + entity Generic = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; entity Host = { kind?: String, id?: String, @@ -11,34 +30,602 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity HostContact = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, }; - entity HostContact; entity Ipaddress = { kind?: String, id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Cname = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Hinfo = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Loc = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Mx = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Naptr = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity NameServer = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity PtrOverride = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Sshfp = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Srv = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Txt = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity BACnetID = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Community = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity HostCommunityMapping = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Label = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity Network = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity NetworkPolicy = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity NetworkPolicyAttribute = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity NetworkPolicyAttributeValue = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity HostGroup = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity NetworkExcludedRange = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity ForwardZone = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity ForwardZoneDelegation = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity ReverseZone = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity ReverseZoneDelegation = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity HostPolicyAtom = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity HostPolicyRole = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, + }; + entity NetGroupRegexPermission = { + kind?: String, + id?: String, + name?: String, + path?: String, + hostname?: String, + ip?: ipaddr, + nameLabels?: Set, + dnsWildcard?: Bool, + dnsWildcardValidDepth?: Bool, + dnsUnderscore?: Bool, + ipReserved?: Bool, + ipRestricted?: Bool, + selfAccess?: Bool, + requesterIsOwner?: Bool, + ownerMutation?: Bool, + descriptionUpdate?: Bool, + roleLabel?: String, + network?: String, }; - entity Cname; - entity Hinfo; - entity Loc; - entity Mx; - entity Naptr; - entity NameServer; - entity PtrOverride; - entity Sshfp; - entity Srv; - entity Txt; - entity BACnetID; - entity Community; - entity HostCommunityMapping; - entity Label; - entity Network; - entity NetworkPolicy; - entity NetworkPolicyAttribute; - entity NetworkPolicyAttributeValue; action "admin_access", + "authenticated_access", "bacnet_id_create", "bacnet_id_delete", "bacnet_id_read", @@ -56,6 +643,14 @@ namespace MREG { "dns_underscore_admin_access", "dns_wildcard_admin_access", "edit_label", + "forward_zone_create", + "forward_zone_delegation_create", + "forward_zone_delegation_delete", + "forward_zone_delegation_read", + "forward_zone_delegation_update", + "forward_zone_delete", + "forward_zone_read", + "forward_zone_update", "hinfo_create", "hinfo_delete", "hinfo_read", @@ -64,13 +659,30 @@ namespace MREG { "host_community_mapping_delete", "host_community_mapping_read", "host_community_mapping_update", + "host_contacts_create", + "host_contacts_delete", "host_contacts_read", "host_create", "host_delete", + "host_group_create", + "host_group_delete", + "host_group_read", + "host_group_update", + "host_policy_atom_create", + "host_policy_atom_delete", + "host_policy_atom_read", + "host_policy_atom_update", + "host_policy_role_create", + "host_policy_role_delete", + "host_policy_role_read", + "host_policy_role_update", "host_read", "host_update", "hostgroup_admin_access", + "hostgroup_membership_update", "hostpolicy_admin_access", + "hostpolicy_role_atom_membership_update", + "hostpolicy_role_host_membership_update", "ip_broadcast_management", "ip_gw_management", "ip_network_management", @@ -101,9 +713,17 @@ namespace MREG { "naptr_delete", "naptr_read", "naptr_update", + "net_group_regex_permission_create", + "net_group_regex_permission_delete", + "net_group_regex_permission_read", + "net_group_regex_permission_update", "network_admin_access", "network_create", "network_delete", + "network_excluded_range_create", + "network_excluded_range_delete", + "network_excluded_range_read", + "network_excluded_range_update", "network_policy_attribute_create", "network_policy_attribute_delete", "network_policy_attribute_read", @@ -122,6 +742,14 @@ namespace MREG { "ptr_override_delete", "ptr_override_read", "ptr_override_update", + "reverse_zone_create", + "reverse_zone_delegation_create", + "reverse_zone_delegation_delete", + "reverse_zone_delegation_read", + "reverse_zone_delegation_update", + "reverse_zone_delete", + "reverse_zone_read", + "reverse_zone_update", "srv_create", "srv_delete", "srv_read", @@ -135,6 +763,7 @@ namespace MREG { "txt_delete", "txt_read", "txt_update", + "user_info_read", "view_label" appliesTo { principal: User, @@ -160,7 +789,16 @@ namespace MREG { Network, NetworkPolicy, NetworkPolicyAttribute, - NetworkPolicyAttributeValue + NetworkPolicyAttributeValue, + HostGroup, + NetworkExcludedRange, + ForwardZone, + ForwardZoneDelegation, + ReverseZone, + ReverseZoneDelegation, + HostPolicyAtom, + HostPolicyRole, + NetGroupRegexPermission ] }; } From 1454bf457a4da0db225543768689287507943dbd Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 19 Aug 2026 12:58:57 +0200 Subject: [PATCH 31/34] Complete synchronous TreeTop policy migration --- .github/workflows/container-image.yml | 10 +- .github/workflows/test.yml | 2 + Dockerfile | 41 +- README.md | 2 +- docs/env.md | 10 +- docs/metrics.md | 14 +- docs/parity_testing.md | 9 +- docs/policies.md | 72 ++- docs/testing.md | 10 +- entrypoint-test.sh | 7 +- entrypoint.sh | 31 +- hostpolicy/api/permissions.py | 23 +- monitoring/grafana/treetop-parity.json | 6 +- monitoring/treetop-alerts.yml | 12 +- mreg/api/permissions.py | 29 +- mreg/api/tests/test_metrics.py | 30 +- mreg/api/treetop.py | 85 ++-- mreg/api/v1/tests/test_logging.py | 20 - mreg/api/views.py | 12 +- mreg/middleware/logging_http.py | 12 +- mreg/middleware/metrics.py | 7 +- mreg/policy/contracts.py | 10 +- mreg/tests/prometheus_test_utils.py | 41 -- mreg/tests/test_gunicorn_conf.py | 37 -- mreg/tests/test_treetop.py | 72 ++- mreg/tests/test_treetop_policy_generator.py | 104 +++++ mregsite/gunicorn_conf.py | 20 - pyproject.toml | 3 +- scripts/build-treetop-bundle.sh | 28 ++ scripts/generate-treetop-policy.py | 448 +++++++++++++++++++ treetop/data/labels.json | 315 ++++++++++++- treetop/data/mreg-bundle.tar.gz | Bin 5102 -> 5253 bytes treetop/data/mreg.cedar | 218 ++------- treetop/data/mreg.cedarschema | 186 -------- treetop/data/netgroup-conversion-report.json | 27 ++ treetop/data/netgroup.cedar | 298 ++++++++++++ treetop/data/treetop-mreg-module.toml | 2 +- treetop/fixtures/hostpolicy-roles.txt | 3 + treetop/fixtures/network-permissions.txt | 9 + uv.lock | 6 +- 40 files changed, 1470 insertions(+), 801 deletions(-) delete mode 100644 mreg/tests/prometheus_test_utils.py delete mode 100644 mreg/tests/test_gunicorn_conf.py create mode 100644 mreg/tests/test_treetop_policy_generator.py delete mode 100644 mregsite/gunicorn_conf.py create mode 100755 scripts/build-treetop-bundle.sh create mode 100644 scripts/generate-treetop-policy.py create mode 100644 treetop/data/netgroup-conversion-report.json create mode 100644 treetop/data/netgroup.cedar create mode 100644 treetop/fixtures/hostpolicy-roles.txt create mode 100644 treetop/fixtures/network-permissions.txt diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index c0d08823..59668f5e 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -26,11 +26,9 @@ jobs: - name: Checkout uses: actions/checkout@v6 - name: Docker build - run: | - docker build --target runtime -t mreg . - docker build --target test -t mreg-test . + run: docker build -t mreg . - name: Save image - run: docker save mreg mreg-test | gzip > mreg.tgz + run: docker save mreg | gzip > mreg.tgz - name: Upload artifact uses: actions/upload-artifact@v7 with: @@ -65,9 +63,9 @@ jobs: run: docker load --input mreg.tgz - name: Run tests run: | - docker run --rm -t --network host \ + docker run --rm -t --network host --entrypoint /app/entrypoint-test.sh \ -e MREG_DB_HOST=localhost -e MREG_DB_PASSWORD=mreg -e MREG_DB_USER=mreg \ - mreg-test + mreg mreg-cli: name: Test with mreg-cli diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b8eca0ea..cbba188b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,8 @@ jobs: TREETOP_BUNDLE_BIN: ./treetop-bundle - name: Check generated Cedar contracts run: python scripts/generate-treetop-schema.py --check + - name: Check generated permission policy + run: python scripts/generate-treetop-policy.py --check test: name: Test diff --git a/Dockerfile b/Dockerfile index 8706daa2..027c6089 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# Runtime dependency build stage. +# build stage FROM python:3.12-alpine AS builder WORKDIR /app ENV PYTHONDONTWRITEBYTECODE=1 @@ -20,23 +20,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ ENTRYPOINT [ "/bin/sh" ] -# Test dependencies are isolated from the production environment. -FROM builder AS test-builder -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --locked --no-editable --group dev - -# Prepare application sources for production without relying on the newer -# Dockerfile COPY --exclude flag used only by recent BuildKit releases. -FROM builder AS runtime-builder -RUN rm -rf \ - /app/mreg/tests \ - /app/mreg/api/tests \ - /app/mreg/api/v1/tests \ - && find /app/mreg -type f -name '*.pyc' -delete \ - && find /app/mreg -depth -type d -name __pycache__ -empty -delete - -# Production runtime stage. -FROM python:3.12-alpine AS runtime +# final stage +FROM python:3.12-alpine EXPOSE 8000 WORKDIR /app @@ -51,8 +36,8 @@ ENV PATH="/app/.venv/bin:$PATH" COPY --from=builder /app/.venv /app/.venv # Copy over application files -COPY entrypoint.sh manage.py /app/ -COPY --from=runtime-builder /app/mreg /app/mreg/ +COPY entrypoint* manage.py /app/ +COPY mreg /app/mreg/ COPY mregsite /app/mregsite/ COPY hostpolicy /app/hostpolicy/ COPY --from=ghcr.io/astral-sh/uv:0.12.0 /uv /uvx /bin/ @@ -62,18 +47,4 @@ RUN apk update && apk upgrade \ && mkdir -p /app/logs \ && chmod a+x /app/entrypoint* -CMD ["/app/entrypoint.sh"] - -# Dedicated test image. Production tests and their dependencies exist only here. -FROM runtime AS test -COPY --from=test-builder /app/.venv /app/.venv -COPY --from=test-builder /app/mreg/tests /app/mreg/tests -COPY --from=test-builder /app/mreg/api/tests /app/mreg/api/tests -COPY --from=test-builder /app/mreg/api/v1/tests /app/mreg/api/v1/tests -COPY entrypoint-test.sh /app/entrypoint-test.sh -RUN chmod a+x /app/entrypoint-test.sh -ENTRYPOINT ["/app/entrypoint-test.sh"] -CMD [] - -# Keep an unqualified `docker build .` production-safe. -FROM runtime AS final +CMD /app/entrypoint.sh diff --git a/README.md b/README.md index c40e3ddc..ae7f6d99 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ mreg supports configuration via environment variables with the `MREG_` prefix. T | `MREG_REQUESTS_THRESHOLD_VERY_SLOW` | `5000` | Very slow request threshold (ms) | | `MREG_REQUESTS_LOG_LEVEL_VERY_SLOW` | `CRITICAL` | Log level for very slow requests | -### TreeTop Policy Parity +### TreeTop Authorization | Variable | Default | Description | | -------- | ------- | ----------- | diff --git a/docs/env.md b/docs/env.md index 991aceaa..b99782a4 100644 --- a/docs/env.md +++ b/docs/env.md @@ -90,13 +90,9 @@ rejection, or other TreeTop failure; there is no legacy fallback. - `MREG_POLICY_CIRCUIT_RESET_SECONDS` (`30.0`): cooldown before one half-open probe is allowed. -The client timeout remains `MREG_POLICY_TIMEOUT_SECONDS` (`5.0`). Each Gunicorn -worker owns its client and thread-safe circuit state. An open circuit returns -the legacy decision in `shadow` and denies in `enforce`. - -The container sets `PROMETHEUS_MULTIPROC_DIR` to an isolated directory so -metrics from every Gunicorn worker are aggregated. Custom Gunicorn deployments -must set this variable to a clean, writable directory before starting Python. +The client timeout remains `MREG_POLICY_TIMEOUT_SECONDS` (`5.0`). Each +application process owns its client and thread-safe circuit state. An open +circuit returns the legacy decision in `shadow` and denies in `enforce`. ## TreeTop enforcement rollout gates diff --git a/docs/metrics.md b/docs/metrics.md index e6cdab85..634fb530 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -39,7 +39,7 @@ uses monotonic clocks. | `mreg_policy_enforcement_results_total` | Counter | `result` | Authoritative `allow`, `deny`, or fail-closed `error_deny` | | `mreg_policy_mode_info` | Gauge | `mode` | Active `off`, `shadow`, or `enforce` mode | | `mreg_policy_stack_size` | Histogram | none | Cedar leaves in the endpoint stack sent by one call | -| `mreg_policy_authorize_calls_per_request` | Histogram | none | TreeTop HTTP calls per MREG request; protected requests should be `1` | +| `mreg_policy_stack_conflicts_total` | Counter | none | Attempts to evaluate two different stacks in one request; should remain `0` | | `mreg_policy_circuit_open` | Gauge | none | Whether a worker's synchronous circuit is open | All protected endpoint checks are synchronous in both active modes. `shadow` @@ -47,9 +47,10 @@ returns the legacy result after recording the comparison; `enforce` returns the TreeTop composite and fails closed. There is no queue, retry worker, persistence metric, and enforcement failures never return the legacy decision. -The two design-invariant metrics are: +The two design metrics are: -- `mreg_policy_authorize_calls_per_request`: alert if observations exceed one. +- `mreg_policy_stack_conflicts_total`: alert on any increase. Identical repeated + checks use the decision cached on the request and never make another call. - `mreg_policy_stack_size`: identify high-count endpoints whose semantic policy can be simplified even though transport is already consolidated. @@ -62,7 +63,7 @@ The two design-invariant metrics are: The default gate requires at least 10,000 endpoint comparisons, no more than 0.1% mismatches, and no more than 0.1% errors over the selected window. The alerts cover mismatch/error rates, any authoritative failure, an open circuit, -and violations of the one-call-per-request invariant. +and violations of the one-stack-per-request invariant. Useful PromQL: @@ -83,8 +84,3 @@ rate(mreg_policy_stack_size_sum[5m]) / clamp_min(rate(mreg_policy_stack_size_count[5m]), 1) ``` - -The container configures Prometheus multiprocess mode and clears its directory -before Gunicorn starts. Other process managers must provide a clean writable -`PROMETHEUS_MULTIPROC_DIR` and call -`prometheus_client.multiprocess.mark_process_dead` when a worker exits. diff --git a/docs/parity_testing.md b/docs/parity_testing.md index 6d043ef1..ccfb0ed9 100644 --- a/docs/parity_testing.md +++ b/docs/parity_testing.md @@ -96,10 +96,11 @@ TreeTop evaluates all leaves and MREG composes their results locally. `shadow` records the comparison and returns the legacy result. `enforce` returns the TreeTop result and fails closed on every integration failure. -The request scope rejects a second different stack, making accidental -checkpoint-by-checkpoint calls visible during development instead of quietly -adding request-path latency. A thread-safe circuit breaker prevents every -request from waiting for the full timeout during an outage. +Request-owned state rejects a second different stack and increments +`mreg_policy_stack_conflicts_total`, making accidental checkpoint-by-checkpoint +calls visible instead of quietly adding request-path latency. A thread-safe +circuit breaker prevents every request from waiting for the full timeout during +an outage. ## Parity Runbook diff --git a/docs/policies.md b/docs/policies.md index feafa5f6..8e7e5879 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -19,14 +19,14 @@ and admin pages are the explicit policy exemptions. Authorization must complete before request processing can continue. Queuing the work would either allow an unauthorised request to proceed or still require the request to wait for the queue result. An async HTTP client would change how the -thread waits, not remove the dependency. The DRF/Gunicorn application is +thread waits, not remove the dependency. The Django/DRF request path is synchronous, so MREG uses the synchronous `treetop-client` API directly. Shadow mode also waits. This ensures its comparison uses the policy bundle that was active for the request and exercises the exact latency, timeout, circuit, and response-validation path that enforcement will use. The former PostgreSQL -outbox, migration, dispatcher, retry/dead-letter state, and Gunicorn background -thread are intentionally absent. +outbox, migration, dispatcher, and retry/dead-letter state are intentionally +absent. ## One endpoint stack and one HTTP call @@ -36,13 +36,15 @@ ordered results locally using the tree's AND/OR structure. Examples include: - all old and new targets required for a hostname rename; - any IP attached to a host matching a NetGroup rule; -- any host-policy role label matching the host's derived labels; -- DNS-name, reserved-address, ownership, and target facts in the same endpoint +- the exact host-policy role together with the candidate hostname and IP; +- DNS-name, reserved-address, ownership, and target checks in the same endpoint decision. -The request scope caches an identical repeated stack and rejects a second -different stack. `mreg_policy_authorize_calls_per_request` makes violations of -the one-call invariant observable. +State attached to the underlying Django request caches an identical repeated +stack. A second different stack is rejected and increments +`mreg_policy_stack_conflicts_total`: shadow mode returns the legacy result and +enforce mode fails closed. This makes the one-stack invariant independent of +middleware and explicit at the authorization boundary. ## Principal, action, resource, and facts @@ -52,8 +54,10 @@ Each leaf sends: memberships; - one explicit action such as `MREG::Action::"host_update"`; - a typed resource such as `MREG::Host::"host.example.org"`; -- raw facts needed by Cedar, including hostname, IP, network, DNS-name shape, - target/self relationship, host-group ownership, or host-policy role label. +- contract-typed attributes needed by Cedar. NetGroup and DNS-name checks send + the raw `hostname` and, when available, `ip`; TreeTop derives `nameLabels` + from the bundle. Other endpoints can send business relationship attributes + such as `selfAccess` or `requesterIsOwner`. MREG does not send a precomputed `allow` fact. Relationship booleans such as `selfAccess` and `requesterIsOwner` describe request state; Cedar decides what @@ -77,12 +81,14 @@ deployed bundle: | `group` | Cedar principal group | | `range` | Cedar `ip.isInRange(...)` or exact network condition | | `regex` | named pattern in `labels.json` | -| `labels` | derived label name used by Cedar/host-policy rules | +| `labels` | conversion-only join key to exact `HostPolicyRole` names | TreeTop applies all regexes in the bundle to the raw `hostname` fact and adds -`nameLabels`. Cedar checks labels such as `netgroup_example_org`; MREG neither -runs the bundle regex nor invents the label. This is the intended TreeTop label -boundary. +`nameLabels`. Cedar checks deterministic generated labels; MREG neither runs +the bundle regex nor sends those labels. Legacy permission and role labels are +not runtime facts. The converter uses them only to discover which exact roles +each network permission used to cover, then writes rules whose resource is that +specific `MREG::HostPolicyRole`. In `enforce`, the NetGroupRegexPermission API remains readable but returns HTTP 409 for POST, PUT, PATCH, and DELETE. This prevents the database from appearing @@ -103,11 +109,11 @@ protected endpoints in `enforce`, including: - DNS wildcard/underscore restrictions; - restricted IP assignment; - host-group ownership and membership changes; -- host-policy role-to-host label matching. +- host-policy role-to-host mapping generated from the legacy permission export. ## Failure behavior -The timeout defaults to five seconds. Each Gunicorn worker owns a reusable +The timeout defaults to five seconds. Each application process owns a reusable client and a thread-safe closed/open/half-open circuit breaker. After the configured consecutive failures, the circuit rejects calls until its cooldown; one request then probes the service. @@ -126,20 +132,40 @@ test scopes; it cannot bypass enforcement. | Organization manifest | `treetop/data/treetop-bundle.toml` | | MREG module manifest | `treetop/data/treetop-mreg-module.toml` | | Global module manifest | `treetop/data/treetop-global-module.toml` | -| Cedar policy | `treetop/data/mreg.cedar` | | Global super policy | `treetop/data/global.cedar` | -| Derived labels | `treetop/data/labels.json` | +| Hand-written endpoint policy | `treetop/data/mreg.cedar` | +| Generated NetGroup/role policy | `treetop/data/netgroup.cedar` | +| Generated TreeTop labels | `treetop/data/labels.json` | +| Conversion report | `treetop/data/netgroup-conversion-report.json` | +| Permission export input | `treetop/fixtures/network-permissions.txt` | +| Role export input | `treetop/fixtures/hostpolicy-roles.txt` | | Generated schema | `treetop/data/mreg.cedarschema` | | Generated archive | `treetop/data/mreg-bundle.tar.gz` | +Refresh the conversion inputs with mreg-cli against the database whose policy +is being migrated: + +```bash +mreg-cli permission network_list > treetop/fixtures/network-permissions.txt +mreg-cli policy list_roles '*' > treetop/fixtures/hostpolicy-roles.txt +python scripts/generate-treetop-policy.py +``` + +The parser consumes the commands' fixed-width tables, validates every CIDR and +regular expression, removes duplicates, collapses redundant ranges, and emits +stable hashed IDs. Review the generated Cedar and +`netgroup-conversion-report.json`, especially unmatched or unused legacy +labels. The checked-in fixtures are sanitized examples, not production policy. + +The restricted-address examples in `mreg.cedar` are also based on the sample +networks. Replace and review them for the deployment before enabling `enforce`. + Build with `treetop-bundle` 0.0.5: ```bash python scripts/generate-treetop-schema.py --check -treetop-bundle check bundle treetop/data/treetop-bundle.toml -treetop-bundle build \ - --manifest treetop/data/treetop-bundle.toml \ - --output treetop/data/mreg-bundle.tar.gz +python scripts/generate-treetop-policy.py --check +TREETOP_BUNDLE_BIN=treetop-bundle scripts/build-treetop-bundle.sh TREETOP_BUNDLE_BIN=treetop-bundle scripts/check-treetop-bundle.sh ``` @@ -188,4 +214,4 @@ container—not its own `localhost`. 4. Add Cedar permits/forbids and derived label rules together. 5. Regenerate the schema and archive. 6. Test legacy behavior, shadow comparison, enforce allow/deny/error behavior, - and the one-call invariant. + and the one-stack invariant. diff --git a/docs/testing.md b/docs/testing.md index c1229a84..f6394e29 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -177,17 +177,13 @@ If a test fails only when run in parallel: ## CI/CD Integration -Container tests use the dedicated `test` target, which contains test modules -and development-only dependencies such as `unittest-parametrize`: +The application image includes the test entrypoint used by CI: ```bash -docker build --target test -t mreg-test . -docker run --rm mreg-test +docker build -t mreg . +docker run --rm --entrypoint /app/entrypoint-test.sh mreg ``` -The default/final image is the `runtime` target and excludes test packages and -test-only dependencies. - The parallel flag is already enabled in `tox.ini` for all test environments: ```ini diff --git a/entrypoint-test.sh b/entrypoint-test.sh index 5abf8b06..37255610 100644 --- a/entrypoint-test.sh +++ b/entrypoint-test.sh @@ -1,6 +1,5 @@ #!/bin/sh -set -eu - +set -e cd /app -python manage.py create_citext_extension --database template1 -python manage.py test --noinput --failfast --parallel +uv run ./manage.py create_citext_extension --database template1 +uv run ./manage.py test --noinput --failfast --parallel diff --git a/entrypoint.sh b/entrypoint.sh index ab7991e8..05f45d6c 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,21 +1,18 @@ #!/bin/sh -set -eu - +set -e cd /app -python manage.py create_citext_extension -python manage.py migrate +uv run ./manage.py create_citext_extension +uv run ./manage.py migrate +#uv run ./manage.py runserver 0.0.0.0:8000 -# Configure multiprocess metrics only for the new Gunicorn process. Doing this -# after one-shot management commands prevents their metric files becoming stale. -PROMETHEUS_MULTIPROC_DIR="${PROMETHEUS_MULTIPROC_DIR:-/tmp/mreg-prometheus-multiproc}" -export PROMETHEUS_MULTIPROC_DIR -mkdir -p "$PROMETHEUS_MULTIPROC_DIR" -find "$PROMETHEUS_MULTIPROC_DIR" -maxdepth 1 -type f -name '*.db' -delete +# pass signals on to the gunicorn process +function sigterm() +{ + echo "Received SIGTERM" + kill -term `cat /var/run/gunicorn.pid` +} +trap sigterm SIGTERM -# Let gunicorn become PID 1 so container stop signals are delivered directly. -exec gunicorn \ - --config /app/mregsite/gunicorn_conf.py \ - --workers 3 \ - --bind 0.0.0.0:8000 \ - --pid /var/run/gunicorn.pid \ - mregsite.wsgi +# doing it this way to be able to forward signals +uv run gunicorn --workers=3 --bind=0.0.0.0 mregsite.wsgi --pid /var/run/gunicorn.pid & +wait $! diff --git a/hostpolicy/api/permissions.py b/hostpolicy/api/permissions.py index 6aa0cccb..f8b9f90b 100644 --- a/hostpolicy/api/permissions.py +++ b/hostpolicy/api/permissions.py @@ -63,11 +63,6 @@ def has_permission(self, request, view): def _authorize_role_host_membership(self, *, request, view, legacy: bool) -> bool: role_name = str(view.kwargs.get("name") or "") hostname = str(view.kwargs.get("host") or request.data.get("name") or "") - role_labels = tuple( - HostPolicyRole.objects.filter(name=role_name).values_list( - "labels__name", flat=True - ) - ) ips = tuple( str(ip) for ip in Host.objects.filter(name=hostname) @@ -77,17 +72,13 @@ def _authorize_role_host_membership(self, *, request, view, legacy: bool) -> boo leaves = tuple( policy_leaf( action="hostpolicy_role_host_membership_update", - resource_kind="Host", - resource_id=hostname or "any", + resource_kind="HostPolicyRole", + resource_id=role_name or "any", resource_attrs={ - "kind": "host", - "name": hostname, "hostname": hostname, "ip": ip, - "roleLabel": str(label), }, ) - for label in role_labels for ip in ips ) root = ( @@ -95,13 +86,9 @@ def _authorize_role_host_membership(self, *, request, view, legacy: bool) -> boo if leaves else policy_leaf( action="hostpolicy_role_host_membership_update", - resource_kind="Host", - resource_id=hostname or "any", - resource_attrs={ - "kind": "host", - "name": hostname, - "hostname": hostname, - }, + resource_kind="HostPolicyRole", + resource_id=role_name or "any", + resource_attrs={"hostname": hostname}, ) ) return authorize_policy_stack( diff --git a/monitoring/grafana/treetop-parity.json b/monitoring/grafana/treetop-parity.json index a3b78461..df63793b 100644 --- a/monitoring/grafana/treetop-parity.json +++ b/monitoring/grafana/treetop-parity.json @@ -20,9 +20,9 @@ }, { "id": 3, - "title": "Authorize calls per request", + "title": "Endpoint stack conflicts", "type": "timeseries", - "targets": [{"expr": "rate(mreg_policy_authorize_calls_per_request_sum[5m]) / clamp_min(rate(mreg_policy_authorize_calls_per_request_count[5m]), 1)", "legendFormat": "average"}], + "targets": [{"expr": "sum(increase(mreg_policy_stack_conflicts_total[5m]))", "legendFormat": "conflicts"}], "fieldConfig": {"defaults": {"thresholds": {"steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}}, "overrides": []}, "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8} }, @@ -70,5 +70,5 @@ "time": {"from": "now-24h", "to": "now"}, "title": "MREG TreeTop endpoint rollout", "uid": "mreg-treetop-parity", - "version": 2 + "version": 3 } diff --git a/monitoring/treetop-alerts.yml b/monitoring/treetop-alerts.yml index b9bc323d..7bd55626 100644 --- a/monitoring/treetop-alerts.yml +++ b/monitoring/treetop-alerts.yml @@ -30,17 +30,13 @@ groups: severity: page annotations: summary: A synchronous TreeTop worker circuit is open - - alert: MregTreeTopMultipleCallsPerRequest - expr: | - sum(rate(mreg_policy_authorize_calls_per_request_count[5m])) - - - sum(rate(mreg_policy_authorize_calls_per_request_bucket{le="1.0"}[5m])) - > 0 - for: 5m + - alert: MregTreeTopStackConflict + expr: increase(mreg_policy_stack_conflicts_total[5m]) > 0 + for: 0m labels: severity: page annotations: - summary: A request made more than one TreeTop authorize call + summary: A request attempted to evaluate two different TreeTop stacks - alert: MregTreeTopEnforcementFailure expr: increase(mreg_policy_enforcement_results_total{result="error_deny"}[5m]) > 0 for: 0m diff --git a/mreg/api/permissions.py b/mreg/api/permissions.py index 9288a829..de6341a9 100644 --- a/mreg/api/permissions.py +++ b/mreg/api/permissions.py @@ -673,40 +673,15 @@ def _target_policy_node( resource_kind: str, resource_id: str, policy_name: str | None = None, - extra_attrs: Mapping[str, str] | None = None, ): - """Build an OR of target-IP leaves with raw authorization facts.""" + """Build an OR of leaves containing only the raw target name and IP.""" checked_name = str(policy_name or hostname) values = tuple(ips) or (None,) leaves = [] for ip in values: - attrs = { - "kind": self._snake_case(resource_kind), - "name": checked_name, - "hostname": str(hostname), - "dnsWildcard": str("*" in checked_name).lower(), - "dnsWildcardValidDepth": str(checked_name.count(".") >= 3).lower(), - "dnsUnderscore": str("_" in checked_name).lower(), - } + attrs = {"hostname": checked_name} if ip is not None: attrs["ip"] = str(ip) - network = Network.objects.filter(network__net_contains=str(ip)).first() - attrs["ipReserved"] = str(bool(network and network.is_reserved_ipaddress(str(ip)))).lower() - attrs["ipRestricted"] = str( - bool( - network - and ( - network.is_reserved_ipaddress(str(ip)) - or ipaddress.ip_address(ip) - in { - network.network.network_address, - network.network.broadcast_address, - } - ) - ) - ).lower() - if extra_attrs: - attrs.update(extra_attrs) leaves.append( policy_leaf( action=action, diff --git a/mreg/api/tests/test_metrics.py b/mreg/api/tests/test_metrics.py index d9d205a9..265a2432 100644 --- a/mreg/api/tests/test_metrics.py +++ b/mreg/api/tests/test_metrics.py @@ -1,6 +1,5 @@ import ldap -import os -from tempfile import TemporaryDirectory +import re from unittest.mock import Mock, patch from rest_framework.test import APIClient @@ -11,23 +10,22 @@ from mreg.models.host import Host, Ipaddress from mreg.middleware.metrics import PrometheusRequestMiddleware -from mreg.tests.prometheus_test_utils import parse_prometheus_metric as _parse_prometheus_metric -class MetricsTests(TestCase): - @patch("mreg.api.views.multiprocess.MultiProcessCollector") - @patch("mreg.api.views.generate_latest", return_value=b"# multiprocess metrics\n") - def test_metrics_endpoint_aggregates_gunicorn_workers(self, generate_latest_mock, collector_mock) -> None: - with TemporaryDirectory() as metrics_dir: - with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": metrics_dir}): - response: Any = APIClient().get("/api/meta/metrics") - - assert response.status_code == 200 - assert response.content == b"# multiprocess metrics\n" - registry = collector_mock.call_args.args[0] - collector_mock.assert_called_once_with(registry, path=metrics_dir) - generate_latest_mock.assert_called_once_with(registry) +def _parse_prometheus_metric(content: str, metric_name: str) -> dict[str, float]: + """Parse Prometheus text exposition and return samples for one metric.""" + result: dict[str, float] = {} + pattern = rf"^{re.escape(metric_name)}(\{{[^}}]*\}})?\s+([0-9.e+-]+)$" + for line in content.split("\n"): + if line.startswith("#"): + continue + match = re.match(pattern, line) + if match: + result[match.group(1) or ""] = float(match.group(2)) + return result + +class MetricsTests(TestCase): def test_metrics_endpoint_exposes_prometheus_metrics(self) -> None: """Test that metrics endpoint returns Prometheus-formatted output.""" client = APIClient() diff --git a/mreg/api/treetop.py b/mreg/api/treetop.py index a3744616..56100691 100644 --- a/mreg/api/treetop.py +++ b/mreg/api/treetop.py @@ -3,7 +3,6 @@ from __future__ import annotations import atexit -import asyncio import ipaddress import logging import os @@ -33,6 +32,7 @@ from mreg.models.auth import User as MregUser from mreg.policy.config import PolicyMode +from mreg.policy.contracts import ENDPOINT_ATTRIBUTE_TYPES logger = structlog.get_logger("mreg.policy.parity") @@ -81,7 +81,6 @@ "mreg_policy_mode_info", "Configured MREG policy decision mode.", ["mode"], - multiprocess_mode="livemax", ) POLICY_MODE_INFO.labels(mode=POLICY_MODE.value).set(1) POLICY_AUTHORIZE_DURATION_SECONDS = Histogram( @@ -95,15 +94,13 @@ "Number of Cedar checks in one endpoint policy stack.", buckets=[1, 2, 3, 5, 8, 13, 21], ) -POLICY_CALLS_PER_REQUEST = Histogram( - "mreg_policy_authorize_calls_per_request", - "Number of TreeTop authorize HTTP calls made by one MREG request.", - buckets=[0, 1, 2], +POLICY_STACK_CONFLICTS_TOTAL = Counter( + "mreg_policy_stack_conflicts_total", + "Attempts to evaluate two different endpoint policy stacks in one request.", ) POLICY_CIRCUIT_OPEN = Gauge( "mreg_policy_circuit_open", "Whether this worker's synchronous TreeTop circuit is open.", - multiprocess_mode="livemax", ) @@ -198,16 +195,12 @@ def policy_any(*nodes: PolicyNode) -> PolicyAny: @dataclass(slots=True) class _RequestPolicyState: - calls: int = 0 fingerprint: tuple[object, ...] | None = None policy_decision: bool | None = None error: str | None = None -_request_state: ContextVar[_RequestPolicyState | None] = ContextVar( - "mreg_policy_request_state", - default=None, -) +_REQUEST_POLICY_STATE_ATTRIBUTE = "_mreg_policy_state" _shadow_disabled_depth: ContextVar[int] = ContextVar( "mreg_policy_shadow_disabled_depth", default=0, @@ -292,11 +285,8 @@ def close_policy_client() -> None: _client_pid = None if client is None: return - try: - asyncio.run(client.aclose()) - except Exception: - with suppress(Exception): - client.close() + with suppress(Exception): + client.close() atexit.register(close_policy_client) @@ -313,22 +303,6 @@ def _record_failure(stage: str, error: str, **context: object) -> None: _safe_log(logging.ERROR, "policy_integration_error", stage=stage, error=error, **context) -@contextmanager -def policy_request_scope(): - """Record and enforce the one-authorize-call-per-request invariant.""" - if _request_state.get() is not None: - yield - return - state = _RequestPolicyState() - token = _request_state.set(state) - try: - yield - finally: - with suppress(Exception): - POLICY_CALLS_PER_REQUEST.observe(float(state.calls)) - _request_state.reset(token) - - @contextmanager def disable_policy_parity(): """Disable synchronous shadow comparisons in a narrow test scope.""" @@ -376,13 +350,14 @@ def _build_resource_attrs(resource_attrs: Mapping[str, str]) -> dict[str, Resour attrs: dict[str, ResourceAttribute] = {} for key, value in resource_attrs.items(): normalized = str(value) - if normalized.lower() in {"true", "false"}: + cedar_type = ENDPOINT_ATTRIBUTE_TYPES.get(key) + if cedar_type == "Bool": + if normalized.lower() not in {"true", "false"}: + raise ValueError(f"Policy attribute {key!r} must be a boolean") attrs[key] = ResourceAttribute.new(normalized.lower(), ResourceAttributeType.BOOLEAN) - continue - try: - ip = ipaddress.ip_address(normalized) - attrs[key] = ResourceAttribute.new(str(ip), ResourceAttributeType.IP) - except ValueError: + elif cedar_type == "ipaddr": + attrs[key] = ResourceAttribute.new(str(ipaddress.ip_address(normalized)), ResourceAttributeType.IP) + else: attrs[key] = ResourceAttribute.new(normalized, ResourceAttributeType.STRING) return attrs @@ -442,6 +417,16 @@ def _node_fingerprint(node: PolicyNode) -> tuple[object, ...]: ) +def _request_policy_state(request: Request) -> _RequestPolicyState: + """Return state owned by the underlying Django request.""" + owner = getattr(request, "_request", request) + state = getattr(owner, _REQUEST_POLICY_STATE_ATTRIBUTE, None) + if state is None: + state = _RequestPolicyState() + setattr(owner, _REQUEST_POLICY_STATE_ATTRIBUTE, state) + return state + + def _result_decision(result: AuthorizeResultBrief, index: int) -> bool: if result.index != index: raise RuntimeError(f"Authorization result index {result.index} does not match {index}") @@ -462,9 +447,11 @@ def _authorize_stack( if not leaves: raise RuntimeError("Endpoint policy stack is empty") fingerprint = _node_fingerprint(root) - state = _request_state.get() - if state is not None and state.fingerprint is not None: + state = _request_policy_state(request) + if state.fingerprint is not None: if state.fingerprint != fingerprint: + with suppress(Exception): + POLICY_STACK_CONFLICTS_TOTAL.inc() raise RuntimeError("A second different endpoint policy stack was evaluated in one request") if state.error is not None: raise RuntimeError(state.error) @@ -472,18 +459,18 @@ def _authorize_stack( raise RuntimeError("Cached endpoint policy stack has no decision") return state.policy_decision + state.fingerprint = fingerprint if not POLICY_BASE_URL: - raise RuntimeError("MREG_POLICY_BASE_URL is not configured") + state.error = "MREG_POLICY_BASE_URL is not configured" + raise RuntimeError(state.error) if not _circuit.allow_call(): - raise RuntimeError("TreeTop circuit breaker is open") + state.error = "TreeTop circuit breaker is open" + raise RuntimeError(state.error) user = MregUser.from_request(request) policy_requests = [_build_policy_request(user, leaf.check, request_id=f"mreg-{index}") for index, leaf in enumerate(leaves)] POLICY_STACK_SIZE.observe(float(len(policy_requests))) started = monotonic() - if state is not None: - state.calls += 1 - state.fingerprint = fingerprint try: response = _get_treetop_client().authorize( policy_requests, @@ -498,16 +485,14 @@ def _authorize_stack( POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="exception").inc() POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="exception").observe(monotonic() - started) error = f"{type(exc).__name__}: {exc}" - if state is not None: - state.error = error + state.error = error raise RuntimeError(error) from exc _circuit.success() POLICY_AUTHORIZE_CALLS_TOTAL.labels(status="success").inc() POLICY_AUTHORIZE_DURATION_SECONDS.labels(status="success").observe(monotonic() - started) policy_decision = _evaluate_tree(root, iter(decisions)) - if state is not None: - state.policy_decision = policy_decision + state.policy_decision = policy_decision if POLICY_PARITY_LOG_DETAILS: context["checks"] = [ { diff --git a/mreg/api/v1/tests/test_logging.py b/mreg/api/v1/tests/test_logging.py index 48646af3..dfdd3d25 100644 --- a/mreg/api/v1/tests/test_logging.py +++ b/mreg/api/v1/tests/test_logging.py @@ -143,26 +143,6 @@ def mock_get_response(_): # Check that the body was logged as '' self.assertEqual(cap_logs[0]["content"], "") - def test_middleware_uses_policy_request_scope(self) -> None: - """Ensure request handling is wrapped in one policy request scope.""" - middleware = LoggingMiddleware(MagicMock()) - - def mock_get_response(_): - return HttpResponse(status=200) - - middleware.get_response = mock_get_response - - request = HttpRequest() - request._body = b"Some request body" - request.user = get_user_model().objects.get(username="superuser") - - with patch("mreg.middleware.logging_http.policy_request_scope") as mock_scope: - middleware(request) - mock_scope.assert_called_once() - mock_scope.return_value.__enter__.assert_called_once() - mock_scope.return_value.__exit__.assert_called_once() - - class TestLoggingMiddleware(MregAPITestCase): """Test logging middleware.""" diff --git a/mreg/api/views.py b/mreg/api/views.py index a6e64c96..6f6e95d4 100644 --- a/mreg/api/views.py +++ b/mreg/api/views.py @@ -1,4 +1,3 @@ -import os import platform import time from time import monotonic @@ -23,11 +22,9 @@ from drf_spectacular.utils import OpenApiParameter, OpenApiTypes, extend_schema from prometheus_client import ( CONTENT_TYPE_LATEST, - CollectorRegistry, Counter, Histogram, generate_latest, - multiprocess, ) from mreg.__about__ import __version__ as mreg_version @@ -363,11 +360,4 @@ class MetricsView(APIView): responses={(status.HTTP_200_OK, "text/plain"): PROMETHEUS_METRICS_TEXT_SCHEMA}, ) def get(self, request: Request): - multiprocess_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR") - if multiprocess_dir: - registry = CollectorRegistry() - multiprocess.MultiProcessCollector(registry, path=multiprocess_dir) - metrics = generate_latest(registry) - else: - metrics = generate_latest() - return HttpResponse(metrics, content_type=CONTENT_TYPE_LATEST) + return HttpResponse(generate_latest(), content_type=CONTENT_TYPE_LATEST) diff --git a/mreg/middleware/logging_http.py b/mreg/middleware/logging_http.py index 2f5b827f..9232e11b 100644 --- a/mreg/middleware/logging_http.py +++ b/mreg/middleware/logging_http.py @@ -10,7 +10,6 @@ import traceback from django.conf import settings from django.http import HttpRequest, HttpResponse -from mreg.api.treetop import policy_request_scope mreg_logger = structlog.getLogger("mreg.http") @@ -48,12 +47,11 @@ def __call__(self, request: HttpRequest) -> HttpResponse: self.log_request(request) - with policy_request_scope(): - try: - response = self.get_response(request) - except Exception as e: # pragma: no cover (this is somewhat tricky to properly test) - self.log_exception(request, e, start_time) - raise + try: + response = self.get_response(request) + except Exception as e: # pragma: no cover (this is somewhat tricky to properly test) + self.log_exception(request, e, start_time) + raise self.log_response(request, response, start_time) return response diff --git a/mreg/middleware/metrics.py b/mreg/middleware/metrics.py index 260ac21f..55745baa 100644 --- a/mreg/middleware/metrics.py +++ b/mreg/middleware/metrics.py @@ -29,12 +29,7 @@ buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], ) -INPROGRESS = Gauge( - "mreg_http_inprogress_requests", - "Inprogress requests", - ["method", "path"], - multiprocess_mode="livesum", -) +INPROGRESS = Gauge("mreg_http_inprogress_requests", "Inprogress requests", ["method", "path"]) # Request/response sizes (bytes) REQUEST_SIZE = Histogram( diff --git a/mreg/policy/contracts.py b/mreg/policy/contracts.py index c0400328..c6a74f7e 100644 --- a/mreg/policy/contracts.py +++ b/mreg/policy/contracts.py @@ -58,20 +58,18 @@ def snake_case(value: str) -> str: ("hostname", "String"), ("ip", "ipaddr"), ("nameLabels", "Set"), - ("dnsWildcard", "Bool"), - ("dnsWildcardValidDepth", "Bool"), - ("dnsUnderscore", "Bool"), - ("ipReserved", "Bool"), - ("ipRestricted", "Bool"), ("selfAccess", "Bool"), ("requesterIsOwner", "Bool"), ("ownerMutation", "Bool"), ("descriptionUpdate", "Bool"), - ("roleLabel", "String"), ("network", "String"), ) ) +ENDPOINT_ATTRIBUTE_TYPES = { + attribute.name: attribute.cedar_type for attribute in ENDPOINT_ATTRIBUTES +} + RESOURCE_CONTRACTS = ( ResourceContract("Generic", attributes=ENDPOINT_ATTRIBUTES), diff --git a/mreg/tests/prometheus_test_utils.py b/mreg/tests/prometheus_test_utils.py deleted file mode 100644 index 2ae7b2ec..00000000 --- a/mreg/tests/prometheus_test_utils.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -import re - -from prometheus_client import generate_latest - - -def parse_prometheus_metric(content: str, metric_name: str) -> dict[str, float]: - """Parse Prometheus text exposition and return samples for one metric name.""" - result: dict[str, float] = {} - pattern = rf"^{re.escape(metric_name)}(\{{[^}}]*\}})?\s+([0-9.e+-]+)$" - for line in content.split("\n"): - if line.startswith("#"): - continue - match = re.match(pattern, line) - if match: - labels = match.group(1) or "" - result[labels] = float(match.group(2)) - return result - - -def prometheus_registry_text() -> str: - """Return the current default Prometheus registry text format.""" - return generate_latest().decode("utf-8") - - -def metric_by_label(metric_name: str, label_filter: str, *, content: str | None = None) -> float: - """Return metric value for the first sample whose label set contains label_filter.""" - raw = content if content is not None else prometheus_registry_text() - values = parse_prometheus_metric(raw, metric_name) - for labels, value in values.items(): - if label_filter in labels: - return value - return 0.0 - - -def metric_total(metric_name: str, *, content: str | None = None) -> float: - """Return sum of all samples for a metric from Prometheus text content.""" - raw = content if content is not None else prometheus_registry_text() - values = parse_prometheus_metric(raw, metric_name) - return sum(values.values()) if values else 0.0 diff --git a/mreg/tests/test_gunicorn_conf.py b/mreg/tests/test_gunicorn_conf.py deleted file mode 100644 index e49d5fef..00000000 --- a/mreg/tests/test_gunicorn_conf.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Tests for worker-scoped Gunicorn lifecycle hooks.""" - -import os -from types import SimpleNamespace -from unittest.mock import patch - -from django.test import SimpleTestCase - -from mregsite import gunicorn_conf - - -class GunicornLifecycleHookTests(SimpleTestCase): - """Ensure worker-local policy clients follow Gunicorn lifecycle.""" - - @patch("mreg.api.treetop.close_policy_client") - def test_worker_exit_closes_policy_client(self, close_client): - gunicorn_conf.worker_exit(None, None) - - close_client.assert_called_once_with() - - @patch("mreg.api.treetop.close_policy_client") - @patch("django.apps.apps") - def test_worker_exit_is_safe_before_django_setup(self, apps, close_client): - apps.ready = False - - gunicorn_conf.worker_exit(None, SimpleNamespace(pid=42)) - - close_client.assert_not_called() - - @patch("prometheus_client.multiprocess.mark_process_dead") - @patch("mreg.api.treetop.close_policy_client") - def test_worker_exit_marks_prometheus_process_dead(self, close_client, mark_process_dead): - with patch.dict(os.environ, {"PROMETHEUS_MULTIPROC_DIR": "/tmp/prometheus"}): - gunicorn_conf.worker_exit(None, SimpleNamespace(pid=42)) - - close_client.assert_called_once_with() - mark_process_dead.assert_called_once_with(42) diff --git a/mreg/tests/test_treetop.py b/mreg/tests/test_treetop.py index fe14e848..82f5de7b 100644 --- a/mreg/tests/test_treetop.py +++ b/mreg/tests/test_treetop.py @@ -6,6 +6,8 @@ from django.test import SimpleTestCase from rest_framework.test import APIRequestFactory +from hostpolicy.api.permissions import IsSuperOrHostPolicyAdminOrReadOnly +from mreg.api.permissions import IsGrantedNetGroupRegexPermission from mreg.api.treetop import ( PolicyAll, PolicyAny, @@ -20,7 +22,6 @@ policy_all, policy_any, policy_leaf, - policy_request_scope, policy_shadow_enabled, ) from mreg.policy.config import PolicyMode @@ -74,11 +75,58 @@ def test_policy_contracts_reject_empty_values(self) -> None: with self.assertRaisesRegex(ValueError, "at least one"): PolicyAny(()) - def test_resource_attributes_detect_bool_ip_and_string(self) -> None: - attrs = _build_resource_attrs({"restricted": "true", "ip": "192.0.2.1", "name": "host"}) - self.assertEqual(attrs["restricted"].type.value, "Bool") + def test_resource_attributes_follow_contract_types(self) -> None: + attrs = _build_resource_attrs({"selfAccess": "true", "ip": "192.0.2.1", "hostname": "true"}) + self.assertEqual(attrs["selfAccess"].type.value, "Bool") self.assertEqual(attrs["ip"].type.value, "Ip") - self.assertEqual(attrs["name"].type.value, "String") + self.assertEqual(attrs["hostname"].type.value, "String") + with self.assertRaisesRegex(ValueError, "boolean"): + _build_resource_attrs({"selfAccess": "yes"}) + + def test_netgroup_targets_send_only_raw_hostname_and_ip(self) -> None: + root = IsGrantedNetGroupRegexPermission()._target_policy_node( + hostname="old.example.org", + policy_name="new.example.org", + ips=("192.0.2.10", "2001:db8::10"), + action="host_update", + resource_kind="Host", + resource_id="old.example.org", + ) + + self.assertIsInstance(root, PolicyAny) + self.assertEqual( + [dict(child.check.resource.attrs) for child in root.children], + [ + {"hostname": "new.example.org", "ip": "192.0.2.10"}, + {"hostname": "new.example.org", "ip": "2001:db8::10"}, + ], + ) + + @patch("hostpolicy.api.permissions.authorize_policy_stack", return_value=True) + @patch("hostpolicy.api.permissions.Host.objects") + def test_hostpolicy_membership_sends_exact_role_and_raw_host(self, host_objects, authorize) -> None: + host_objects.filter.return_value.exclude.return_value.values_list.return_value = ["192.0.2.10"] + request = self.factory.post("/api/v1/hostpolicy/roles/web/hosts/") + view = SimpleNamespace(kwargs={"name": "web", "host": "web-1.example.org"}) + + self.assertTrue( + IsSuperOrHostPolicyAdminOrReadOnly()._authorize_role_host_membership( + request=request, + view=view, + legacy=False, + ) + ) + + root = authorize.call_args.kwargs["root"] + self.assertIsInstance(root, PolicyAny) + self.assertEqual(len(root.children), 1) + leaf = root.children[0] + self.assertEqual(leaf.check.resource.kind, "HostPolicyRole") + self.assertEqual(leaf.check.resource.id, "web") + self.assertEqual( + dict(leaf.check.resource.attrs), + {"hostname": "web-1.example.org", "ip": "192.0.2.10"}, + ) @patch("mreg.api.treetop.MregUser.from_request") @patch("mreg.api.treetop._get_treetop_client") @@ -94,7 +142,6 @@ def test_nested_stack_uses_one_batched_authorize_call(self, get_client, from_req with ( patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - policy_request_scope(), ): self.assertTrue(authorize_policy_stack(False, request=self._request(), root=root)) @@ -110,13 +157,13 @@ def test_identical_stack_is_cached_inside_request(self, get_client, from_request client = self._client(True) get_client.return_value = client root = self._leaf() + request = self._request() with ( patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - policy_request_scope(), ): - self.assertTrue(authorize_policy_stack(False, request=self._request(), root=root)) - self.assertTrue(authorize_policy_stack(False, request=self._request(), root=root)) + self.assertTrue(authorize_policy_stack(False, request=request, root=root)) + self.assertTrue(authorize_policy_stack(False, request=request, root=root)) client.authorize.assert_called_once() @patch("mreg.api.treetop.MregUser.from_request") @@ -125,13 +172,13 @@ def test_second_different_stack_denies_without_second_call(self, get_client, fro from_request.return_value = self.user client = self._client(True) get_client.return_value = client + request = self._request() with ( patch("mreg.api.treetop.POLICY_MODE", PolicyMode.ENFORCE), patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - policy_request_scope(), ): - self.assertTrue(authorize_policy_stack(False, request=self._request(), root=self._leaf("one"))) - self.assertFalse(authorize_policy_stack(True, request=self._request(), root=self._leaf("two"))) + self.assertTrue(authorize_policy_stack(False, request=request, root=self._leaf("one"))) + self.assertFalse(authorize_policy_stack(True, request=request, root=self._leaf("two"))) client.authorize.assert_called_once() @patch("mreg.api.treetop._get_treetop_client") @@ -166,7 +213,6 @@ def test_shadow_calls_synchronously_but_returns_legacy(self, get_client, from_re patch("mreg.api.treetop.POLICY_MODE", PolicyMode.SHADOW), patch("mreg.api.treetop.POLICY_PARITY_ENABLED", True), patch("mreg.api.treetop.POLICY_BASE_URL", "http://policy"), - policy_request_scope(), ): self.assertFalse(authorize_policy_stack(False, request=self._request(), root=self._leaf())) get_client.return_value.authorize.assert_called_once() diff --git a/mreg/tests/test_treetop_policy_generator.py b/mreg/tests/test_treetop_policy_generator.py new file mode 100644 index 00000000..c684a9cf --- /dev/null +++ b/mreg/tests/test_treetop_policy_generator.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +from unittest import TestCase + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts/generate-treetop-policy.py" +SPEC = importlib.util.spec_from_file_location("_generate_treetop_policy", SCRIPT) +if SPEC is None or SPEC.loader is None: # pragma: no cover + raise RuntimeError(f"Unable to load {SCRIPT}") +generator = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = generator +SPEC.loader.exec_module(generator) + + +class TreeTopPolicyGeneratorTests(TestCase): + permission_table = """\ +Range Group Regex Labels +10.0.0.0/24 group two ^web [0-9]+\\.example$ Shared, Unused +10.0.0.1/32 group two ^web [0-9]+\\.example$ Shared +2001:db8::1/128 ipv6 .*\\.example$ +""" + role_table = """\ +Name Description with spaces Labels +role1 A delegated role Shared +role2 A role without permission Missing +""" + + def test_parser_preserves_spaces_and_normalizes_values(self) -> None: + permissions = generator.parse_permissions(self.permission_table) + self.assertEqual(permissions[0].group, "group two") + self.assertEqual(permissions[0].regex, r"^web [0-9]+\.example$") + self.assertEqual(permissions[0].labels, ("Shared", "Unused")) + self.assertEqual(permissions[-1].network, "2001:db8::1/128") + + roles = generator.parse_roles(self.role_table) + self.assertEqual(roles[0].name, "role1") + self.assertEqual(roles[0].labels, ("Shared",)) + + def test_generation_uses_derived_labels_and_exact_roles(self) -> None: + permissions = generator.parse_permissions(self.permission_table) + roles = generator.parse_roles(self.role_table) + result = generator.generate_policy(permissions, roles) + report = json.loads(result.report) + + derived_label = report["derived_labels"][r"^web [0-9]+\.example$"] + self.assertIn(f'resource.nameLabels.contains("{derived_label}")', result.cedar) + self.assertIn('resource == MREG::HostPolicyRole::"role1"', result.cedar) + self.assertNotIn("Shared", result.cedar) + self.assertNotIn("Unused", result.labels) + self.assertEqual(report["generated_role_rules"], 1) + self.assertEqual(report["unused_permission_labels"], ["Unused"]) + self.assertEqual(report["unmatched_role_labels"], ["Missing"]) + + def test_duplicate_rows_are_deduplicated_and_output_is_stable(self) -> None: + permissions = generator.parse_permissions(self.permission_table + self.permission_table.splitlines()[1] + "\n") + roles = generator.parse_roles(self.role_table) + first = generator.generate_policy(permissions, roles) + second = generator.generate_policy(tuple(reversed(permissions)), tuple(reversed(roles))) + self.assertEqual(first, second) + + def test_rejects_malformed_input(self) -> None: + with self.assertRaises(generator.ConversionError): + generator.parse_permissions("Range Group Regex Labels\n10.0.0.0/24 g .* x\n") + with self.assertRaises(generator.ConversionError): + generator.parse_permissions( + "Range Group Regex Labels\n10.0.0.1/24 g .* \n" + ) + + def test_cli_check_detects_stale_output(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output_dir = Path(directory) + self.assertEqual( + generator.main( + [ + "--permissions", + str(ROOT / "treetop/fixtures/network-permissions.txt"), + "--roles", + str(ROOT / "treetop/fixtures/hostpolicy-roles.txt"), + "--output-dir", + str(output_dir), + ] + ), + 0, + ) + self.assertEqual( + generator.main( + [ + "--permissions", + str(ROOT / "treetop/fixtures/network-permissions.txt"), + "--roles", + str(ROOT / "treetop/fixtures/hostpolicy-roles.txt"), + "--output-dir", + str(output_dir), + "--check", + ] + ), + 0, + ) diff --git a/mregsite/gunicorn_conf.py b/mregsite/gunicorn_conf.py deleted file mode 100644 index 0bb59696..00000000 --- a/mregsite/gunicorn_conf.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Gunicorn lifecycle hooks for process-local runtime resources.""" - -import os - - -def worker_exit(server, worker): # noqa: ARG001 - """Close worker-local clients and mark its multiprocess metrics dead.""" - from django.apps import apps - - if not apps.ready: - return - - from mreg.api.treetop import close_policy_client - - close_policy_client() - - if os.environ.get("PROMETHEUS_MULTIPROC_DIR"): - from prometheus_client import multiprocess - - multiprocess.mark_process_dead(worker.pid) diff --git a/pyproject.toml b/pyproject.toml index 994c41f6..0df83bdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ dependencies = [ "drf-spectacular[sidecar]==0.29.0", "treetop-client>=0.0.12", "prometheus-client>=0.24", + # The production image also carries the Django test entrypoint. + "unittest-parametrize", ] dynamic = ["version"] @@ -34,7 +36,6 @@ dynamic = ["version"] dev = [ "tox-uv>=1.29", "coverage[toml]", - "unittest-parametrize", "uv>=0.10", "tblib>=3", {include-group = "profile"}, diff --git a/scripts/build-treetop-bundle.sh b/scripts/build-treetop-bundle.sh new file mode 100755 index 00000000..c5e2165f --- /dev/null +++ b/scripts/build-treetop-bundle.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -eu + +bundle_bin=${TREETOP_BUNDLE_BIN:-treetop-bundle} +manifest=${TREETOP_BUNDLE_MANIFEST:-treetop/data/treetop-bundle.toml} +archive=${TREETOP_BUNDLE_ARCHIVE:-treetop/data/mreg-bundle.tar.gz} + +if ! command -v "$bundle_bin" >/dev/null 2>&1; then + echo "treetop-bundle executable not found: $bundle_bin" >&2 + exit 127 +fi + +tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/mreg-treetop-bundle-build.XXXXXX") +trap 'rm -rf "$tmpdir"' EXIT HUP INT TERM +generated_archive="$tmpdir/mreg-bundle.tar.gz" + +"$bundle_bin" check bundle "$manifest" --format human +"$bundle_bin" build \ + --manifest "$manifest" \ + --output "$generated_archive" \ + --format human +"$bundle_bin" check archive "$generated_archive" \ + --signature-policy allow-unsigned \ + --format human +mv "$generated_archive" "$archive" + +echo "Wrote reproducible unsigned TreeTop bundle to $archive" diff --git a/scripts/generate-treetop-policy.py b/scripts/generate-treetop-policy.py new file mode 100644 index 00000000..601c28f5 --- /dev/null +++ b/scripts/generate-treetop-policy.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +"""Generate TreeTop policy data from deterministic mreg-cli table output.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import ipaddress +import json +from pathlib import Path +import re +import sys +from typing import Iterable, Sequence + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_PERMISSIONS = ROOT / "treetop/fixtures/network-permissions.txt" +DEFAULT_ROLES = ROOT / "treetop/fixtures/hostpolicy-roles.txt" +DEFAULT_OUTPUT_DIR = ROOT / "treetop/data" + +PERMISSION_HEADERS = ("Range", "Group", "Regex", "Labels") +ROLE_HEADERS = ("Name", "Description", "Labels") + +LABEL_RESOURCE_KINDS = ( + "MREG::Host", + "MREG::Ipaddress", + "MREG::Cname", + "MREG::Hinfo", + "MREG::Loc", + "MREG::Mx", + "MREG::Naptr", + "MREG::NameServer", + "MREG::PtrOverride", + "MREG::Sshfp", + "MREG::Srv", + "MREG::Txt", + "MREG::BACnetID", + "MREG::HostPolicyRole", +) + +STATIC_NAME_PATTERNS = ( + {"name": "dns_wildcard", "regex": r"\*"}, + {"name": "dns_wildcard_valid_depth", "regex": r"^(?:[^.]+\.){3,}[^.]+$"}, + {"name": "dns_underscore", "regex": "_"}, +) + +IP_SCOPED_ACTIONS = ( + "host_create", + "host_update", + "host_delete", + "host_contacts_create", + "host_contacts_delete", + "ipaddress_create", + "ipaddress_update", + "ipaddress_delete", + "hinfo_create", + "hinfo_update", + "hinfo_delete", + "loc_create", + "loc_update", + "loc_delete", + "mx_create", + "mx_update", + "mx_delete", + "naptr_create", + "naptr_update", + "naptr_delete", + "name_server_create", + "name_server_update", + "name_server_delete", + "ptr_override_create", + "ptr_override_update", + "ptr_override_delete", + "sshfp_create", + "sshfp_update", + "sshfp_delete", + "srv_create", + "srv_update", + "srv_delete", + "txt_create", + "txt_update", + "txt_delete", + "bacnet_id_create", + "bacnet_id_update", + "bacnet_id_delete", +) + +HOSTNAME_SCOPED_ACTIONS = ( + "cname_create", + "cname_update", + "cname_delete", +) + +NETWORK_SCOPED_ACTIONS = ( + "community_create", + "community_update", + "community_delete", + "host_create", + "host_delete", +) + +ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +class ConversionError(ValueError): + """Raised when mreg-cli output cannot be converted safely.""" + + +@dataclass(frozen=True, order=True) +class NetworkPermission: + network: str + group: str + regex: str + labels: tuple[str, ...] + + +@dataclass(frozen=True, order=True) +class HostPolicyRole: + name: str + labels: tuple[str, ...] + + +@dataclass(frozen=True) +class GeneratedPolicy: + labels: str + cedar: str + report: str + + +def _column_starts(header_line: str, headers: Sequence[str]) -> tuple[int, ...]: + starts: list[int] = [] + cursor = 0 + for header in headers: + index = header_line.find(header, cursor) + if index < 0: + raise ConversionError(f"Expected table header {header!r}: {header_line!r}") + if starts and index - cursor < 3: + raise ConversionError(f"Table columns are not separated by at least three spaces: {header_line!r}") + starts.append(index) + cursor = index + len(header) + if header_line[: starts[0]].strip() or header_line[cursor:].strip(): + raise ConversionError(f"Unexpected content in table header: {header_line!r}") + return tuple(starts) + + +def parse_fixed_width_table(text: str, headers: Sequence[str]) -> list[tuple[str, ...]]: + """Parse an OutputManager fixed-width table without splitting field content.""" + lines = [ANSI_ESCAPE.sub("", line.rstrip()) for line in text.splitlines() if line.strip()] + if not lines: + raise ConversionError("mreg-cli output is empty") + starts = _column_starts(lines[0], headers) + rows: list[tuple[str, ...]] = [] + for line_number, line in enumerate(lines[1:], start=2): + if len(line) <= starts[-2]: + raise ConversionError(f"Row {line_number} is shorter than the required columns: {line!r}") + # OutputManager pads an empty final column with spaces. Be tolerant of + # users or editors stripping that trailing whitespace from a capture. + line = line.ljust(starts[-1]) + values = tuple( + line[start : starts[index + 1] if index + 1 < len(starts) else None].strip() + for index, start in enumerate(starts) + ) + if not any(values): + continue + if any(not value for value in values[:-1]): + raise ConversionError(f"Row {line_number} has an empty required column: {line!r}") + rows.append(values) + if not rows: + raise ConversionError("mreg-cli output contains no data rows") + return rows + + +def _parse_labels(value: str) -> tuple[str, ...]: + return tuple(sorted({label.strip() for label in value.split(",") if label.strip()})) + + +def parse_permissions(text: str) -> tuple[NetworkPermission, ...]: + permissions: set[NetworkPermission] = set() + for network_value, group, regex, labels_value in parse_fixed_width_table(text, PERMISSION_HEADERS): + try: + network = str(ipaddress.ip_network(network_value, strict=True)) + except ValueError as exc: + raise ConversionError(f"Invalid permission range {network_value!r}: {exc}") from exc + try: + re.compile(regex) + except re.error as exc: + raise ConversionError(f"Invalid permission regex {regex!r}: {exc}") from exc + permissions.add( + NetworkPermission( + network=network, + group=group, + regex=regex, + labels=_parse_labels(labels_value), + ) + ) + return tuple(sorted(permissions)) + + +def parse_roles(text: str) -> tuple[HostPolicyRole, ...]: + roles_by_name: dict[str, HostPolicyRole] = {} + for name, _description, labels_value in parse_fixed_width_table(text, ROLE_HEADERS): + role = HostPolicyRole(name=name, labels=_parse_labels(labels_value)) + if name in roles_by_name: + raise ConversionError(f"Duplicate host-policy role {name!r}") + roles_by_name[name] = role + return tuple(sorted(roles_by_name.values())) + + +def _stable_name(prefix: str, *parts: str) -> str: + digest = hashlib.sha256("\0".join(parts).encode()).hexdigest()[:12] + return f"{prefix}_{digest}" + + +def _quote(value: str) -> str: + return json.dumps(value, ensure_ascii=False) + + +def _actions(actions: Sequence[str], indent: str = " ") -> str: + values = [f'MREG::Action::{_quote(action)}' for action in actions] + return "[" + (",\n" + indent).join(values) + "]" + + +def _ranges(networks: Sequence[str], *, attribute: str = "ip") -> str: + checks = [f'resource.{attribute}.isInRange(ip({_quote(network)}))' for network in networks] + return "(" + (" ||\n ").join(checks) + ")" + + +def _network_values(networks: Sequence[str]) -> str: + checks = [f'resource.network == {_quote(network)}' for network in networks] + return "(" + (" ||\n ").join(checks) + ")" + + +def _permit_ip_rule(group: str, regex: str, networks: Sequence[str]) -> str: + label = _stable_name("netgroup", regex) + policy_id = _stable_name("netgroup_ip", group, regex) + return f'''@id("MREG.generated.{policy_id}") +permit ( + principal in MREG::Group::{_quote(group)}, + action in + {_actions(IP_SCOPED_ACTIONS)}, + resource +) +when {{ + resource has nameLabels && + resource.nameLabels.contains({_quote(label)}) && + resource has ip && + {_ranges(networks)} +}}; +''' + + +def _permit_hostname_rule(group: str, regex: str) -> str: + label = _stable_name("netgroup", regex) + policy_id = _stable_name("netgroup_hostname", group, regex) + return f'''@id("MREG.generated.{policy_id}") +permit ( + principal in MREG::Group::{_quote(group)}, + action in + {_actions(HOSTNAME_SCOPED_ACTIONS)}, + resource is MREG::Cname +) +when {{ + resource has nameLabels && + resource.nameLabels.contains({_quote(label)}) +}}; +''' + + +def _permit_network_rule(group: str, networks: Sequence[str]) -> str: + policy_id = _stable_name("netgroup_network", group) + return f'''@id("MREG.generated.{policy_id}") +permit ( + principal in MREG::Group::{_quote(group)}, + action in + {_actions(NETWORK_SCOPED_ACTIONS)}, + resource +) +when {{ + resource has network && + {_network_values(networks)} +}}; +''' + + +def _permit_role_rule(group: str, regex: str, role_name: str, networks: Sequence[str]) -> str: + label = _stable_name("netgroup", regex) + policy_id = _stable_name("hostpolicy_role", group, regex, role_name) + return f'''@id("MREG.generated.{policy_id}") +permit ( + principal in MREG::Group::{_quote(group)}, + action == MREG::Action::"hostpolicy_role_host_membership_update", + resource == MREG::HostPolicyRole::{_quote(role_name)} +) +when {{ + resource has nameLabels && + resource.nameLabels.contains({_quote(label)}) && + resource has ip && + {_ranges(networks)} +}}; +''' + + +def _group_values(permissions: Iterable[NetworkPermission], *fields: str) -> dict[tuple[str, ...], set[str]]: + grouped: dict[tuple[str, ...], set[str]] = {} + for permission in permissions: + key = tuple(str(getattr(permission, field)) for field in fields) + grouped.setdefault(key, set()).add(permission.network) + return grouped + + +def _normalized_networks(networks: Iterable[str]) -> tuple[str, ...]: + """Sort and collapse redundant ranges without mixing address families.""" + parsed = [ipaddress.ip_network(network) for network in networks] + collapsed = [ + network + for version in (4, 6) + for network in ipaddress.collapse_addresses( + network for network in parsed if network.version == version + ) + ] + return tuple(str(network) for network in collapsed) + + +def generate_policy(permissions: Sequence[NetworkPermission], roles: Sequence[HostPolicyRole]) -> GeneratedPolicy: + permissions = tuple(sorted(set(permissions))) + roles = tuple(sorted(set(roles))) + regexes = sorted({permission.regex for permission in permissions}) + patterns = [ + {"name": _stable_name("netgroup", regex), "regex": regex} + for regex in regexes + ] + labels = [ + { + "kind": kind, + "field": "hostname", + "output": "nameLabels", + "patterns": [ + *(STATIC_NAME_PATTERNS if kind != "MREG::HostPolicyRole" else ()), + *patterns, + ], + } + for kind in LABEL_RESOURCE_KINDS + ] + + rules: list[str] = [ + "// Generated from mreg-cli permission data. Do not edit by hand.\n", + ] + rule_ids: list[str] = [] + + by_group_regex = _group_values(permissions, "group", "regex") + for (group, regex), networks_set in sorted(by_group_regex.items()): + networks = _normalized_networks(networks_set) + rules.append(_permit_ip_rule(group, regex, networks)) + rules.append(_permit_hostname_rule(group, regex)) + rule_ids.extend( + ( + _stable_name("netgroup_ip", group, regex), + _stable_name("netgroup_hostname", group, regex), + ) + ) + + by_group = _group_values(permissions, "group") + for (group,), networks_set in sorted(by_group.items()): + networks = _normalized_networks(networks_set) + rules.append(_permit_network_rule(group, networks)) + rule_ids.append(_stable_name("netgroup_network", group)) + + role_networks: dict[tuple[str, str, str], set[str]] = {} + used_legacy_labels: set[str] = set() + for permission in permissions: + permission_labels = set(permission.labels) + if not permission_labels: + continue + for role in roles: + shared = permission_labels.intersection(role.labels) + if not shared: + continue + used_legacy_labels.update(shared) + key = (permission.group, permission.regex, role.name) + role_networks.setdefault(key, set()).add(permission.network) + + for (group, regex, role_name), networks_set in sorted(role_networks.items()): + rules.append(_permit_role_rule(group, regex, role_name, _normalized_networks(networks_set))) + rule_ids.append(_stable_name("hostpolicy_role", group, regex, role_name)) + + permission_labels = {label for permission in permissions for label in permission.labels} + role_labels = {label for role in roles for label in role.labels} + report_data = { + "permission_rows": len(permissions), + "role_rows": len(roles), + "unique_regexes": len(regexes), + "generated_rules": len(rule_ids), + "generated_role_rules": len(role_networks), + "unused_permission_labels": sorted(permission_labels - used_legacy_labels), + "unmatched_role_labels": sorted(role_labels - permission_labels), + "derived_labels": {regex: _stable_name("netgroup", regex) for regex in regexes}, + "policy_ids": sorted(rule_ids), + } + return GeneratedPolicy( + labels=json.dumps(labels, indent=2, ensure_ascii=False) + "\n", + cedar="\n".join(rules).rstrip() + "\n", + report=json.dumps(report_data, indent=2, ensure_ascii=False, sort_keys=True) + "\n", + ) + + +def _outputs(output_dir: Path, policy: GeneratedPolicy) -> dict[Path, str]: + return { + output_dir / "labels.json": policy.labels, + output_dir / "netgroup.cedar": policy.cedar, + output_dir / "netgroup-conversion-report.json": policy.report, + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--permissions", type=Path, default=DEFAULT_PERMISSIONS) + parser.add_argument("--roles", type=Path, default=DEFAULT_ROLES) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) + parser.add_argument("--check", action="store_true", help="fail if generated output differs") + args = parser.parse_args(argv) + + try: + permissions = parse_permissions(args.permissions.read_text()) + roles = parse_roles(args.roles.read_text()) + generated = generate_policy(permissions, roles) + except (ConversionError, OSError) as exc: + print(f"Unable to generate TreeTop policy: {exc}", file=sys.stderr) + return 2 + + outputs = _outputs(args.output_dir, generated) + if args.check: + stale = [path for path, content in outputs.items() if not path.exists() or path.read_text() != content] + if stale: + print("Generated TreeTop policy is stale: " + ", ".join(str(path) for path in stale), file=sys.stderr) + return 1 + print("Generated TreeTop permission policy matches the mreg-cli fixtures") + return 0 + + args.output_dir.mkdir(parents=True, exist_ok=True) + for path, content in outputs.items(): + path.write_text(content) + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/treetop/data/labels.json b/treetop/data/labels.json index d479858b..babf094d 100644 --- a/treetop/data/labels.json +++ b/treetop/data/labels.json @@ -4,12 +4,26 @@ "field": "hostname", "output": "nameLabels", "patterns": [ - {"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}, - {"name": "netgroup_host_example_org", "regex": "^ho.*\\.example\\.org$"}, - {"name": "safelabel", "regex": ".*\\.example\\.org$"}, - {"name": "webserver", "regex": "^web-\\d+"}, - {"name": "admin_subdomain", "regex": "^admin\\."}, - {"name": "staging_environment", "regex": "^staging\\."} + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } ] }, { @@ -17,8 +31,26 @@ "field": "hostname", "output": "nameLabels", "patterns": [ - {"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}, - {"name": "netgroup_host_example_org", "regex": "^ho.*\\.example\\.org$"} + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } ] }, { @@ -26,68 +58,311 @@ "field": "hostname", "output": "nameLabels", "patterns": [ - {"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}, - {"name": "netgroup_host_example_org", "regex": "^ho.*\\.example\\.org$"} + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } ] }, { "kind": "MREG::Hinfo", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] }, { "kind": "MREG::Loc", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] }, { "kind": "MREG::Mx", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] }, { "kind": "MREG::Naptr", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] }, { "kind": "MREG::NameServer", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] }, { "kind": "MREG::PtrOverride", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] }, { "kind": "MREG::Sshfp", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] }, { "kind": "MREG::Srv", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] }, { "kind": "MREG::Txt", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] }, { "kind": "MREG::BACnetID", "field": "hostname", "output": "nameLabels", - "patterns": [{"name": "netgroup_example_org", "regex": ".*\\.example\\.org$"}] + "patterns": [ + { + "name": "dns_wildcard", + "regex": "\\*" + }, + { + "name": "dns_wildcard_valid_depth", + "regex": "^(?:[^.]+\\.){3,}[^.]+$" + }, + { + "name": "dns_underscore", + "regex": "_" + }, + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] + }, + { + "kind": "MREG::HostPolicyRole", + "field": "hostname", + "output": "nameLabels", + "patterns": [ + { + "name": "netgroup_5ffe5b162fe5", + "regex": ".*\\.example\\.org$" + }, + { + "name": "netgroup_6eb5380aba41", + "regex": "^web-\\d+" + } + ] } ] diff --git a/treetop/data/mreg-bundle.tar.gz b/treetop/data/mreg-bundle.tar.gz index 437855c84c3b3bda07dea9d867bf0bda7a094ae6..45d5573d51cfc5f0a2b951b8026453fa7baaa782 100644 GIT binary patch literal 5253 zcmV;06ng6)iwFP!00000|LtAtlH10y-rxHaFe{akk`+bpPP$y@$g+;2Dl2ly^7#=< zVc{~|MN9$+0Pe+`efRVXuDs5G%N3c-2UUp+pl7-<-CuW40~pLL-Tlod6z8(zW~*zq&st|+VOzwB6&c2vrrUkPiV|;XIZ50Y|9amDt>mgP zZiKGNj^3&8kfvxk(C5dV|?F8H8yT^ zn`XPohdYTFOM-Bc%0XvZ%~rElIpG9RezxG71*qGJ)8cIMKq*kN7OP38LR7>t|NF*m5qop@E((LYJ;( z+u1pr+oYQ-FRW5z1GC<_=U-bh&z%I+uOCLsV1iyGB47C00Uz zn@A#Z?2+g2aWADwMOc?|d2BXcMRkM}u5+EFdR04#$}8JSmRUV>x#+26UN{-F$p>|u z4jcBmJ1DuV+?cQj> z%t2p)d@wW}%jyq0LraC+ZgpL=-L)8VoOSYwO+*2C!O^hScdfS5?v7kJb=w} z%Ow95-0=S=DJiwdsOAy7oCwm?FO@Q7y+4#faM~>nbj@dML&Ppus|YG3fF4R=^^9Rj zy@$hJT~s}t=TP?q>V9rdP}g)Bn|y#;M3qOIT~T#+RNfkM6#`YAu)IC1kj2!s{O1q> ze&2LyE&1Bw#V(Yb;FpK2$o@CoS|L=8so&MVVzt=G_|Njcx%a=wr4SbQ# z`o;!#2HzTdk~}sz_EXb1CrWcPpWJXw!IOIafv)1qtK+s3eQ& zuh0t3vyTnjkxpMx4gqo#QIYZ8jPIng1mQ3f3fJ!m@#W=c)nqbF2pWXLFiJ9 z@4gcp#^YCFe>`q%POL;5;v6g@hn+wFw#vQm>`ELxU+nSi*A>>b{p$cW5I3&Tm4!uZWRp=!&WIL`)Mkl>2QQndg1&)klek-F8hDXT*L03*1 zCUPQH>f@5#P6UABSi$`Ae2d*WvxUnOB+I{Ks;vsmQch(rlKtCpZ&tUpBDI8!k8VJE zh&)#+K*W7*%<4jXshNZ}j;^I7@zqocyyP~+#h0BO7TBZ!T;Zv)pUdV z>}EBkC?|La)Xk3IpT;q|PCl{S9X9gm?`|p%b2(KJIn`JfXb3G~3l2ppNB$d0zVxlE?kp-#TO%|l8;b(Qary7S=+o=V` z^>%XaDkfKQ|0>Aj9#)XaUC{Pq=I(tSe*La(`*lLcsnq zr|tKbJZ=A<;`WuCypqS+c<=GhQ&ZWzEfqlRs#03DaFx<3jcH0+-gnG7eAzZ@{tGn{ zqp)>Amu|N1Sz+0B$vfrQT4u>^E3oKY{DBL%RBYLBa=iza#ZKgfT)JO`JWE}yon0y0 z?6_6^i1FE`y5o+NxWe~~L{tiS^KXjwOQ$vr(fZx^`hYC^*&CJB^1T{1t1Z+JT5X}` z`>HnyYDpzp)RIcHP?DVudAaV>6^rSv9)Ajz&V@6J?2qaYWE*tP zFL{x6Ot|$uYZh~R$?g(z>$_$K-O5}l+$lZH@%)_bUfG_z@143s7V8dMr#fr_>Ts2% zLlunpcA3tj>gFSCk zMxAf+IQ$FglV2Wxys44Xee+BqK8Yq-R4O<;`$k=^N$RIu7A9BJ!@vHO4w({j>gPq? zPcFmr)muzE9x#Ldga3p7J4LTm1?kuGTgpzS6ff^$m)BW-X7!Yu&ePnTe7w=C8<@XnJ=&Oiv}y0rrn3*98p?0CTISfbhvV@}vpuZa#@we- zd!I&~tkJ2t`3-kpQYTMy^*6U6e}+6U%GrJ9xcS_jhcV_F99>-DcV{MRobBrq9&q<$onlnGyO;4B4Q#rRzgFKk4Ho3e} zCbgr$qUj7T9F;BVRsJ8PtGbV-tuj|JtzGXTJxPIdv)D@_sK~whMZx-#Z?$TQn3NIhl7_h~Bes_o`AeGYl9d{3R6K#L|aVNE`?;X<%_1VQ% zO6iqtu7RkPKsHjzIBu+ysFX=I(8zdgs-!3sQr1aijJA|o)G93-spK4Ylw;~tV|LKW zWY|52tyQ<#MJy9zON~dZ#Iun~%5hU2T%`=Yfkwt_W6f!M$!RBzjMt7Q4YyLt_^v%? z_;f|FA{Of6o1G3#lG|NA|5~YFDX#qopVrD3zTy}zL#MxYMH}Uc$ zP)b{!{i$2t(n7OO>W1~Uyp>S7RY5yfbbh?+-ubM`uGh*|VqMYCUx@rC(flK|rv7** zX5+u(*3{0$xF7ANp$nAk6iaqHS8ji_*M53KB*u;ZyqU~~hs4r_y7R}xvSqu=hs4Cb zk-bd&8w(Sz11}M;c^m2HVvMva!)A`Ph-ucqX%fB`Zs1$9*_~m9;ml((@6U3gAkLTY zExx57R_skxmp$sMvV8bfEBS8YW6)ItSq+7`k#1Sey#Dik)yJrJGd?@{t@72Sv!~j2 zuK(R>b^E5dw*Ggk+r#?bUtzmXzuYvwZ=^OHi$AMRqw@U};0Ns;z`=7m1~8(q`?pKf{Y-@kfIj`!UUyz@NqE=0)o->2(+aemxsb0%pP zT>kY_{!N#q{a5)CJ6liuKVNz$FLVwI(8vPTs>aa1mfFt> z3(&rn+80Y}L-$(hKASB=`x8>L*ubt)*?RC{WBE6n^N0ir4^Q$+t$I!lx+DEk4 zS^J3i`syB0UqkJSSU(XTnSthY(tH}{p?M88zhikR7=mVouz8U*n1M!Sz}I&}9MDhi z)0BDov2GN7G&caZ{Kz|O0k)9a0?L7~h1?cU9E2sLmViIeF@RzH$Gjg^mjkEZGm z1^L9H4iNug1ao8oFoHR90hjGUKl&Li~09IEDkd;sQ9F$W|G zFowbykU3~+4qQU4@}#{5AW_s;_s9zM)ji8ZoJ3qkxB-|#ZVK=UFonz%Fi(Ub2t&Xe zQD0LCMT?1*Uk!2v7(;0c$QED>g)tysfGK3AfQ$jgP#D8CzZ~QZFo{)@K=uHWSTPCY z5HN@(gFq&sy-6T*(B1@)DQIs3$P6?$0AvFCngGDBtLA}v{dCXYXbQ4%MfR_|0U-C+ z+yD^#wN^he0R0RgO1qu>PW>VS(A@ysqGQ+sY@x6PB!;ks+!l}{!V*$TK)MK9Kv>Y( z01ykbRzCuOeg+Un*O(e=s&~YDJ@t+-ucOux-8IxZA4tS^Z8eVwudC(}HgYdHm&H!xg`EE{MU)WQ95=oX0zRk_`{%-o82ON? zHH#UAlsn#2#TV35fj_Nt?DU`uLfc2^FdL;<=A(A6D9X!b)qk|IPCsDZ%5% zfVUJzEtEtiGAGq;>T+uj+j#xVl1BNm=i{UdZpcHtUnHsdKEiN9E4g}{D4deT35(vv zAGlPHktSxd>8qk3vR^L`#hH)H2_jbuzz8q`j7J7zkyZeZ03-l;bU;=UX+RQ? z1SC%hl9hQXzzJ{yoJR*|UY`RT0Y|{`_;3_u;_>}MUIqs)flJ`>=(rR&uYg5h5m-D4 zEUw+Y1vY_AVDm(pH>2p03-l;0)VVMp9fR|l|bbQp;9~~3seG? zK;_XG#mbE_sPrwuKJOy~>gAOPP@Wbj%Ox^_PM{O$JW+I}>(m0GfG8k(bcj}0wFV>sNkH=GkgP7d4oCu$ zfaEDb5^G8VoB-!3fs=oN1yBSO0mb7(QG68%U<4Qe#-oFgf0PVR1QY?qqeD@AuaLT& zv$@Tp57SR~`S%LNN?H6vcn{}i$JxX;7vttX;=n(9s;xy^zuOhRw5|M_X3Om4$Fk#Q ztJUeBwa&i6wuloeGK@1#w_ooYQ;*HauuC_d`@+Wik8j?N$Jw_ugGCZ9$Qf0glz=dy z&*$b`V8-?1$&EL29g6_Kv2)38>G0>ze53se*pTL* z$HkByY)I2D4P!tLHlX68%^1&vjc5M-YYgcDhqUr3ag6K1#&!9t>=@XC4Q&2_dkpE} zhLmnpVL%T!puEC~0X^V=e*fwRE}en`7Bm9g)mS3i*7JUO{aIso6X~z%anDaV z5iR}QGG1RjjuyGtsOxa;xb%f)17v0e$tJS&5In0{Yfx@8@s(8aCq1cI(ZToBC`4=f z6r5`)P=^e^L*N6(<8m4==;4iI!+X#GUk0T)8XX3qn3$*4m?AOa(kG< zc|Ob1WB!9JSiwB=Ovz_JT-=@?g{#rJ@|H|ap=Enw_w`UE_j}7)!zv+bh2oE6nBplH zHNW58>oumZI*b*0Qhz0O<>HCDJTaNh7!_yoT7DBUa&YE?n8--KCkm){TR1>G6*uQO z_&j2Wuy8PbwM*J2Bq(?oC2r}rWM}T|LT@&>o^C@z2PG?Q{9cCK`=`KTS?w^&LWP z*_^$+Jc}isQ=slY6)#w+_)qKzIJZj`U@loU&Q~NX;`Ll43@=4{`+QxfY)(DH!cRVI z?!dql4DYdd@#o=SR9o;Jj)2)Fht%TYRBbrMZ=(qIZkeON5jL#yRj%L?l~T z(QNrjsKTEF4V`5sCvfW>LH6_|`^WFl^3C11>a3j_U{qAX8&ABmitx>Bt9->vlX9c! z4$*dHW4PB(c&bl1PJtIGU1f;9mU;D}T7cV2yx3U!=6J8M*!_#UO2I1YOvH z5JAc73uKG=uWn_rUuaKlsw?V!V#*lEE$e6~yLEq6*^fS>INytrkyen%WR>HP!KWTV zJv3KSres@GWI9_#*Fw&tHkRk{BBhAhqFrb>!BndCedUbJ_%RslA`tGk+!a8XvGM5L z`SY!2*rP%VRqtbrLz`=EpW*5m^FD<&dj5(vog}q2dI|y0Gb!c8$tGj>%B{fkRXb#( zxA}{{T=qjeZVujY#77Qtg_qAKs)U`WD*xQ+<5=mE&orGLdc>_kSZ*RvmYJc}Fj>+G z&Z8A8r!lkitpQc3*E-x-H{?zulHhgpaXe;hH~gV@8Y4ZkYio~M+qHO=Tf4F83ARD= zwI8FNSzSdNy4iTMDU?D)WAg4#0@w@%yrJX$#$N3i@s=efh&774ligd-J!t<7%Ord= zjo(|7#eE1jj|W~w6lbbaB9M$hfAp158oN#;)}0L5o%fLV=u0X1v3=?V}k`IJuVPD2zK^1-&sJ16F0Q)UyErKgE6t zWEs@jsvr|9@rmkD@l>9}UsbXA1=;q|LMM6T99&pwluf8|1?SBI&SLIJ1&enX>|7|M)Zw*>7TYt*r8{5MomuqK zJht0doW?|1v)PC$RB1u+EI2W}EI84Jglht_L4^h|TpEKDcUEO-Trs4A?& zWyQ)ZxkxHd-pQ>VpYJE&E$;G;yt=%tLt|>Zozm4-!AV9l#H!1jWo8Ax>t-<2zVOYu zIcap5QM%opK*tKCh?5fai{lFnvdH87Kg%{$e$>)VD0HqT(Uw-%9&h5-((csgXwa*^ zYq&AdMcMKWE8CSDC)YGLObQD-W#ylb^2SU(YFczYA4D2u)}OiF9;JV}SXhUaYblTn z&N`{b*;KrX60QByTo8Cxqj#o|C^;))8T9-m2SxO^M8V(UUeY0&>z2TyPMi;*El>cv zwO!@5!SC-U>oljV zZ0uKPYdy&uhwe_i+8fdol&XTZRi-yO?ijj0X$=xqo7|i>OVJ0Z^Dr=PoE+!JeK(nhdzWL+h%w#wwJN0RK+|Z5bZUR+e5m}{OGd8b|%*%riZYnn5lOhHk5_m&^{yJ~8Izfn1DAa$aCW`A zxxPM4#DTXqMlN4tha;I}{X2pkC?XLtZzTH7m&spB6Q_@vAh`d^Tz8mqU5G>W~KahkFoDVin?Tjr^rDlrsQ;ZV(M(=DGfNq8+)gJ;tjru@Fb zu5kv%$kt{ObO5ZoxybUcJOm%5X9`6Ya)DdY=1S=DcHuKb$deE$nq)JE5~sZgf`TL*^%sWc~`==M}ZkzeM8*vr`vqkgkMTBHrHDkcBc_! zKv5$sdYS4atKBS{`a@L9a1hdOwx~Ut+GKL~2Q2@_-F`XT%3~DS+euX`+Rr(3fL9=-ei>6`L#uSp8ttPAp=8{ZdOB#52vsMYahjx95K5!|rv z&;$jNR_AmIwbISqQrsK|>~0Qs@;y>Uwb7umMIMb87jkOH^T2EmH|pn)DHL(HF`W8J zC09rF0|IG@GHRjYD-t+8qHad1`o!@^tfY@Oi3hQo{61-IM(XYx$Llk#O>K>*!GYDA z8P{8~Z{>tj9y#ZEP>;Iidk3FepkQ60h|8t0Lby2)s-%+_W)0!U42u#dBt+>YPmCWQ z#~|&UpJFfv`X-5Sf9r^!jbYL@^kS^wY@GE!Osew|aK%Bfn%s9EFtAl+Zmk`v)e8u; zr@Mrts89SJmRbIHE7-oO=3?!1re3hE!+$oV!<+6+qyLs#jpyc$p&O9B(-2DHn?s5} z%ZY`7OYD#KrUKtvSg;k+(H{b%=|eLxqj7uk+&2gVhx8f7a}bRW2B9woE(to01CI0) zK!>p9Sr1wXrf+u#*S9u!a>%&)%x{NFt9dFtJ=kz-Mn?@UGXNC&1Q0|D!ome$U}K1a zhD36&PZel|H8E&86sgd#FsJK;O7!J=0!~SI@5IjKIO;r-Mq_5H>i!TMKy5rDub8d8 z6~YxrO#zQ!{9mEy0Cb9K0IhGRur2=e`q^M8764ro1Aw*+M#I1-C;_3@c`=SgUse{a zaenGt#l~e0^E)Wdp?xCe7mJJShX+8P!-)+DpnYmU7lKX3flefCoOj@K0r1nJZE6a> zn5pj|f?{Kp;Gl7ZmY5KcDw01r#}daPW{&j##h@XYQ+HAAE^LZ2-dEIJFea9Sp)eB+ z)*O9a$byDhP5v~OJneNK(xP_!QkEe% z_UV|eTy0MytEEem3oz>0c;%P5Ad;7G-DEJNs_A|WHPaxJ*)cR5a-}}`po%=X{i{l? zuQuF+EVo3s{yp1xZN=i=Wv|R}&&88xZv3>=07|%na4g_efr0R54g+Oi@UJZ?Hw;=u zRpezT=cBg6%~_UBl<8PP5WoB%F)$ zW(XEa%@Jg?*hD#_tme0@5&8<-H5>*A(*yb79eK$MLhTcda?u+q9WG8d+90}v5 zy<6U0RcT7>EG0^)rLsgd`_bC)PbR;~k8L!j6gVqi-WB(YDa^1SqusLt}Of{z@^bgtXvLV&!P^awWw;R2NMQy)X zx**KN#2^qb$X}$fXo4pYGs<5amY<_EVey1~6wK=@2`ivg#)pPFzo8qA>hxEJmFB1_ zvM|@96~3$uqE(7zmLz5X5!#C6!ArZYk3MKhqwxVV{N=ie41PYfVM(d*gLUN?D86DQ zt$Ptrwq|bDgH8ub>%-bAO5$baIkD>#kKgKc8Up$GK&8Lq6p#S^%I8W&?@3v(4%|#JbA$I z@fYgKULAjV_s6gBy?KB{SN^Kf3A1Y*K}mcz3(jEI05u;P&v)1YIg!FolX=bGNNIfoR3r0ca1B8dyId*CaRWq{Qu8t z;zZnxtUFnarAf$BhCTdsJ{C?jJQ+eOnHP?HlQG$qs6c~OWjJ^ zmg_sD{^;W%4%g3yOsQ|F>6N2wOq?9{O3Oes5f>P)z}i^SR%`TmNHfE!ZQ8C5LwSwF zWp12Mh?14ybouqV{hb^ydWqy!v5SNxWUsABOY6szOgf zkXTQ$!bg&He|N(0P{&ZF+<|}p?B!!UA%DJ|UC8QpzbE}4VjclP!YpV27IeVB0OEbZ A7XSbN diff --git a/treetop/data/mreg.cedar b/treetop/data/mreg.cedar index f795e755..a92fd2ba 100644 --- a/treetop/data/mreg.cedar +++ b/treetop/data/mreg.cedar @@ -205,195 +205,6 @@ permit ( resource ); -// Static bundle representation of the testgroup NetGroupRegexPermission rows. -// The regex portion is a TreeTop-derived label; MREG sends only hostname/IP facts. -@id("MREG.testgroup_netgroup") -permit ( - principal in MREG::Group::"testgroup", - action in - [MREG::Action::"host_create", - MREG::Action::"host_update", - MREG::Action::"host_delete", - MREG::Action::"host_contacts_create", - MREG::Action::"host_contacts_delete", - MREG::Action::"ipaddress_create", - MREG::Action::"ipaddress_update", - MREG::Action::"ipaddress_delete", - MREG::Action::"hinfo_create", - MREG::Action::"hinfo_update", - MREG::Action::"hinfo_delete", - MREG::Action::"loc_create", - MREG::Action::"loc_update", - MREG::Action::"loc_delete", - MREG::Action::"mx_create", - MREG::Action::"mx_update", - MREG::Action::"mx_delete", - MREG::Action::"naptr_create", - MREG::Action::"naptr_update", - MREG::Action::"naptr_delete", - MREG::Action::"name_server_create", - MREG::Action::"name_server_update", - MREG::Action::"name_server_delete", - MREG::Action::"ptr_override_create", - MREG::Action::"ptr_override_update", - MREG::Action::"ptr_override_delete", - MREG::Action::"sshfp_create", - MREG::Action::"sshfp_update", - MREG::Action::"sshfp_delete", - MREG::Action::"srv_create", - MREG::Action::"srv_update", - MREG::Action::"srv_delete", - MREG::Action::"txt_create", - MREG::Action::"txt_update", - MREG::Action::"txt_delete", - MREG::Action::"bacnet_id_create", - MREG::Action::"bacnet_id_update", - MREG::Action::"bacnet_id_delete"], - resource -) -when { - resource has nameLabels && - resource.nameLabels.contains("netgroup_example_org") && - resource has ip && - (resource.ip.isInRange(ip("10.0.0.0/24")) || - resource.ip.isInRange(ip("10.1.0.0/25")) || - resource.ip.isInRange(ip("192.168.1.0/24")) || - resource.ip.isInRange(ip("192.168.2.1/32")) || - resource.ip.isInRange(ip("2001:db8::/64")) || - resource.ip.isInRange(ip("2002:db9::/64"))) -}; - -@id("MREG.testgroup_cname_netgroup") -permit ( - principal in MREG::Group::"testgroup", - action in - [MREG::Action::"cname_create", - MREG::Action::"cname_update", - MREG::Action::"cname_delete"], - resource is MREG::Cname -) -when { - resource has nameLabels && - resource.nameLabels.contains("netgroup_host_example_org") -}; - -// Network administrators need an explicit NetGroup mapping for host/DNS CRUD, -// just like the legacy database permission path. -@id("MREG.network_admin_netgroup") -permit ( - principal in MREG::Group::"default-networkadmin-group", - action in - [MREG::Action::"host_create", - MREG::Action::"host_update", - MREG::Action::"host_delete", - MREG::Action::"ipaddress_create", - MREG::Action::"ipaddress_update", - MREG::Action::"ipaddress_delete", - MREG::Action::"hinfo_create", - MREG::Action::"hinfo_update", - MREG::Action::"hinfo_delete", - MREG::Action::"loc_create", - MREG::Action::"loc_update", - MREG::Action::"loc_delete", - MREG::Action::"mx_create", - MREG::Action::"mx_update", - MREG::Action::"mx_delete", - MREG::Action::"naptr_create", - MREG::Action::"naptr_update", - MREG::Action::"naptr_delete", - MREG::Action::"name_server_create", - MREG::Action::"name_server_update", - MREG::Action::"name_server_delete", - MREG::Action::"ptr_override_create", - MREG::Action::"ptr_override_update", - MREG::Action::"ptr_override_delete", - MREG::Action::"sshfp_create", - MREG::Action::"sshfp_update", - MREG::Action::"sshfp_delete", - MREG::Action::"srv_create", - MREG::Action::"srv_update", - MREG::Action::"srv_delete", - MREG::Action::"txt_create", - MREG::Action::"txt_update", - MREG::Action::"txt_delete", - MREG::Action::"bacnet_id_create", - MREG::Action::"bacnet_id_update", - MREG::Action::"bacnet_id_delete"], - resource -) -when { - resource has nameLabels && - resource.nameLabels.contains("netgroup_example_org") && - resource has ip && - (resource.ip.isInRange(ip("10.0.0.0/24")) || - resource.ip.isInRange(ip("10.1.0.0/25")) || - resource.ip.isInRange(ip("192.168.1.0/24")) || - resource.ip.isInRange(ip("192.168.2.1/32")) || - resource.ip.isInRange(ip("2001:db8::/64")) || - resource.ip.isInRange(ip("2002:db9::/64"))) -}; - -@id("MREG.network_admin_cname_netgroup") -permit ( - principal in MREG::Group::"default-networkadmin-group", - action in - [MREG::Action::"cname_create", - MREG::Action::"cname_update", - MREG::Action::"cname_delete"], - resource is MREG::Cname -) -when { - resource has nameLabels && - resource.nameLabels.contains("netgroup_host_example_org") -}; - -@id("MREG.testgroup_community_network") -permit ( - principal in MREG::Group::"testgroup", - action in - [MREG::Action::"community_create", - MREG::Action::"community_update", - MREG::Action::"community_delete", - MREG::Action::"host_create", - MREG::Action::"host_delete"], - resource -) -when { - resource has network && - (resource.network == "10.0.0.0/24" || - resource.network == "10.1.0.0/25" || - resource.network == "192.168.1.0/24") -}; - -@id("MREG.dummygroup_hostpolicy_role_host") -permit ( - principal in MREG::Group::"dummygroup", - action == MREG::Action::"hostpolicy_role_host_membership_update", - resource is MREG::Host -) -when { - resource has nameLabels && - resource has roleLabel && - resource.nameLabels.contains(resource.roleLabel) && - resource has ip && resource.ip.isInRange(ip("11.22.33.0/24")) -}; - -@id("MREG.webadmins") -permit ( - principal in MREG::Group::"webadmins", - action in - [MREG::Action::"host_create", - MREG::Action::"host_update", - MREG::Action::"host_delete"], - resource is MREG::Host -) -when { - resource has nameLabels && - resource has ip && - resource.nameLabels.contains("webserver") && - resource.ip.isInRange(ip("192.168.1.0/24")) -}; - // These rules replace local post-policy denials in authoritative mode. @id("MREG.invalid_or_unprivileged_dns_wildcard") forbid ( @@ -424,10 +235,11 @@ forbid ( resource ) when { - resource has dnsWildcard && resource.dnsWildcard && + resource has nameLabels && + resource.nameLabels.contains("dns_wildcard") && principal != MREG::User::"super" && !(principal in MREG::Group::"default-super-group") && - (resource has dnsWildcardValidDepth && !resource.dnsWildcardValidDepth || + (!resource.nameLabels.contains("dns_wildcard_valid_depth") || !(principal in MREG::Group::"default-dns-wildcard-group")) }; @@ -458,7 +270,8 @@ forbid ( resource ) when { - resource has dnsUnderscore && resource.dnsUnderscore && + resource has nameLabels && + resource.nameLabels.contains("dns_underscore") && principal != MREG::User::"super" && !(principal in MREG::Group::"default-super-group") && !(principal in MREG::Group::"default-dns-underscore-group") @@ -477,7 +290,26 @@ forbid ( resource ) when { - resource has ipRestricted && resource.ipRestricted && + resource has ip && + (resource.ip.isInRange(ip("10.0.0.0/32")) || + resource.ip.isInRange(ip("10.0.0.1/32")) || + resource.ip.isInRange(ip("10.0.0.2/32")) || + resource.ip.isInRange(ip("10.0.0.3/32")) || + resource.ip.isInRange(ip("10.0.0.255/32")) || + resource.ip.isInRange(ip("10.1.0.0/32")) || + resource.ip.isInRange(ip("10.1.0.1/32")) || + resource.ip.isInRange(ip("10.1.0.2/32")) || + resource.ip.isInRange(ip("10.1.0.3/32")) || + resource.ip.isInRange(ip("10.1.0.127/32")) || + resource.ip.isInRange(ip("192.168.1.0/32")) || + resource.ip.isInRange(ip("192.168.1.1/32")) || + resource.ip.isInRange(ip("192.168.1.2/32")) || + resource.ip.isInRange(ip("192.168.1.3/32")) || + resource.ip.isInRange(ip("192.168.1.255/32")) || + resource.ip.isInRange(ip("2001:db8::/128")) || + resource.ip.isInRange(ip("2001:db8::1/128")) || + resource.ip.isInRange(ip("2001:db8::2/128")) || + resource.ip.isInRange(ip("2001:db8::3/128"))) && principal != MREG::User::"super" && !(principal in MREG::Group::"default-super-group") && !(principal in MREG::Group::"default-networkadmin-group") diff --git a/treetop/data/mreg.cedarschema b/treetop/data/mreg.cedarschema index b6db6422..adfb0628 100644 --- a/treetop/data/mreg.cedarschema +++ b/treetop/data/mreg.cedarschema @@ -10,16 +10,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Host = { @@ -30,16 +24,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity HostContact = { @@ -50,16 +38,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Ipaddress = { @@ -70,16 +52,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Cname = { @@ -90,16 +66,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Hinfo = { @@ -110,16 +80,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Loc = { @@ -130,16 +94,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Mx = { @@ -150,16 +108,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Naptr = { @@ -170,16 +122,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity NameServer = { @@ -190,16 +136,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity PtrOverride = { @@ -210,16 +150,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Sshfp = { @@ -230,16 +164,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Srv = { @@ -250,16 +178,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Txt = { @@ -270,16 +192,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity BACnetID = { @@ -290,16 +206,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Community = { @@ -310,16 +220,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity HostCommunityMapping = { @@ -330,16 +234,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Label = { @@ -350,16 +248,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity Network = { @@ -370,16 +262,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity NetworkPolicy = { @@ -390,16 +276,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity NetworkPolicyAttribute = { @@ -410,16 +290,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity NetworkPolicyAttributeValue = { @@ -430,16 +304,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity HostGroup = { @@ -450,16 +318,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity NetworkExcludedRange = { @@ -470,16 +332,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity ForwardZone = { @@ -490,16 +346,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity ForwardZoneDelegation = { @@ -510,16 +360,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity ReverseZone = { @@ -530,16 +374,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity ReverseZoneDelegation = { @@ -550,16 +388,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity HostPolicyAtom = { @@ -570,16 +402,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity HostPolicyRole = { @@ -590,16 +416,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; entity NetGroupRegexPermission = { @@ -610,16 +430,10 @@ namespace MREG { hostname?: String, ip?: ipaddr, nameLabels?: Set, - dnsWildcard?: Bool, - dnsWildcardValidDepth?: Bool, - dnsUnderscore?: Bool, - ipReserved?: Bool, - ipRestricted?: Bool, selfAccess?: Bool, requesterIsOwner?: Bool, ownerMutation?: Bool, descriptionUpdate?: Bool, - roleLabel?: String, network?: String, }; diff --git a/treetop/data/netgroup-conversion-report.json b/treetop/data/netgroup-conversion-report.json new file mode 100644 index 00000000..bf96113b --- /dev/null +++ b/treetop/data/netgroup-conversion-report.json @@ -0,0 +1,27 @@ +{ + "derived_labels": { + ".*\\.example\\.org$": "netgroup_5ffe5b162fe5", + "^web-\\d+": "netgroup_6eb5380aba41" + }, + "generated_role_rules": 3, + "generated_rules": 12, + "permission_rows": 8, + "policy_ids": [ + "hostpolicy_role_2b9b9645d997", + "hostpolicy_role_9856da2c249d", + "hostpolicy_role_fec5919a9dae", + "netgroup_hostname_06d46be4843b", + "netgroup_hostname_39b7f3522b17", + "netgroup_hostname_72fdb3c3e854", + "netgroup_ip_06d46be4843b", + "netgroup_ip_39b7f3522b17", + "netgroup_ip_72fdb3c3e854", + "netgroup_network_781caa6738a6", + "netgroup_network_828db597e176", + "netgroup_network_b04d124aeecc" + ], + "role_rows": 2, + "unique_regexes": 2, + "unmatched_role_labels": [], + "unused_permission_labels": [] +} diff --git a/treetop/data/netgroup.cedar b/treetop/data/netgroup.cedar new file mode 100644 index 00000000..4ec9398a --- /dev/null +++ b/treetop/data/netgroup.cedar @@ -0,0 +1,298 @@ +// Generated from mreg-cli permission data. Do not edit by hand. + +@id("MREG.generated.netgroup_ip_72fdb3c3e854") +permit ( + principal in MREG::Group::"dummygroup", + action in + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"host_delete", + MREG::Action::"host_contacts_create", + MREG::Action::"host_contacts_delete", + MREG::Action::"ipaddress_create", + MREG::Action::"ipaddress_update", + MREG::Action::"ipaddress_delete", + MREG::Action::"hinfo_create", + MREG::Action::"hinfo_update", + MREG::Action::"hinfo_delete", + MREG::Action::"loc_create", + MREG::Action::"loc_update", + MREG::Action::"loc_delete", + MREG::Action::"mx_create", + MREG::Action::"mx_update", + MREG::Action::"mx_delete", + MREG::Action::"naptr_create", + MREG::Action::"naptr_update", + MREG::Action::"naptr_delete", + MREG::Action::"name_server_create", + MREG::Action::"name_server_update", + MREG::Action::"name_server_delete", + MREG::Action::"ptr_override_create", + MREG::Action::"ptr_override_update", + MREG::Action::"ptr_override_delete", + MREG::Action::"sshfp_create", + MREG::Action::"sshfp_update", + MREG::Action::"sshfp_delete", + MREG::Action::"srv_create", + MREG::Action::"srv_update", + MREG::Action::"srv_delete", + MREG::Action::"txt_create", + MREG::Action::"txt_update", + MREG::Action::"txt_delete", + MREG::Action::"bacnet_id_create", + MREG::Action::"bacnet_id_update", + MREG::Action::"bacnet_id_delete"], + resource +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_5ffe5b162fe5") && + resource has ip && + (resource.ip.isInRange(ip("11.22.33.0/24"))) +}; + +@id("MREG.generated.netgroup_hostname_72fdb3c3e854") +permit ( + principal in MREG::Group::"dummygroup", + action in + [MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete"], + resource is MREG::Cname +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_5ffe5b162fe5") +}; + +@id("MREG.generated.netgroup_ip_06d46be4843b") +permit ( + principal in MREG::Group::"testgroup", + action in + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"host_delete", + MREG::Action::"host_contacts_create", + MREG::Action::"host_contacts_delete", + MREG::Action::"ipaddress_create", + MREG::Action::"ipaddress_update", + MREG::Action::"ipaddress_delete", + MREG::Action::"hinfo_create", + MREG::Action::"hinfo_update", + MREG::Action::"hinfo_delete", + MREG::Action::"loc_create", + MREG::Action::"loc_update", + MREG::Action::"loc_delete", + MREG::Action::"mx_create", + MREG::Action::"mx_update", + MREG::Action::"mx_delete", + MREG::Action::"naptr_create", + MREG::Action::"naptr_update", + MREG::Action::"naptr_delete", + MREG::Action::"name_server_create", + MREG::Action::"name_server_update", + MREG::Action::"name_server_delete", + MREG::Action::"ptr_override_create", + MREG::Action::"ptr_override_update", + MREG::Action::"ptr_override_delete", + MREG::Action::"sshfp_create", + MREG::Action::"sshfp_update", + MREG::Action::"sshfp_delete", + MREG::Action::"srv_create", + MREG::Action::"srv_update", + MREG::Action::"srv_delete", + MREG::Action::"txt_create", + MREG::Action::"txt_update", + MREG::Action::"txt_delete", + MREG::Action::"bacnet_id_create", + MREG::Action::"bacnet_id_update", + MREG::Action::"bacnet_id_delete"], + resource +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_5ffe5b162fe5") && + resource has ip && + (resource.ip.isInRange(ip("10.0.0.0/24")) || + resource.ip.isInRange(ip("10.1.0.0/25")) || + resource.ip.isInRange(ip("192.168.1.0/24")) || + resource.ip.isInRange(ip("192.168.2.1/32")) || + resource.ip.isInRange(ip("2001:db8::/64")) || + resource.ip.isInRange(ip("2002:db9::/64"))) +}; + +@id("MREG.generated.netgroup_hostname_06d46be4843b") +permit ( + principal in MREG::Group::"testgroup", + action in + [MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete"], + resource is MREG::Cname +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_5ffe5b162fe5") +}; + +@id("MREG.generated.netgroup_ip_39b7f3522b17") +permit ( + principal in MREG::Group::"webadmins", + action in + [MREG::Action::"host_create", + MREG::Action::"host_update", + MREG::Action::"host_delete", + MREG::Action::"host_contacts_create", + MREG::Action::"host_contacts_delete", + MREG::Action::"ipaddress_create", + MREG::Action::"ipaddress_update", + MREG::Action::"ipaddress_delete", + MREG::Action::"hinfo_create", + MREG::Action::"hinfo_update", + MREG::Action::"hinfo_delete", + MREG::Action::"loc_create", + MREG::Action::"loc_update", + MREG::Action::"loc_delete", + MREG::Action::"mx_create", + MREG::Action::"mx_update", + MREG::Action::"mx_delete", + MREG::Action::"naptr_create", + MREG::Action::"naptr_update", + MREG::Action::"naptr_delete", + MREG::Action::"name_server_create", + MREG::Action::"name_server_update", + MREG::Action::"name_server_delete", + MREG::Action::"ptr_override_create", + MREG::Action::"ptr_override_update", + MREG::Action::"ptr_override_delete", + MREG::Action::"sshfp_create", + MREG::Action::"sshfp_update", + MREG::Action::"sshfp_delete", + MREG::Action::"srv_create", + MREG::Action::"srv_update", + MREG::Action::"srv_delete", + MREG::Action::"txt_create", + MREG::Action::"txt_update", + MREG::Action::"txt_delete", + MREG::Action::"bacnet_id_create", + MREG::Action::"bacnet_id_update", + MREG::Action::"bacnet_id_delete"], + resource +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_6eb5380aba41") && + resource has ip && + (resource.ip.isInRange(ip("192.168.1.0/24"))) +}; + +@id("MREG.generated.netgroup_hostname_39b7f3522b17") +permit ( + principal in MREG::Group::"webadmins", + action in + [MREG::Action::"cname_create", + MREG::Action::"cname_update", + MREG::Action::"cname_delete"], + resource is MREG::Cname +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_6eb5380aba41") +}; + +@id("MREG.generated.netgroup_network_828db597e176") +permit ( + principal in MREG::Group::"dummygroup", + action in + [MREG::Action::"community_create", + MREG::Action::"community_update", + MREG::Action::"community_delete", + MREG::Action::"host_create", + MREG::Action::"host_delete"], + resource +) +when { + resource has network && + (resource.network == "11.22.33.0/24") +}; + +@id("MREG.generated.netgroup_network_781caa6738a6") +permit ( + principal in MREG::Group::"testgroup", + action in + [MREG::Action::"community_create", + MREG::Action::"community_update", + MREG::Action::"community_delete", + MREG::Action::"host_create", + MREG::Action::"host_delete"], + resource +) +when { + resource has network && + (resource.network == "10.0.0.0/24" || + resource.network == "10.1.0.0/25" || + resource.network == "192.168.1.0/24" || + resource.network == "192.168.2.1/32" || + resource.network == "2001:db8::/64" || + resource.network == "2002:db9::/64") +}; + +@id("MREG.generated.netgroup_network_b04d124aeecc") +permit ( + principal in MREG::Group::"webadmins", + action in + [MREG::Action::"community_create", + MREG::Action::"community_update", + MREG::Action::"community_delete", + MREG::Action::"host_create", + MREG::Action::"host_delete"], + resource +) +when { + resource has network && + (resource.network == "192.168.1.0/24") +}; + +@id("MREG.generated.hostpolicy_role_9856da2c249d") +permit ( + principal in MREG::Group::"dummygroup", + action == MREG::Action::"hostpolicy_role_host_membership_update", + resource == MREG::HostPolicyRole::"role1" +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_5ffe5b162fe5") && + resource has ip && + (resource.ip.isInRange(ip("11.22.33.0/24"))) +}; + +@id("MREG.generated.hostpolicy_role_2b9b9645d997") +permit ( + principal in MREG::Group::"testgroup", + action == MREG::Action::"hostpolicy_role_host_membership_update", + resource == MREG::HostPolicyRole::"role1" +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_5ffe5b162fe5") && + resource has ip && + (resource.ip.isInRange(ip("10.0.0.0/24")) || + resource.ip.isInRange(ip("10.1.0.0/25")) || + resource.ip.isInRange(ip("192.168.1.0/24")) || + resource.ip.isInRange(ip("192.168.2.1/32")) || + resource.ip.isInRange(ip("2001:db8::/64")) || + resource.ip.isInRange(ip("2002:db9::/64"))) +}; + +@id("MREG.generated.hostpolicy_role_fec5919a9dae") +permit ( + principal in MREG::Group::"webadmins", + action == MREG::Action::"hostpolicy_role_host_membership_update", + resource == MREG::HostPolicyRole::"web" +) +when { + resource has nameLabels && + resource.nameLabels.contains("netgroup_6eb5380aba41") && + resource has ip && + (resource.ip.isInRange(ip("192.168.1.0/24"))) +}; diff --git a/treetop/data/treetop-mreg-module.toml b/treetop/data/treetop-mreg-module.toml index acc920f6..e34f5b03 100644 --- a/treetop/data/treetop-mreg-module.toml +++ b/treetop/data/treetop-mreg-module.toml @@ -1,6 +1,6 @@ format_version = 1 name = "MREG" namespace = "MREG" -policies = ["mreg.cedar"] +policies = ["mreg.cedar", "netgroup.cedar"] schemas = ["mreg.cedarschema"] labels = ["labels.json"] diff --git a/treetop/fixtures/hostpolicy-roles.txt b/treetop/fixtures/hostpolicy-roles.txt new file mode 100644 index 00000000..9b3160b4 --- /dev/null +++ b/treetop/fixtures/hostpolicy-roles.txt @@ -0,0 +1,3 @@ +Name Description Labels +role1 Example generated role Safelabel +web Example web-server role Webserver diff --git a/treetop/fixtures/network-permissions.txt b/treetop/fixtures/network-permissions.txt new file mode 100644 index 00000000..e6aa3d4d --- /dev/null +++ b/treetop/fixtures/network-permissions.txt @@ -0,0 +1,9 @@ +Range Group Regex Labels +10.0.0.0/24 testgroup .*\.example\.org$ Safelabel +10.1.0.0/25 testgroup .*\.example\.org$ Safelabel +11.22.33.0/24 dummygroup .*\.example\.org$ Safelabel +192.168.1.0/24 testgroup .*\.example\.org$ Safelabel +192.168.1.0/24 webadmins ^web-\d+ Webserver +192.168.2.1/32 testgroup .*\.example\.org$ Safelabel +2001:db8::/64 testgroup .*\.example\.org$ Safelabel +2002:db9::/64 testgroup .*\.example\.org$ Safelabel diff --git a/uv.lock b/uv.lock index c0518869..5cf7e36f 100644 --- a/uv.lock +++ b/uv.lock @@ -689,6 +689,7 @@ dependencies = [ { name = "structlog" }, { name = "treetop-client" }, { name = "tzdata" }, + { name = "unittest-parametrize" }, ] [package.dev-dependencies] @@ -699,7 +700,6 @@ ci = [ { name = "tblib" }, { name = "tox-gh-actions" }, { name = "tox-uv" }, - { name = "unittest-parametrize" }, { name = "uv" }, ] dev = [ @@ -707,7 +707,6 @@ dev = [ { name = "django-silk", extra = ["formatting"] }, { name = "tblib" }, { name = "tox-uv" }, - { name = "unittest-parametrize" }, { name = "uv" }, ] django52 = [ @@ -740,6 +739,7 @@ requires-dist = [ { name = "structlog", specifier = ">=25" }, { name = "treetop-client", specifier = ">=0.0.12" }, { name = "tzdata", specifier = ">=2025.3" }, + { name = "unittest-parametrize" }, ] [package.metadata.requires-dev] @@ -750,7 +750,6 @@ ci = [ { name = "tblib", specifier = ">=3" }, { name = "tox-gh-actions" }, { name = "tox-uv", specifier = ">=1.29" }, - { name = "unittest-parametrize" }, { name = "uv", specifier = ">=0.10" }, ] dev = [ @@ -758,7 +757,6 @@ dev = [ { name = "django-silk", extras = ["formatting"], specifier = ">=5.5.0" }, { name = "tblib", specifier = ">=3" }, { name = "tox-uv", specifier = ">=1.29" }, - { name = "unittest-parametrize" }, { name = "uv", specifier = ">=0.10" }, ] django52 = [{ name = "django", specifier = ">=5.2,<5.3" }] From 2c44a8df0831c670e1dff7a0a0987e5c6e9f2ed3 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 19 Aug 2026 17:37:55 +0200 Subject: [PATCH 32/34] Make TreeTop converter importable in container tests --- mreg/policy/treetop_generator.py | 448 ++++++++++++++++++++ mreg/tests/test_treetop_policy_generator.py | 56 +-- scripts/generate-treetop-policy.py | 438 +------------------ 3 files changed, 473 insertions(+), 469 deletions(-) create mode 100644 mreg/policy/treetop_generator.py diff --git a/mreg/policy/treetop_generator.py b/mreg/policy/treetop_generator.py new file mode 100644 index 00000000..380580b3 --- /dev/null +++ b/mreg/policy/treetop_generator.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +"""Generate TreeTop policy data from deterministic mreg-cli table output.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import ipaddress +import json +from pathlib import Path +import re +import sys +from typing import Iterable, Sequence + + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_PERMISSIONS = ROOT / "treetop/fixtures/network-permissions.txt" +DEFAULT_ROLES = ROOT / "treetop/fixtures/hostpolicy-roles.txt" +DEFAULT_OUTPUT_DIR = ROOT / "treetop/data" + +PERMISSION_HEADERS = ("Range", "Group", "Regex", "Labels") +ROLE_HEADERS = ("Name", "Description", "Labels") + +LABEL_RESOURCE_KINDS = ( + "MREG::Host", + "MREG::Ipaddress", + "MREG::Cname", + "MREG::Hinfo", + "MREG::Loc", + "MREG::Mx", + "MREG::Naptr", + "MREG::NameServer", + "MREG::PtrOverride", + "MREG::Sshfp", + "MREG::Srv", + "MREG::Txt", + "MREG::BACnetID", + "MREG::HostPolicyRole", +) + +STATIC_NAME_PATTERNS = ( + {"name": "dns_wildcard", "regex": r"\*"}, + {"name": "dns_wildcard_valid_depth", "regex": r"^(?:[^.]+\.){3,}[^.]+$"}, + {"name": "dns_underscore", "regex": "_"}, +) + +IP_SCOPED_ACTIONS = ( + "host_create", + "host_update", + "host_delete", + "host_contacts_create", + "host_contacts_delete", + "ipaddress_create", + "ipaddress_update", + "ipaddress_delete", + "hinfo_create", + "hinfo_update", + "hinfo_delete", + "loc_create", + "loc_update", + "loc_delete", + "mx_create", + "mx_update", + "mx_delete", + "naptr_create", + "naptr_update", + "naptr_delete", + "name_server_create", + "name_server_update", + "name_server_delete", + "ptr_override_create", + "ptr_override_update", + "ptr_override_delete", + "sshfp_create", + "sshfp_update", + "sshfp_delete", + "srv_create", + "srv_update", + "srv_delete", + "txt_create", + "txt_update", + "txt_delete", + "bacnet_id_create", + "bacnet_id_update", + "bacnet_id_delete", +) + +HOSTNAME_SCOPED_ACTIONS = ( + "cname_create", + "cname_update", + "cname_delete", +) + +NETWORK_SCOPED_ACTIONS = ( + "community_create", + "community_update", + "community_delete", + "host_create", + "host_delete", +) + +ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +class ConversionError(ValueError): + """Raised when mreg-cli output cannot be converted safely.""" + + +@dataclass(frozen=True, order=True) +class NetworkPermission: + network: str + group: str + regex: str + labels: tuple[str, ...] + + +@dataclass(frozen=True, order=True) +class HostPolicyRole: + name: str + labels: tuple[str, ...] + + +@dataclass(frozen=True) +class GeneratedPolicy: + labels: str + cedar: str + report: str + + +def _column_starts(header_line: str, headers: Sequence[str]) -> tuple[int, ...]: + starts: list[int] = [] + cursor = 0 + for header in headers: + index = header_line.find(header, cursor) + if index < 0: + raise ConversionError(f"Expected table header {header!r}: {header_line!r}") + if starts and index - cursor < 3: + raise ConversionError(f"Table columns are not separated by at least three spaces: {header_line!r}") + starts.append(index) + cursor = index + len(header) + if header_line[: starts[0]].strip() or header_line[cursor:].strip(): + raise ConversionError(f"Unexpected content in table header: {header_line!r}") + return tuple(starts) + + +def parse_fixed_width_table(text: str, headers: Sequence[str]) -> list[tuple[str, ...]]: + """Parse an OutputManager fixed-width table without splitting field content.""" + lines = [ANSI_ESCAPE.sub("", line.rstrip()) for line in text.splitlines() if line.strip()] + if not lines: + raise ConversionError("mreg-cli output is empty") + starts = _column_starts(lines[0], headers) + rows: list[tuple[str, ...]] = [] + for line_number, line in enumerate(lines[1:], start=2): + if len(line) <= starts[-2]: + raise ConversionError(f"Row {line_number} is shorter than the required columns: {line!r}") + # OutputManager pads an empty final column with spaces. Be tolerant of + # users or editors stripping that trailing whitespace from a capture. + line = line.ljust(starts[-1]) + values = tuple( + line[start : starts[index + 1] if index + 1 < len(starts) else None].strip() + for index, start in enumerate(starts) + ) + if not any(values): + continue + if any(not value for value in values[:-1]): + raise ConversionError(f"Row {line_number} has an empty required column: {line!r}") + rows.append(values) + if not rows: + raise ConversionError("mreg-cli output contains no data rows") + return rows + + +def _parse_labels(value: str) -> tuple[str, ...]: + return tuple(sorted({label.strip() for label in value.split(",") if label.strip()})) + + +def parse_permissions(text: str) -> tuple[NetworkPermission, ...]: + permissions: set[NetworkPermission] = set() + for network_value, group, regex, labels_value in parse_fixed_width_table(text, PERMISSION_HEADERS): + try: + network = str(ipaddress.ip_network(network_value, strict=True)) + except ValueError as exc: + raise ConversionError(f"Invalid permission range {network_value!r}: {exc}") from exc + try: + re.compile(regex) + except re.error as exc: + raise ConversionError(f"Invalid permission regex {regex!r}: {exc}") from exc + permissions.add( + NetworkPermission( + network=network, + group=group, + regex=regex, + labels=_parse_labels(labels_value), + ) + ) + return tuple(sorted(permissions)) + + +def parse_roles(text: str) -> tuple[HostPolicyRole, ...]: + roles_by_name: dict[str, HostPolicyRole] = {} + for name, _description, labels_value in parse_fixed_width_table(text, ROLE_HEADERS): + role = HostPolicyRole(name=name, labels=_parse_labels(labels_value)) + if name in roles_by_name: + raise ConversionError(f"Duplicate host-policy role {name!r}") + roles_by_name[name] = role + return tuple(sorted(roles_by_name.values())) + + +def _stable_name(prefix: str, *parts: str) -> str: + digest = hashlib.sha256("\0".join(parts).encode()).hexdigest()[:12] + return f"{prefix}_{digest}" + + +def _quote(value: str) -> str: + return json.dumps(value, ensure_ascii=False) + + +def _actions(actions: Sequence[str], indent: str = " ") -> str: + values = [f'MREG::Action::{_quote(action)}' for action in actions] + return "[" + (",\n" + indent).join(values) + "]" + + +def _ranges(networks: Sequence[str], *, attribute: str = "ip") -> str: + checks = [f'resource.{attribute}.isInRange(ip({_quote(network)}))' for network in networks] + return "(" + (" ||\n ").join(checks) + ")" + + +def _network_values(networks: Sequence[str]) -> str: + checks = [f'resource.network == {_quote(network)}' for network in networks] + return "(" + (" ||\n ").join(checks) + ")" + + +def _permit_ip_rule(group: str, regex: str, networks: Sequence[str]) -> str: + label = _stable_name("netgroup", regex) + policy_id = _stable_name("netgroup_ip", group, regex) + return f'''@id("MREG.generated.{policy_id}") +permit ( + principal in MREG::Group::{_quote(group)}, + action in + {_actions(IP_SCOPED_ACTIONS)}, + resource +) +when {{ + resource has nameLabels && + resource.nameLabels.contains({_quote(label)}) && + resource has ip && + {_ranges(networks)} +}}; +''' + + +def _permit_hostname_rule(group: str, regex: str) -> str: + label = _stable_name("netgroup", regex) + policy_id = _stable_name("netgroup_hostname", group, regex) + return f'''@id("MREG.generated.{policy_id}") +permit ( + principal in MREG::Group::{_quote(group)}, + action in + {_actions(HOSTNAME_SCOPED_ACTIONS)}, + resource is MREG::Cname +) +when {{ + resource has nameLabels && + resource.nameLabels.contains({_quote(label)}) +}}; +''' + + +def _permit_network_rule(group: str, networks: Sequence[str]) -> str: + policy_id = _stable_name("netgroup_network", group) + return f'''@id("MREG.generated.{policy_id}") +permit ( + principal in MREG::Group::{_quote(group)}, + action in + {_actions(NETWORK_SCOPED_ACTIONS)}, + resource +) +when {{ + resource has network && + {_network_values(networks)} +}}; +''' + + +def _permit_role_rule(group: str, regex: str, role_name: str, networks: Sequence[str]) -> str: + label = _stable_name("netgroup", regex) + policy_id = _stable_name("hostpolicy_role", group, regex, role_name) + return f'''@id("MREG.generated.{policy_id}") +permit ( + principal in MREG::Group::{_quote(group)}, + action == MREG::Action::"hostpolicy_role_host_membership_update", + resource == MREG::HostPolicyRole::{_quote(role_name)} +) +when {{ + resource has nameLabels && + resource.nameLabels.contains({_quote(label)}) && + resource has ip && + {_ranges(networks)} +}}; +''' + + +def _group_values(permissions: Iterable[NetworkPermission], *fields: str) -> dict[tuple[str, ...], set[str]]: + grouped: dict[tuple[str, ...], set[str]] = {} + for permission in permissions: + key = tuple(str(getattr(permission, field)) for field in fields) + grouped.setdefault(key, set()).add(permission.network) + return grouped + + +def _normalized_networks(networks: Iterable[str]) -> tuple[str, ...]: + """Sort and collapse redundant ranges without mixing address families.""" + parsed = [ipaddress.ip_network(network) for network in networks] + collapsed = [ + network + for version in (4, 6) + for network in ipaddress.collapse_addresses( + network for network in parsed if network.version == version + ) + ] + return tuple(str(network) for network in collapsed) + + +def generate_policy(permissions: Sequence[NetworkPermission], roles: Sequence[HostPolicyRole]) -> GeneratedPolicy: + permissions = tuple(sorted(set(permissions))) + roles = tuple(sorted(set(roles))) + regexes = sorted({permission.regex for permission in permissions}) + patterns = [ + {"name": _stable_name("netgroup", regex), "regex": regex} + for regex in regexes + ] + labels = [ + { + "kind": kind, + "field": "hostname", + "output": "nameLabels", + "patterns": [ + *(STATIC_NAME_PATTERNS if kind != "MREG::HostPolicyRole" else ()), + *patterns, + ], + } + for kind in LABEL_RESOURCE_KINDS + ] + + rules: list[str] = [ + "// Generated from mreg-cli permission data. Do not edit by hand.\n", + ] + rule_ids: list[str] = [] + + by_group_regex = _group_values(permissions, "group", "regex") + for (group, regex), networks_set in sorted(by_group_regex.items()): + networks = _normalized_networks(networks_set) + rules.append(_permit_ip_rule(group, regex, networks)) + rules.append(_permit_hostname_rule(group, regex)) + rule_ids.extend( + ( + _stable_name("netgroup_ip", group, regex), + _stable_name("netgroup_hostname", group, regex), + ) + ) + + by_group = _group_values(permissions, "group") + for (group,), networks_set in sorted(by_group.items()): + networks = _normalized_networks(networks_set) + rules.append(_permit_network_rule(group, networks)) + rule_ids.append(_stable_name("netgroup_network", group)) + + role_networks: dict[tuple[str, str, str], set[str]] = {} + used_legacy_labels: set[str] = set() + for permission in permissions: + permission_labels = set(permission.labels) + if not permission_labels: + continue + for role in roles: + shared = permission_labels.intersection(role.labels) + if not shared: + continue + used_legacy_labels.update(shared) + key = (permission.group, permission.regex, role.name) + role_networks.setdefault(key, set()).add(permission.network) + + for (group, regex, role_name), networks_set in sorted(role_networks.items()): + rules.append(_permit_role_rule(group, regex, role_name, _normalized_networks(networks_set))) + rule_ids.append(_stable_name("hostpolicy_role", group, regex, role_name)) + + permission_labels = {label for permission in permissions for label in permission.labels} + role_labels = {label for role in roles for label in role.labels} + report_data = { + "permission_rows": len(permissions), + "role_rows": len(roles), + "unique_regexes": len(regexes), + "generated_rules": len(rule_ids), + "generated_role_rules": len(role_networks), + "unused_permission_labels": sorted(permission_labels - used_legacy_labels), + "unmatched_role_labels": sorted(role_labels - permission_labels), + "derived_labels": {regex: _stable_name("netgroup", regex) for regex in regexes}, + "policy_ids": sorted(rule_ids), + } + return GeneratedPolicy( + labels=json.dumps(labels, indent=2, ensure_ascii=False) + "\n", + cedar="\n".join(rules).rstrip() + "\n", + report=json.dumps(report_data, indent=2, ensure_ascii=False, sort_keys=True) + "\n", + ) + + +def _outputs(output_dir: Path, policy: GeneratedPolicy) -> dict[Path, str]: + return { + output_dir / "labels.json": policy.labels, + output_dir / "netgroup.cedar": policy.cedar, + output_dir / "netgroup-conversion-report.json": policy.report, + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--permissions", type=Path, default=DEFAULT_PERMISSIONS) + parser.add_argument("--roles", type=Path, default=DEFAULT_ROLES) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) + parser.add_argument("--check", action="store_true", help="fail if generated output differs") + args = parser.parse_args(argv) + + try: + permissions = parse_permissions(args.permissions.read_text()) + roles = parse_roles(args.roles.read_text()) + generated = generate_policy(permissions, roles) + except (ConversionError, OSError) as exc: + print(f"Unable to generate TreeTop policy: {exc}", file=sys.stderr) + return 2 + + outputs = _outputs(args.output_dir, generated) + if args.check: + stale = [path for path, content in outputs.items() if not path.exists() or path.read_text() != content] + if stale: + print("Generated TreeTop policy is stale: " + ", ".join(str(path) for path in stale), file=sys.stderr) + return 1 + print("Generated TreeTop permission policy matches the mreg-cli fixtures") + return 0 + + args.output_dir.mkdir(parents=True, exist_ok=True) + for path, content in outputs.items(): + path.write_text(content) + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/mreg/tests/test_treetop_policy_generator.py b/mreg/tests/test_treetop_policy_generator.py index c684a9cf..29492b5e 100644 --- a/mreg/tests/test_treetop_policy_generator.py +++ b/mreg/tests/test_treetop_policy_generator.py @@ -1,21 +1,11 @@ from __future__ import annotations -import importlib.util import json from pathlib import Path -import sys import tempfile from unittest import TestCase - -ROOT = Path(__file__).resolve().parents[2] -SCRIPT = ROOT / "scripts/generate-treetop-policy.py" -SPEC = importlib.util.spec_from_file_location("_generate_treetop_policy", SCRIPT) -if SPEC is None or SPEC.loader is None: # pragma: no cover - raise RuntimeError(f"Unable to load {SCRIPT}") -generator = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = generator -SPEC.loader.exec_module(generator) +from mreg.policy import treetop_generator as generator class TreeTopPolicyGeneratorTests(TestCase): @@ -74,31 +64,29 @@ def test_rejects_malformed_input(self) -> None: def test_cli_check_detects_stale_output(self) -> None: with tempfile.TemporaryDirectory() as directory: - output_dir = Path(directory) + temp_dir = Path(directory) + permissions_path = temp_dir / "permissions.txt" + roles_path = temp_dir / "roles.txt" + output_dir = temp_dir / "output" + permissions_path.write_text(self.permission_table) + roles_path.write_text(self.role_table) + arguments = [ + "--permissions", + str(permissions_path), + "--roles", + str(roles_path), + "--output-dir", + str(output_dir), + ] + + self.assertEqual(generator.main(arguments), 0) self.assertEqual( - generator.main( - [ - "--permissions", - str(ROOT / "treetop/fixtures/network-permissions.txt"), - "--roles", - str(ROOT / "treetop/fixtures/hostpolicy-roles.txt"), - "--output-dir", - str(output_dir), - ] - ), + generator.main([*arguments, "--check"]), 0, ) + + (output_dir / "netgroup.cedar").write_text("stale\n") self.assertEqual( - generator.main( - [ - "--permissions", - str(ROOT / "treetop/fixtures/network-permissions.txt"), - "--roles", - str(ROOT / "treetop/fixtures/hostpolicy-roles.txt"), - "--output-dir", - str(output_dir), - "--check", - ] - ), - 0, + generator.main([*arguments, "--check"]), + 1, ) diff --git a/scripts/generate-treetop-policy.py b/scripts/generate-treetop-policy.py index 601c28f5..f713a07a 100644 --- a/scripts/generate-treetop-policy.py +++ b/scripts/generate-treetop-policy.py @@ -1,447 +1,15 @@ #!/usr/bin/env python3 """Generate TreeTop policy data from deterministic mreg-cli table output.""" -from __future__ import annotations - -import argparse -from dataclasses import dataclass -import hashlib -import ipaddress -import json from pathlib import Path -import re import sys -from typing import Iterable, Sequence ROOT = Path(__file__).resolve().parents[1] -DEFAULT_PERMISSIONS = ROOT / "treetop/fixtures/network-permissions.txt" -DEFAULT_ROLES = ROOT / "treetop/fixtures/hostpolicy-roles.txt" -DEFAULT_OUTPUT_DIR = ROOT / "treetop/data" - -PERMISSION_HEADERS = ("Range", "Group", "Regex", "Labels") -ROLE_HEADERS = ("Name", "Description", "Labels") - -LABEL_RESOURCE_KINDS = ( - "MREG::Host", - "MREG::Ipaddress", - "MREG::Cname", - "MREG::Hinfo", - "MREG::Loc", - "MREG::Mx", - "MREG::Naptr", - "MREG::NameServer", - "MREG::PtrOverride", - "MREG::Sshfp", - "MREG::Srv", - "MREG::Txt", - "MREG::BACnetID", - "MREG::HostPolicyRole", -) - -STATIC_NAME_PATTERNS = ( - {"name": "dns_wildcard", "regex": r"\*"}, - {"name": "dns_wildcard_valid_depth", "regex": r"^(?:[^.]+\.){3,}[^.]+$"}, - {"name": "dns_underscore", "regex": "_"}, -) - -IP_SCOPED_ACTIONS = ( - "host_create", - "host_update", - "host_delete", - "host_contacts_create", - "host_contacts_delete", - "ipaddress_create", - "ipaddress_update", - "ipaddress_delete", - "hinfo_create", - "hinfo_update", - "hinfo_delete", - "loc_create", - "loc_update", - "loc_delete", - "mx_create", - "mx_update", - "mx_delete", - "naptr_create", - "naptr_update", - "naptr_delete", - "name_server_create", - "name_server_update", - "name_server_delete", - "ptr_override_create", - "ptr_override_update", - "ptr_override_delete", - "sshfp_create", - "sshfp_update", - "sshfp_delete", - "srv_create", - "srv_update", - "srv_delete", - "txt_create", - "txt_update", - "txt_delete", - "bacnet_id_create", - "bacnet_id_update", - "bacnet_id_delete", -) - -HOSTNAME_SCOPED_ACTIONS = ( - "cname_create", - "cname_update", - "cname_delete", -) - -NETWORK_SCOPED_ACTIONS = ( - "community_create", - "community_update", - "community_delete", - "host_create", - "host_delete", -) - -ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") - - -class ConversionError(ValueError): - """Raised when mreg-cli output cannot be converted safely.""" - - -@dataclass(frozen=True, order=True) -class NetworkPermission: - network: str - group: str - regex: str - labels: tuple[str, ...] - - -@dataclass(frozen=True, order=True) -class HostPolicyRole: - name: str - labels: tuple[str, ...] - - -@dataclass(frozen=True) -class GeneratedPolicy: - labels: str - cedar: str - report: str - - -def _column_starts(header_line: str, headers: Sequence[str]) -> tuple[int, ...]: - starts: list[int] = [] - cursor = 0 - for header in headers: - index = header_line.find(header, cursor) - if index < 0: - raise ConversionError(f"Expected table header {header!r}: {header_line!r}") - if starts and index - cursor < 3: - raise ConversionError(f"Table columns are not separated by at least three spaces: {header_line!r}") - starts.append(index) - cursor = index + len(header) - if header_line[: starts[0]].strip() or header_line[cursor:].strip(): - raise ConversionError(f"Unexpected content in table header: {header_line!r}") - return tuple(starts) - - -def parse_fixed_width_table(text: str, headers: Sequence[str]) -> list[tuple[str, ...]]: - """Parse an OutputManager fixed-width table without splitting field content.""" - lines = [ANSI_ESCAPE.sub("", line.rstrip()) for line in text.splitlines() if line.strip()] - if not lines: - raise ConversionError("mreg-cli output is empty") - starts = _column_starts(lines[0], headers) - rows: list[tuple[str, ...]] = [] - for line_number, line in enumerate(lines[1:], start=2): - if len(line) <= starts[-2]: - raise ConversionError(f"Row {line_number} is shorter than the required columns: {line!r}") - # OutputManager pads an empty final column with spaces. Be tolerant of - # users or editors stripping that trailing whitespace from a capture. - line = line.ljust(starts[-1]) - values = tuple( - line[start : starts[index + 1] if index + 1 < len(starts) else None].strip() - for index, start in enumerate(starts) - ) - if not any(values): - continue - if any(not value for value in values[:-1]): - raise ConversionError(f"Row {line_number} has an empty required column: {line!r}") - rows.append(values) - if not rows: - raise ConversionError("mreg-cli output contains no data rows") - return rows - - -def _parse_labels(value: str) -> tuple[str, ...]: - return tuple(sorted({label.strip() for label in value.split(",") if label.strip()})) - - -def parse_permissions(text: str) -> tuple[NetworkPermission, ...]: - permissions: set[NetworkPermission] = set() - for network_value, group, regex, labels_value in parse_fixed_width_table(text, PERMISSION_HEADERS): - try: - network = str(ipaddress.ip_network(network_value, strict=True)) - except ValueError as exc: - raise ConversionError(f"Invalid permission range {network_value!r}: {exc}") from exc - try: - re.compile(regex) - except re.error as exc: - raise ConversionError(f"Invalid permission regex {regex!r}: {exc}") from exc - permissions.add( - NetworkPermission( - network=network, - group=group, - regex=regex, - labels=_parse_labels(labels_value), - ) - ) - return tuple(sorted(permissions)) - - -def parse_roles(text: str) -> tuple[HostPolicyRole, ...]: - roles_by_name: dict[str, HostPolicyRole] = {} - for name, _description, labels_value in parse_fixed_width_table(text, ROLE_HEADERS): - role = HostPolicyRole(name=name, labels=_parse_labels(labels_value)) - if name in roles_by_name: - raise ConversionError(f"Duplicate host-policy role {name!r}") - roles_by_name[name] = role - return tuple(sorted(roles_by_name.values())) - - -def _stable_name(prefix: str, *parts: str) -> str: - digest = hashlib.sha256("\0".join(parts).encode()).hexdigest()[:12] - return f"{prefix}_{digest}" - - -def _quote(value: str) -> str: - return json.dumps(value, ensure_ascii=False) - - -def _actions(actions: Sequence[str], indent: str = " ") -> str: - values = [f'MREG::Action::{_quote(action)}' for action in actions] - return "[" + (",\n" + indent).join(values) + "]" - - -def _ranges(networks: Sequence[str], *, attribute: str = "ip") -> str: - checks = [f'resource.{attribute}.isInRange(ip({_quote(network)}))' for network in networks] - return "(" + (" ||\n ").join(checks) + ")" - - -def _network_values(networks: Sequence[str]) -> str: - checks = [f'resource.network == {_quote(network)}' for network in networks] - return "(" + (" ||\n ").join(checks) + ")" - - -def _permit_ip_rule(group: str, regex: str, networks: Sequence[str]) -> str: - label = _stable_name("netgroup", regex) - policy_id = _stable_name("netgroup_ip", group, regex) - return f'''@id("MREG.generated.{policy_id}") -permit ( - principal in MREG::Group::{_quote(group)}, - action in - {_actions(IP_SCOPED_ACTIONS)}, - resource -) -when {{ - resource has nameLabels && - resource.nameLabels.contains({_quote(label)}) && - resource has ip && - {_ranges(networks)} -}}; -''' - - -def _permit_hostname_rule(group: str, regex: str) -> str: - label = _stable_name("netgroup", regex) - policy_id = _stable_name("netgroup_hostname", group, regex) - return f'''@id("MREG.generated.{policy_id}") -permit ( - principal in MREG::Group::{_quote(group)}, - action in - {_actions(HOSTNAME_SCOPED_ACTIONS)}, - resource is MREG::Cname -) -when {{ - resource has nameLabels && - resource.nameLabels.contains({_quote(label)}) -}}; -''' - - -def _permit_network_rule(group: str, networks: Sequence[str]) -> str: - policy_id = _stable_name("netgroup_network", group) - return f'''@id("MREG.generated.{policy_id}") -permit ( - principal in MREG::Group::{_quote(group)}, - action in - {_actions(NETWORK_SCOPED_ACTIONS)}, - resource -) -when {{ - resource has network && - {_network_values(networks)} -}}; -''' - - -def _permit_role_rule(group: str, regex: str, role_name: str, networks: Sequence[str]) -> str: - label = _stable_name("netgroup", regex) - policy_id = _stable_name("hostpolicy_role", group, regex, role_name) - return f'''@id("MREG.generated.{policy_id}") -permit ( - principal in MREG::Group::{_quote(group)}, - action == MREG::Action::"hostpolicy_role_host_membership_update", - resource == MREG::HostPolicyRole::{_quote(role_name)} -) -when {{ - resource has nameLabels && - resource.nameLabels.contains({_quote(label)}) && - resource has ip && - {_ranges(networks)} -}}; -''' - - -def _group_values(permissions: Iterable[NetworkPermission], *fields: str) -> dict[tuple[str, ...], set[str]]: - grouped: dict[tuple[str, ...], set[str]] = {} - for permission in permissions: - key = tuple(str(getattr(permission, field)) for field in fields) - grouped.setdefault(key, set()).add(permission.network) - return grouped - - -def _normalized_networks(networks: Iterable[str]) -> tuple[str, ...]: - """Sort and collapse redundant ranges without mixing address families.""" - parsed = [ipaddress.ip_network(network) for network in networks] - collapsed = [ - network - for version in (4, 6) - for network in ipaddress.collapse_addresses( - network for network in parsed if network.version == version - ) - ] - return tuple(str(network) for network in collapsed) - - -def generate_policy(permissions: Sequence[NetworkPermission], roles: Sequence[HostPolicyRole]) -> GeneratedPolicy: - permissions = tuple(sorted(set(permissions))) - roles = tuple(sorted(set(roles))) - regexes = sorted({permission.regex for permission in permissions}) - patterns = [ - {"name": _stable_name("netgroup", regex), "regex": regex} - for regex in regexes - ] - labels = [ - { - "kind": kind, - "field": "hostname", - "output": "nameLabels", - "patterns": [ - *(STATIC_NAME_PATTERNS if kind != "MREG::HostPolicyRole" else ()), - *patterns, - ], - } - for kind in LABEL_RESOURCE_KINDS - ] - - rules: list[str] = [ - "// Generated from mreg-cli permission data. Do not edit by hand.\n", - ] - rule_ids: list[str] = [] - - by_group_regex = _group_values(permissions, "group", "regex") - for (group, regex), networks_set in sorted(by_group_regex.items()): - networks = _normalized_networks(networks_set) - rules.append(_permit_ip_rule(group, regex, networks)) - rules.append(_permit_hostname_rule(group, regex)) - rule_ids.extend( - ( - _stable_name("netgroup_ip", group, regex), - _stable_name("netgroup_hostname", group, regex), - ) - ) - - by_group = _group_values(permissions, "group") - for (group,), networks_set in sorted(by_group.items()): - networks = _normalized_networks(networks_set) - rules.append(_permit_network_rule(group, networks)) - rule_ids.append(_stable_name("netgroup_network", group)) - - role_networks: dict[tuple[str, str, str], set[str]] = {} - used_legacy_labels: set[str] = set() - for permission in permissions: - permission_labels = set(permission.labels) - if not permission_labels: - continue - for role in roles: - shared = permission_labels.intersection(role.labels) - if not shared: - continue - used_legacy_labels.update(shared) - key = (permission.group, permission.regex, role.name) - role_networks.setdefault(key, set()).add(permission.network) - - for (group, regex, role_name), networks_set in sorted(role_networks.items()): - rules.append(_permit_role_rule(group, regex, role_name, _normalized_networks(networks_set))) - rule_ids.append(_stable_name("hostpolicy_role", group, regex, role_name)) - - permission_labels = {label for permission in permissions for label in permission.labels} - role_labels = {label for role in roles for label in role.labels} - report_data = { - "permission_rows": len(permissions), - "role_rows": len(roles), - "unique_regexes": len(regexes), - "generated_rules": len(rule_ids), - "generated_role_rules": len(role_networks), - "unused_permission_labels": sorted(permission_labels - used_legacy_labels), - "unmatched_role_labels": sorted(role_labels - permission_labels), - "derived_labels": {regex: _stable_name("netgroup", regex) for regex in regexes}, - "policy_ids": sorted(rule_ids), - } - return GeneratedPolicy( - labels=json.dumps(labels, indent=2, ensure_ascii=False) + "\n", - cedar="\n".join(rules).rstrip() + "\n", - report=json.dumps(report_data, indent=2, ensure_ascii=False, sort_keys=True) + "\n", - ) - - -def _outputs(output_dir: Path, policy: GeneratedPolicy) -> dict[Path, str]: - return { - output_dir / "labels.json": policy.labels, - output_dir / "netgroup.cedar": policy.cedar, - output_dir / "netgroup-conversion-report.json": policy.report, - } - - -def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--permissions", type=Path, default=DEFAULT_PERMISSIONS) - parser.add_argument("--roles", type=Path, default=DEFAULT_ROLES) - parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) - parser.add_argument("--check", action="store_true", help="fail if generated output differs") - args = parser.parse_args(argv) - - try: - permissions = parse_permissions(args.permissions.read_text()) - roles = parse_roles(args.roles.read_text()) - generated = generate_policy(permissions, roles) - except (ConversionError, OSError) as exc: - print(f"Unable to generate TreeTop policy: {exc}", file=sys.stderr) - return 2 - - outputs = _outputs(args.output_dir, generated) - if args.check: - stale = [path for path, content in outputs.items() if not path.exists() or path.read_text() != content] - if stale: - print("Generated TreeTop policy is stale: " + ", ".join(str(path) for path in stale), file=sys.stderr) - return 1 - print("Generated TreeTop permission policy matches the mreg-cli fixtures") - return 0 +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) - args.output_dir.mkdir(parents=True, exist_ok=True) - for path, content in outputs.items(): - path.write_text(content) - print(f"wrote {path}") - return 0 +from mreg.policy.treetop_generator import main # noqa: E402 if __name__ == "__main__": From 78cc307993d92dfff5be8971a150373d3346994f Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 19 Aug 2026 17:40:05 +0200 Subject: [PATCH 33/34] Support TreeTop generation before package install --- scripts/generate-treetop-policy.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/generate-treetop-policy.py b/scripts/generate-treetop-policy.py index f713a07a..eff090fc 100644 --- a/scripts/generate-treetop-policy.py +++ b/scripts/generate-treetop-policy.py @@ -2,14 +2,11 @@ """Generate TreeTop policy data from deterministic mreg-cli table output.""" from pathlib import Path -import sys +from runpy import run_path ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from mreg.policy.treetop_generator import main # noqa: E402 +main = run_path(str(ROOT / "mreg/policy/treetop_generator.py"))["main"] if __name__ == "__main__": From 8f9a5243fa2845746ae4ecb5ffb36fa7c0bf6055 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 19 Aug 2026 22:42:45 +0200 Subject: [PATCH 34/34] Generate TreeTop policy from MREG API --- README.md | 3 + docs/env.md | 14 + docs/policies.md | 39 +- mreg/policy/treetop_generator.py | 376 +++++++++++++++----- mreg/tests/test_treetop_policy_generator.py | 370 ++++++++++++++++--- scripts/generate-treetop-policy.py | 2 +- treetop/data/mreg-bundle.tar.gz | Bin 5253 -> 5268 bytes treetop/data/netgroup.cedar | 2 +- treetop/fixtures/hostpolicy-roles.txt | 3 - treetop/fixtures/network-permissions.txt | 9 - treetop/fixtures/policy-source.json | 83 +++++ 11 files changed, 747 insertions(+), 154 deletions(-) delete mode 100644 treetop/fixtures/hostpolicy-roles.txt delete mode 100644 treetop/fixtures/network-permissions.txt create mode 100644 treetop/fixtures/policy-source.json diff --git a/README.md b/README.md index ae7f6d99..6e1e00ed 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,9 @@ mreg supports configuration via environment variables with the `MREG_` prefix. T | `MREG_POLICY_ROLLOUT_MAX_MISMATCH_RATE` | `0.001` | Maximum accepted mismatch ratio | | `MREG_POLICY_ROLLOUT_MAX_ERROR_RATE` | `0.001` | Maximum accepted policy error ratio | +TreeTop bundle generation reads the three existing MREG policy endpoints with +`MREG_API_BASE_URL` and `MREG_API_TOKEN`; see [the policy documentation](docs/policies.md#bundle-source-and-build). + ### Network Policy Configuration | Variable | Default | Description | diff --git a/docs/env.md b/docs/env.md index b99782a4..e81f596b 100644 --- a/docs/env.md +++ b/docs/env.md @@ -103,6 +103,20 @@ operator enables policy enforcement. Defaults can be tuned with: - `MREG_POLICY_ROLLOUT_MAX_MISMATCH_RATE` (`0.001`) - `MREG_POLICY_ROLLOUT_MAX_ERROR_RATE` (`0.001`) +## TreeTop bundle generation + +These variables are used only by `scripts/generate-treetop-policy.py`; they are +not Django runtime settings: + +- `MREG_API_BASE_URL`: MREG base URL to read policy source data from. When + omitted, the generator uses `treetop/fixtures/policy-source.json`. +- `MREG_API_TOKEN`: API token sent to the three existing MREG endpoints. It is + required when `MREG_API_BASE_URL` is set and is never persisted. +- `MREG_API_TIMEOUT`: per-page API timeout in seconds. Default: `20`. + +Use HTTPS for a remote MREG instance. The token needs authenticated read access +to labels, NetGroup regex permissions, and host-policy roles. + ## `MREG_LOG_FILE_SIZE` Maximum file size of the log file in bytes. Default: `52428800` (50MB). diff --git a/docs/policies.md b/docs/policies.md index 8e7e5879..bbc596c0 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -137,25 +137,42 @@ test scopes; it cannot bypass enforcement. | Generated NetGroup/role policy | `treetop/data/netgroup.cedar` | | Generated TreeTop labels | `treetop/data/labels.json` | | Conversion report | `treetop/data/netgroup-conversion-report.json` | -| Permission export input | `treetop/fixtures/network-permissions.txt` | -| Role export input | `treetop/fixtures/hostpolicy-roles.txt` | +| Normalized API snapshot | `treetop/fixtures/policy-source.json` | | Generated schema | `treetop/data/mreg.cedarschema` | | Generated archive | `treetop/data/mreg-bundle.tar.gz` | -Refresh the conversion inputs with mreg-cli against the database whose policy -is being migrated: +Refresh the conversion input directly from the MREG instance whose policy is +being migrated: ```bash -mreg-cli permission network_list > treetop/fixtures/network-permissions.txt -mreg-cli policy list_roles '*' > treetop/fixtures/hostpolicy-roles.txt +export MREG_API_BASE_URL=https://mreg.example +export MREG_API_TOKEN='replace-with-an-MREG-API-token' python scripts/generate-treetop-policy.py +unset MREG_API_TOKEN ``` -The parser consumes the commands' fixed-width tables, validates every CIDR and -regular expression, removes duplicates, collapses redundant ranges, and emits -stable hashed IDs. Review the generated Cedar and -`netgroup-conversion-report.json`, especially unmatched or unused legacy -labels. The checked-in fixtures are sanitized examples, not production policy. +The generator paginates the existing `/api/v1/labels/`, +`/api/v1/permissions/netgroupregex/`, and `/api/v1/hostpolicy/roles/` +endpoints. It authenticates with `Authorization: Token`, resolves label IDs to +names, and writes a deterministic snapshot containing only the fields needed by +the conversion. No `mreg-cli` installation or new export endpoint is required. +Use HTTPS outside a trusted local environment, and use a token with authenticated +read access to all three endpoints. The token is read only from the environment +and is never written to the snapshot. + +The converter validates every CIDR and regular expression, removes duplicate +permission rows, collapses redundant ranges, and emits stable hashed IDs. +Review the snapshot, generated Cedar, and `netgroup-conversion-report.json`, +especially unmatched or unused legacy labels. The checked-in snapshot is a +sanitized example, not production policy. `MREG_API_TIMEOUT` optionally changes +the per-page timeout from 20 seconds. + +Without `MREG_API_BASE_URL`, the generator uses the checked-in snapshot. CI uses +that offline path: + +```bash +python scripts/generate-treetop-policy.py --check +``` The restricted-address examples in `mreg.cedar` are also based on the sample networks. Replace and review them for the deployment before enabling `enforce`. diff --git a/mreg/policy/treetop_generator.py b/mreg/policy/treetop_generator.py index 380580b3..d2f13bac 100644 --- a/mreg/policy/treetop_generator.py +++ b/mreg/policy/treetop_generator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate TreeTop policy data from deterministic mreg-cli table output.""" +"""Generate TreeTop policy data from existing MREG API endpoints.""" from __future__ import annotations @@ -8,19 +8,22 @@ import hashlib import ipaddress import json +import math +import os from pathlib import Path import re import sys -from typing import Iterable, Sequence +from typing import Any, Iterable, Mapping, Sequence +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode, urljoin, urlparse +from urllib.request import Request, urlopen ROOT = Path(__file__).resolve().parents[2] -DEFAULT_PERMISSIONS = ROOT / "treetop/fixtures/network-permissions.txt" -DEFAULT_ROLES = ROOT / "treetop/fixtures/hostpolicy-roles.txt" +DEFAULT_SNAPSHOT = ROOT / "treetop/fixtures/policy-source.json" DEFAULT_OUTPUT_DIR = ROOT / "treetop/data" - -PERMISSION_HEADERS = ("Range", "Group", "Regex", "Labels") -ROLE_HEADERS = ("Name", "Description", "Labels") +SNAPSHOT_SCHEMA_VERSION = 1 +API_PAGE_SIZE = 1000 LABEL_RESOURCE_KINDS = ( "MREG::Host", @@ -100,11 +103,8 @@ "host_delete", ) -ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") - - class ConversionError(ValueError): - """Raised when mreg-cli output cannot be converted safely.""" + """Raised when MREG API data cannot be converted safely.""" @dataclass(frozen=True, order=True) @@ -128,83 +128,261 @@ class GeneratedPolicy: report: str -def _column_starts(header_line: str, headers: Sequence[str]) -> tuple[int, ...]: - starts: list[int] = [] - cursor = 0 - for header in headers: - index = header_line.find(header, cursor) - if index < 0: - raise ConversionError(f"Expected table header {header!r}: {header_line!r}") - if starts and index - cursor < 3: - raise ConversionError(f"Table columns are not separated by at least three spaces: {header_line!r}") - starts.append(index) - cursor = index + len(header) - if header_line[: starts[0]].strip() or header_line[cursor:].strip(): - raise ConversionError(f"Unexpected content in table header: {header_line!r}") - return tuple(starts) - - -def parse_fixed_width_table(text: str, headers: Sequence[str]) -> list[tuple[str, ...]]: - """Parse an OutputManager fixed-width table without splitting field content.""" - lines = [ANSI_ESCAPE.sub("", line.rstrip()) for line in text.splitlines() if line.strip()] - if not lines: - raise ConversionError("mreg-cli output is empty") - starts = _column_starts(lines[0], headers) - rows: list[tuple[str, ...]] = [] - for line_number, line in enumerate(lines[1:], start=2): - if len(line) <= starts[-2]: - raise ConversionError(f"Row {line_number} is shorter than the required columns: {line!r}") - # OutputManager pads an empty final column with spaces. Be tolerant of - # users or editors stripping that trailing whitespace from a capture. - line = line.ljust(starts[-1]) - values = tuple( - line[start : starts[index + 1] if index + 1 < len(starts) else None].strip() - for index, start in enumerate(starts) - ) - if not any(values): - continue - if any(not value for value in values[:-1]): - raise ConversionError(f"Row {line_number} has an empty required column: {line!r}") - rows.append(values) - if not rows: - raise ConversionError("mreg-cli output contains no data rows") - return rows +def _object(value: Any, context: str) -> Mapping[str, Any]: + if not isinstance(value, dict): + raise ConversionError(f"{context} must be a JSON object") + return value -def _parse_labels(value: str) -> tuple[str, ...]: - return tuple(sorted({label.strip() for label in value.split(",") if label.strip()})) +def _array(value: Any, context: str) -> list[Any]: + if not isinstance(value, list): + raise ConversionError(f"{context} must be a JSON array") + return value -def parse_permissions(text: str) -> tuple[NetworkPermission, ...]: - permissions: set[NetworkPermission] = set() - for network_value, group, regex, labels_value in parse_fixed_width_table(text, PERMISSION_HEADERS): - try: - network = str(ipaddress.ip_network(network_value, strict=True)) - except ValueError as exc: - raise ConversionError(f"Invalid permission range {network_value!r}: {exc}") from exc +def _string(row: Mapping[str, Any], field: str, context: str) -> str: + value = row.get(field) + if not isinstance(value, str) or not value: + raise ConversionError(f"{context}.{field} must be a non-empty string") + return value + + +def _label_names(value: Any, context: str) -> tuple[str, ...]: + labels = _array(value, context) + if any(not isinstance(label, str) or not label for label in labels): + raise ConversionError(f"{context} must contain only non-empty strings") + return tuple(sorted(set(labels))) + + +def _permission(row: Mapping[str, Any], context: str) -> NetworkPermission: + network_value = _string(row, "range", context) + group = _string(row, "group", context) + regex = _string(row, "regex", context) + try: + network = str(ipaddress.ip_network(network_value, strict=True)) + except ValueError as exc: + raise ConversionError(f"Invalid permission range {network_value!r}: {exc}") from exc + try: + re.compile(regex) + except re.error as exc: + raise ConversionError(f"Invalid permission regex {regex!r}: {exc}") from exc + return NetworkPermission( + network=network, + group=group, + regex=regex, + labels=_label_names(row.get("labels"), f"{context}.labels"), + ) + + +def _role(row: Mapping[str, Any], context: str) -> HostPolicyRole: + return HostPolicyRole( + name=_string(row, "name", context), + labels=_label_names(row.get("labels"), f"{context}.labels"), + ) + + +def parse_snapshot(text: str) -> tuple[tuple[NetworkPermission, ...], tuple[HostPolicyRole, ...]]: + """Parse the deterministic, normalized snapshot produced from MREG endpoints.""" + try: + payload = _object(json.loads(text), "snapshot") + except json.JSONDecodeError as exc: + raise ConversionError(f"Snapshot is not valid JSON: {exc}") from exc + schema_version = payload.get("schema_version") + if isinstance(schema_version, bool) or schema_version != SNAPSHOT_SCHEMA_VERSION: + raise ConversionError(f"snapshot.schema_version must be {SNAPSHOT_SCHEMA_VERSION}") + + permission_rows = _array(payload.get("permissions"), "snapshot.permissions") + permissions = { + _permission(_object(row, f"snapshot.permissions[{index}]"), f"snapshot.permissions[{index}]") + for index, row in enumerate(permission_rows) + } + if not permissions: + raise ConversionError("snapshot.permissions contains no data rows") + + role_rows = _array(payload.get("roles"), "snapshot.roles") + roles_by_name: dict[str, HostPolicyRole] = {} + for index, value in enumerate(role_rows): + context = f"snapshot.roles[{index}]" + role = _role(_object(value, context), context) + if role.name in roles_by_name: + raise ConversionError(f"Duplicate host-policy role {role.name!r}") + roles_by_name[role.name] = role + if not roles_by_name: + raise ConversionError("snapshot.roles contains no data rows") + return tuple(sorted(permissions)), tuple(sorted(roles_by_name.values())) + + +def serialize_snapshot( + permissions: Iterable[NetworkPermission], + roles: Iterable[HostPolicyRole], +) -> str: + """Serialize only the endpoint fields needed to reproduce generated policy.""" + payload = { + "schema_version": SNAPSHOT_SCHEMA_VERSION, + "permissions": [ + { + "group": permission.group, + "labels": list(permission.labels), + "range": permission.network, + "regex": permission.regex, + } + for permission in sorted(set(permissions)) + ], + "roles": [ + {"labels": list(role.labels), "name": role.name} + for role in sorted(set(roles)) + ], + } + return json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=True) + "\n" + + +def snapshot_from_endpoint_rows( + permission_rows: Sequence[Mapping[str, Any]], + role_rows: Sequence[Mapping[str, Any]], + label_rows: Sequence[Mapping[str, Any]], +) -> str: + """Normalize the three existing endpoint responses into one stable snapshot.""" + label_names: dict[int, str] = {} + names_seen: set[str] = set() + for index, row in enumerate(label_rows): + context = f"labels[{index}]" + label_id = row.get("id") + if isinstance(label_id, bool) or not isinstance(label_id, int): + raise ConversionError(f"{context}.id must be an integer") + name = _string(row, "name", context) + if label_id in label_names: + raise ConversionError(f"Duplicate label id {label_id}") + if name in names_seen: + raise ConversionError(f"Duplicate label name {name!r}") + label_names[label_id] = name + names_seen.add(name) + + def resolve_labels(row: Mapping[str, Any], context: str) -> tuple[str, ...]: + label_ids = _array(row.get("labels"), f"{context}.labels") + resolved: set[str] = set() + for label_id in label_ids: + if isinstance(label_id, bool) or not isinstance(label_id, int): + raise ConversionError(f"{context}.labels must contain only integer label ids") + try: + resolved.add(label_names[label_id]) + except KeyError as exc: + raise ConversionError(f"{context} references unknown label id {label_id}") from exc + return tuple(sorted(resolved)) + + permissions: list[NetworkPermission] = [] + for index, row in enumerate(permission_rows): + context = f"permissions[{index}]" + normalized = dict(row) + normalized["labels"] = list(resolve_labels(row, context)) + permissions.append(_permission(normalized, context)) + + roles: list[HostPolicyRole] = [] + for index, row in enumerate(role_rows): + context = f"roles[{index}]" + normalized = dict(row) + normalized["labels"] = list(resolve_labels(row, context)) + roles.append(_role(normalized, context)) + return serialize_snapshot(permissions, roles) + + +def _validated_api_base_url(value: str) -> str: + url = value.rstrip("/") + parsed = urlparse(url) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: + raise ConversionError("MREG API base URL must be an absolute HTTP(S) URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ConversionError("MREG API base URL must not contain credentials, a query, or a fragment") + return url + + +def _same_origin(url: str, base_url: str) -> bool: + parsed = urlparse(url) + base = urlparse(base_url) + return (parsed.scheme.lower(), parsed.netloc.lower()) == (base.scheme.lower(), base.netloc.lower()) + + +def _fetch_paginated_rows( + *, + base_url: str, + path: str, + token: str, + timeout: float, + ordering: str, +) -> list[Mapping[str, Any]]: + query = urlencode({"ordering": ordering, "page_size": API_PAGE_SIZE}) + next_url: str | None = f"{urljoin(base_url + '/', path.lstrip('/'))}?{query}" + seen_urls: set[str] = set() + rows: list[Mapping[str, Any]] = [] + + while next_url is not None: + if next_url in seen_urls: + raise ConversionError(f"MREG API pagination loop detected at {next_url}") + if not _same_origin(next_url, base_url): + raise ConversionError(f"MREG API pagination URL changed origin: {next_url}") + seen_urls.add(next_url) + request = Request(next_url, headers={"Accept": "application/json"}) + # Do not allow urllib to copy the API token to a redirected request. + # The endpoint URLs already include their canonical trailing slash. + request.add_unredirected_header("Authorization", f"Token {token}") try: - re.compile(regex) - except re.error as exc: - raise ConversionError(f"Invalid permission regex {regex!r}: {exc}") from exc - permissions.add( - NetworkPermission( - network=network, - group=group, - regex=regex, - labels=_parse_labels(labels_value), - ) + with urlopen(request, timeout=timeout) as response: # noqa: S310 - URL scheme and origin are validated. + response_url = response.geturl() + if not _same_origin(response_url, base_url): + raise ConversionError(f"MREG API response changed origin: {response_url}") + payload = _object(json.loads(response.read()), f"response from {next_url}") + except HTTPError as exc: + raise ConversionError(f"MREG API returned HTTP {exc.code} for {next_url}") from exc + except URLError as exc: + raise ConversionError(f"Unable to reach MREG API at {next_url}: {exc.reason}") from exc + except json.JSONDecodeError as exc: + raise ConversionError(f"MREG API returned invalid JSON for {next_url}: {exc}") from exc + + page_rows = _array(payload.get("results"), f"response from {next_url}.results") + rows.extend( + _object(row, f"response from {next_url}.results[{index}]") + for index, row in enumerate(page_rows) ) - return tuple(sorted(permissions)) + following = payload.get("next") + if following is None: + next_url = None + elif isinstance(following, str) and following: + next_url = urljoin(next_url, following) + else: + raise ConversionError(f"response from {next_url}.next must be a URL or null") + return rows -def parse_roles(text: str) -> tuple[HostPolicyRole, ...]: - roles_by_name: dict[str, HostPolicyRole] = {} - for name, _description, labels_value in parse_fixed_width_table(text, ROLE_HEADERS): - role = HostPolicyRole(name=name, labels=_parse_labels(labels_value)) - if name in roles_by_name: - raise ConversionError(f"Duplicate host-policy role {name!r}") - roles_by_name[name] = role - return tuple(sorted(roles_by_name.values())) +def fetch_policy_snapshot(base_url: str, token: str, timeout: float = 20.0) -> str: + """Fetch current policy inputs from existing MREG endpoints.""" + base_url = _validated_api_base_url(base_url) + token = token.strip() + if not token or "\r" in token or "\n" in token: + raise ConversionError("MREG_API_TOKEN must be a non-empty HTTP header value") + if not math.isfinite(timeout) or timeout <= 0: + raise ConversionError("MREG API timeout must be a finite number greater than zero") + + labels = _fetch_paginated_rows( + base_url=base_url, + path="/api/v1/labels/", + token=token, + timeout=timeout, + ordering="name", + ) + permissions = _fetch_paginated_rows( + base_url=base_url, + path="/api/v1/permissions/netgroupregex/", + token=token, + timeout=timeout, + ordering="range,group", + ) + roles = _fetch_paginated_rows( + base_url=base_url, + path="/api/v1/hostpolicy/roles/", + token=token, + timeout=timeout, + ordering="name", + ) + return snapshot_from_endpoint_rows(permissions, roles, labels) def _stable_name(prefix: str, *parts: str) -> str: @@ -344,7 +522,7 @@ def generate_policy(permissions: Sequence[NetworkPermission], roles: Sequence[Ho ] rules: list[str] = [ - "// Generated from mreg-cli permission data. Do not edit by hand.\n", + "// Generated from the normalized MREG API policy snapshot. Do not edit by hand.\n", ] rule_ids: list[str] = [] @@ -414,15 +592,33 @@ def _outputs(output_dir: Path, policy: GeneratedPolicy) -> dict[Path, str]: def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--permissions", type=Path, default=DEFAULT_PERMISSIONS) - parser.add_argument("--roles", type=Path, default=DEFAULT_ROLES) + parser.add_argument("--snapshot", type=Path, default=DEFAULT_SNAPSHOT) + parser.add_argument( + "--api-base-url", + default=os.environ.get("MREG_API_BASE_URL"), + help="fetch current inputs from MREG instead of using the checked-in snapshot", + ) + parser.add_argument( + "--api-timeout", + default=os.environ.get("MREG_API_TIMEOUT", "20"), + type=float, + help="per-request MREG API timeout in seconds (default: 20)", + ) parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) parser.add_argument("--check", action="store_true", help="fail if generated output differs") args = parser.parse_args(argv) try: - permissions = parse_permissions(args.permissions.read_text()) - roles = parse_roles(args.roles.read_text()) + fetched_snapshot = None + if args.api_base_url: + token = os.environ.get("MREG_API_TOKEN", "") + if not token: + raise ConversionError("MREG_API_TOKEN is required when MREG_API_BASE_URL is configured") + fetched_snapshot = fetch_policy_snapshot(args.api_base_url, token, args.api_timeout) + snapshot = fetched_snapshot + else: + snapshot = args.snapshot.read_text() + permissions, roles = parse_snapshot(snapshot) generated = generate_policy(permissions, roles) except (ConversionError, OSError) as exc: print(f"Unable to generate TreeTop policy: {exc}", file=sys.stderr) @@ -431,12 +627,20 @@ def main(argv: Sequence[str] | None = None) -> int: outputs = _outputs(args.output_dir, generated) if args.check: stale = [path for path, content in outputs.items() if not path.exists() or path.read_text() != content] + if fetched_snapshot is not None and ( + not args.snapshot.exists() or args.snapshot.read_text() != fetched_snapshot + ): + stale.append(args.snapshot) if stale: print("Generated TreeTop policy is stale: " + ", ".join(str(path) for path in stale), file=sys.stderr) return 1 - print("Generated TreeTop permission policy matches the mreg-cli fixtures") + print("Generated TreeTop permission policy matches the MREG API snapshot") return 0 + if fetched_snapshot is not None: + args.snapshot.parent.mkdir(parents=True, exist_ok=True) + args.snapshot.write_text(fetched_snapshot) + print(f"wrote {args.snapshot}") args.output_dir.mkdir(parents=True, exist_ok=True) for path, content in outputs.items(): path.write_text(content) diff --git a/mreg/tests/test_treetop_policy_generator.py b/mreg/tests/test_treetop_policy_generator.py index 29492b5e..11498217 100644 --- a/mreg/tests/test_treetop_policy_generator.py +++ b/mreg/tests/test_treetop_policy_generator.py @@ -4,37 +4,79 @@ from pathlib import Path import tempfile from unittest import TestCase +from unittest.mock import patch from mreg.policy import treetop_generator as generator +class JsonResponse: + def __init__(self, url: str, payload: object) -> None: + self.url = url + self.payload = payload + + def __enter__(self) -> JsonResponse: + return self + + def __exit__(self, *args: object) -> None: + return None + + def geturl(self) -> str: + return self.url + + def read(self) -> bytes: + return json.dumps(self.payload).encode() + + +class RawResponse(JsonResponse): + def read(self) -> bytes: + assert isinstance(self.payload, bytes) + return self.payload + + class TreeTopPolicyGeneratorTests(TestCase): - permission_table = """\ -Range Group Regex Labels -10.0.0.0/24 group two ^web [0-9]+\\.example$ Shared, Unused -10.0.0.1/32 group two ^web [0-9]+\\.example$ Shared -2001:db8::1/128 ipv6 .*\\.example$ -""" - role_table = """\ -Name Description with spaces Labels -role1 A delegated role Shared -role2 A role without permission Missing -""" - - def test_parser_preserves_spaces_and_normalizes_values(self) -> None: - permissions = generator.parse_permissions(self.permission_table) + snapshot_payload = { + "schema_version": generator.SNAPSHOT_SCHEMA_VERSION, + "permissions": [ + { + "range": "10.0.0.0/24", + "group": "group two", + "regex": r"^web [0-9]+\.example$", + "labels": ["Shared", "Unused"], + }, + { + "range": "10.0.0.1/32", + "group": "group two", + "regex": r"^web [0-9]+\.example$", + "labels": ["Shared"], + }, + { + "range": "2001:db8::1/128", + "group": "ipv6", + "regex": r".*\.example$", + "labels": [], + }, + ], + "roles": [ + {"name": "role1", "labels": ["Shared"]}, + {"name": "role2", "labels": ["Missing"]}, + ], + } + + def snapshot(self, payload: object | None = None) -> str: + return json.dumps(self.snapshot_payload if payload is None else payload) + + def test_snapshot_parser_preserves_spaces_and_normalizes_values(self) -> None: + permissions, roles = generator.parse_snapshot(self.snapshot()) + self.assertEqual(permissions[0].group, "group two") self.assertEqual(permissions[0].regex, r"^web [0-9]+\.example$") self.assertEqual(permissions[0].labels, ("Shared", "Unused")) self.assertEqual(permissions[-1].network, "2001:db8::1/128") - - roles = generator.parse_roles(self.role_table) self.assertEqual(roles[0].name, "role1") self.assertEqual(roles[0].labels, ("Shared",)) def test_generation_uses_derived_labels_and_exact_roles(self) -> None: - permissions = generator.parse_permissions(self.permission_table) - roles = generator.parse_roles(self.role_table) + permissions, roles = generator.parse_snapshot(self.snapshot()) result = generator.generate_policy(permissions, roles) report = json.loads(result.report) @@ -47,46 +89,288 @@ def test_generation_uses_derived_labels_and_exact_roles(self) -> None: self.assertEqual(report["unused_permission_labels"], ["Unused"]) self.assertEqual(report["unmatched_role_labels"], ["Missing"]) - def test_duplicate_rows_are_deduplicated_and_output_is_stable(self) -> None: - permissions = generator.parse_permissions(self.permission_table + self.permission_table.splitlines()[1] + "\n") - roles = generator.parse_roles(self.role_table) + def test_duplicate_permission_rows_are_deduplicated_and_output_is_stable(self) -> None: + payload = json.loads(self.snapshot()) + payload["permissions"].append(dict(payload["permissions"][0])) + permissions, roles = generator.parse_snapshot(self.snapshot(payload)) + + self.assertEqual(len(permissions), 3) first = generator.generate_policy(permissions, roles) second = generator.generate_policy(tuple(reversed(permissions)), tuple(reversed(roles))) self.assertEqual(first, second) - def test_rejects_malformed_input(self) -> None: - with self.assertRaises(generator.ConversionError): - generator.parse_permissions("Range Group Regex Labels\n10.0.0.0/24 g .* x\n") - with self.assertRaises(generator.ConversionError): - generator.parse_permissions( - "Range Group Regex Labels\n10.0.0.1/24 g .* \n" + def test_rejects_malformed_snapshot(self) -> None: + with self.assertRaisesRegex(generator.ConversionError, "valid JSON"): + generator.parse_snapshot("{") + + payload = json.loads(self.snapshot()) + payload["schema_version"] = 99 + with self.assertRaisesRegex(generator.ConversionError, "schema_version"): + generator.parse_snapshot(self.snapshot(payload)) + + payload = json.loads(self.snapshot()) + payload["permissions"][0]["range"] = "10.0.0.1/24" + with self.assertRaisesRegex(generator.ConversionError, "Invalid permission range"): + generator.parse_snapshot(self.snapshot(payload)) + + payload = json.loads(self.snapshot()) + payload["roles"].append(dict(payload["roles"][0])) + with self.assertRaisesRegex(generator.ConversionError, "Duplicate host-policy role"): + generator.parse_snapshot(self.snapshot(payload)) + + invalid_payloads = [ + ([], "snapshot must be a JSON object"), + ({"schema_version": 1, "permissions": None, "roles": []}, "permissions must be a JSON array"), + ( + { + "schema_version": 1, + "permissions": [{"range": "10.0.0.0/24", "regex": ".*", "labels": []}], + "roles": [{"name": "role1", "labels": []}], + }, + "group must be a non-empty string", + ), + ( + { + "schema_version": 1, + "permissions": [{"range": "10.0.0.0/24", "group": "group", "regex": "[", "labels": []}], + "roles": [{"name": "role1", "labels": []}], + }, + "Invalid permission regex", + ), + ( + { + "schema_version": 1, + "permissions": [{"range": "10.0.0.0/24", "group": "group", "regex": ".*", "labels": [1]}], + "roles": [{"name": "role1", "labels": []}], + }, + "must contain only non-empty strings", + ), + ({"schema_version": 1, "permissions": [], "roles": []}, "permissions contains no data rows"), + ( + { + "schema_version": 1, + "permissions": [{"range": "10.0.0.0/24", "group": "group", "regex": ".*", "labels": []}], + "roles": [], + }, + "roles contains no data rows", + ), + ] + for invalid_payload, error in invalid_payloads: + with self.subTest(error=error), self.assertRaisesRegex(generator.ConversionError, error): + generator.parse_snapshot(self.snapshot(invalid_payload)) + + def test_endpoint_rows_resolve_label_ids_to_names(self) -> None: + snapshot = generator.snapshot_from_endpoint_rows( + permission_rows=[ + { + "range": "10.0.0.0/24", + "group": "group two", + "regex": r".*\.example$", + "labels": [2, 1], + } + ], + role_rows=[{"name": "role1", "labels": [1]}], + label_rows=[{"id": 1, "name": "Shared"}, {"id": 2, "name": "Unused"}], + ) + + permissions, roles = generator.parse_snapshot(snapshot) + self.assertEqual(permissions[0].labels, ("Shared", "Unused")) + self.assertEqual(roles[0].labels, ("Shared",)) + + with self.assertRaisesRegex(generator.ConversionError, "unknown label id 3"): + generator.snapshot_from_endpoint_rows( + permission_rows=[ + { + "range": "10.0.0.0/24", + "group": "group two", + "regex": ".*", + "labels": [3], + } + ], + role_rows=[{"name": "role1", "labels": []}], + label_rows=[{"id": 1, "name": "Shared"}], + ) + + label_errors = [ + ([{"id": True, "name": "Shared"}], "id must be an integer"), + ([{"id": 1, "name": "Shared"}, {"id": 1, "name": "Other"}], "Duplicate label id"), + ([{"id": 1, "name": "Shared"}, {"id": 2, "name": "Shared"}], "Duplicate label name"), + ] + for labels, error in label_errors: + with self.subTest(error=error), self.assertRaisesRegex(generator.ConversionError, error): + generator.snapshot_from_endpoint_rows([], [], labels) + + with self.assertRaisesRegex(generator.ConversionError, "integer label ids"): + generator.snapshot_from_endpoint_rows( + permission_rows=[ + {"range": "10.0.0.0/24", "group": "group", "regex": ".*", "labels": ["Shared"]} + ], + role_rows=[], + label_rows=[{"id": 1, "name": "Shared"}], ) + def test_fetches_all_endpoint_pages_with_token_authentication(self) -> None: + calls: list[tuple[str, str | None, float]] = [] + + def fake_urlopen(request, timeout: float) -> JsonResponse: + calls.append((request.full_url, request.get_header("Authorization"), timeout)) + if "/labels/" in request.full_url and "page=2" not in request.full_url: + return JsonResponse( + request.full_url, + { + "next": "/api/v1/labels/?ordering=name&page=2&page_size=1000", + "results": [{"id": 1, "name": "Shared"}], + }, + ) + if "/labels/" in request.full_url: + return JsonResponse( + request.full_url, + {"next": None, "results": [{"id": 2, "name": "Unused"}]}, + ) + if "/permissions/netgroupregex/" in request.full_url: + return JsonResponse( + request.full_url, + { + "next": None, + "results": [ + { + "range": "10.0.0.0/24", + "group": "group two", + "regex": r".*\.example$", + "labels": [2, 1], + } + ], + }, + ) + return JsonResponse( + request.full_url, + {"next": None, "results": [{"name": "role1", "labels": [1]}]}, + ) + + with patch.object(generator, "urlopen", side_effect=fake_urlopen): + snapshot = generator.fetch_policy_snapshot("https://mreg.example/", "secret", 3.5) + + permissions, roles = generator.parse_snapshot(snapshot) + self.assertEqual(permissions[0].labels, ("Shared", "Unused")) + self.assertEqual(roles[0].labels, ("Shared",)) + self.assertEqual(len(calls), 4) + self.assertTrue(all(auth == "Token secret" for _url, auth, _timeout in calls)) + self.assertTrue(all(timeout == 3.5 for _url, _auth, timeout in calls)) + self.assertTrue(all(url.startswith("https://mreg.example/") for url, _auth, _timeout in calls)) + self.assertIn("page_size=1000", calls[0][0]) + + def test_rejects_invalid_endpoint_configuration_and_responses(self) -> None: + invalid_configurations = [ + (("mreg.example", "secret", 1), "absolute HTTP"), + (("https://user:password@mreg.example", "secret", 1), "must not contain credentials"), + (("https://mreg.example", "", 1), "must be a non-empty HTTP header"), + (("https://mreg.example", "value\nInjected: header", 1), "must be a non-empty HTTP header"), + (("https://mreg.example", "secret", 0), "greater than zero"), + (("https://mreg.example", "secret", float("nan")), "finite number"), + ] + for arguments, error in invalid_configurations: + with self.subTest(error=error), self.assertRaisesRegex(generator.ConversionError, error): + generator.fetch_policy_snapshot(*arguments) + + def fetch_one(response: object) -> None: + with patch.object(generator, "urlopen", return_value=response): + generator._fetch_paginated_rows( + base_url="https://mreg.example", + path="/api/v1/labels/", + token="secret", + timeout=1, + ordering="name", + ) + + invalid_responses = [ + (JsonResponse("https://other.example/api/v1/labels/", {"next": None, "results": []}), "response changed origin"), + (RawResponse("https://mreg.example/api/v1/labels/", b"not json"), "returned invalid JSON"), + (JsonResponse("https://mreg.example/api/v1/labels/", []), "must be a JSON object"), + (JsonResponse("https://mreg.example/api/v1/labels/", {"next": None, "results": {}}), "results must be a JSON array"), + (JsonResponse("https://mreg.example/api/v1/labels/", {"next": None, "results": [1]}), "must be a JSON object"), + (JsonResponse("https://mreg.example/api/v1/labels/", {"next": 1, "results": []}), "next must be a URL or null"), + ] + for response, error in invalid_responses: + with self.subTest(error=error), self.assertRaisesRegex(generator.ConversionError, error): + fetch_one(response) + + initial_url = "https://mreg.example/api/v1/labels/?ordering=name&page_size=1000" + pagination_errors = [ + ({"next": initial_url, "results": []}, "pagination loop"), + ({"next": "https://other.example/api/v1/labels/", "results": []}, "pagination URL changed origin"), + ] + for payload, error in pagination_errors: + with self.subTest(error=error), self.assertRaisesRegex(generator.ConversionError, error): + fetch_one(JsonResponse(initial_url, payload)) + + http_error = generator.HTTPError(initial_url, 401, "Unauthorized", {}, None) + transport_errors = [ + (http_error, "returned HTTP 401"), + (generator.URLError("connection refused"), "Unable to reach MREG API"), + ] + for error_response, error in transport_errors: + with self.subTest(error=error), patch.object(generator, "urlopen", side_effect=error_response): + with self.assertRaisesRegex(generator.ConversionError, error): + generator._fetch_paginated_rows( + base_url="https://mreg.example", + path="/api/v1/labels/", + token="secret", + timeout=1, + ordering="name", + ) + + def test_cli_fetches_and_persists_current_api_snapshot(self) -> None: + with tempfile.TemporaryDirectory() as directory: + temp_dir = Path(directory) + snapshot_path = temp_dir / "fixtures" / "policy-source.json" + output_dir = temp_dir / "output" + snapshot = generator.serialize_snapshot(*generator.parse_snapshot(self.snapshot())) + arguments = [ + "--api-base-url", + "https://mreg.example", + "--snapshot", + str(snapshot_path), + "--output-dir", + str(output_dir), + ] + + with ( + patch.dict(generator.os.environ, {"MREG_API_TOKEN": "secret"}), + patch.object(generator, "fetch_policy_snapshot", return_value=snapshot) as fetch, + ): + self.assertEqual(generator.main(arguments), 0) + self.assertEqual(generator.main([*arguments, "--check"]), 0) + + fetch.assert_called_with("https://mreg.example", "secret", 20.0) + self.assertEqual(snapshot_path.read_text(), snapshot) + + snapshot_path.write_text("stale\n") + with ( + patch.dict(generator.os.environ, {"MREG_API_TOKEN": "secret"}), + patch.object(generator, "fetch_policy_snapshot", return_value=snapshot), + ): + self.assertEqual(generator.main([*arguments, "--check"]), 1) + + with patch.dict(generator.os.environ, {}, clear=True): + self.assertEqual(generator.main(arguments), 2) + def test_cli_check_detects_stale_output(self) -> None: with tempfile.TemporaryDirectory() as directory: temp_dir = Path(directory) - permissions_path = temp_dir / "permissions.txt" - roles_path = temp_dir / "roles.txt" + snapshot_path = temp_dir / "policy-source.json" output_dir = temp_dir / "output" - permissions_path.write_text(self.permission_table) - roles_path.write_text(self.role_table) + snapshot_path.write_text(self.snapshot()) arguments = [ - "--permissions", - str(permissions_path), - "--roles", - str(roles_path), + "--api-base-url", + "", + "--snapshot", + str(snapshot_path), "--output-dir", str(output_dir), ] self.assertEqual(generator.main(arguments), 0) - self.assertEqual( - generator.main([*arguments, "--check"]), - 0, - ) + self.assertEqual(generator.main([*arguments, "--check"]), 0) (output_dir / "netgroup.cedar").write_text("stale\n") - self.assertEqual( - generator.main([*arguments, "--check"]), - 1, - ) + self.assertEqual(generator.main([*arguments, "--check"]), 1) diff --git a/scripts/generate-treetop-policy.py b/scripts/generate-treetop-policy.py index eff090fc..f6bb3b84 100644 --- a/scripts/generate-treetop-policy.py +++ b/scripts/generate-treetop-policy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate TreeTop policy data from deterministic mreg-cli table output.""" +"""Generate TreeTop policy data from existing MREG API endpoints.""" from pathlib import Path from runpy import run_path diff --git a/treetop/data/mreg-bundle.tar.gz b/treetop/data/mreg-bundle.tar.gz index 45d5573d51cfc5f0a2b951b8026453fa7baaa782..62e700e0d52706b20d38eca37e04918fe92189b7 100644 GIT binary patch literal 5268 zcmV;F6l?1riwFP!00000|Lt9UlbblwpTFl*u&%4CBEGkvank=s9QbFC zbLeo4hC}`f$M)~Y7!>!i>xMow%rpJ$8yxF6A(5t?DSF)XNR)UB;v{iz_S;=AB*|rO z)(eBB=Xf;kJJcmn?^!RtB&IcjJA;9nUzYUsA(*rSO% z9@@G!nHtmSfLarxkESEX8Pn;Q)#ZJqy_soFri1%uMU$~}NmqpHVYR)HGa4+2s~g0! zM~i{s*n_daJssY->C;2NP<6Mw>&+of5+~^u@w4fYKi6 zgR+Cekukmh@~mgCeRoOco(q+mi;}DCC24hF3bYcvkpSn9T2hL!$FfG&3 z$1t9{Y0>G>p<|0Uqlslqq4Gr#tw=JzqEYMt=QHD3?}GX?B1sVSW_P_*Vzb_`Z0KkKamcfF*kLU^k%h`4e2k4q zmAEU<-@SYp#K}bztiw$LelmBWwVPk@X>uJzzt4B?Y*o>4FSv$*p07g}`qFfFMOQY+ z=F$tR6xqP+R_^&%Wa+u{0P6L_$h-2E^a2LT_2c=qw{#s66|_49AW>+F%dd_k0>d78 z4jcE9o0NrJE;q-fz8BRIlIZ6yNwrt4L}kw!$ug~{E@M6A%nRpZbK%;6B08~#EtHH` zroXcEP_sUAha;N~C&Pg)Q?n#J)I6Bl0Bq^VshdCF?PFPUJp4F+__W;7w9&D2VQN(!Kd5?D87*bwjG@YfYpk7qgL zErGn9+Y{6sO-7v!kn^ZwZ`0;g+r4t5&y{dgUEFeWR3Y-IJMqsU9Q?lF(oXQzV8xzE z8No02=Vy4F{INOFJA0I)eg1C^Ez8=O|62o`|9^|)#S1N)20l+`eKU<2gYPvqNuFs8 z`>AT210}noQQr*%&rdWKZ3L0`l_W3^?Q5@zbE$<IKVf+-DdN;0%24Kk zLlUgI(DM*hhF(Aey5qW_bi!`&^p=n)a6}}l8xaLpJO~~Dx?<8W5fjl;9vAdx&H)t1 z66%-7TlB_Ru3Z)&5&we9mMSC*F_pbe_Ake*S=Z8%)B-X#x&g=`@?0SU5wo>1?F#XY zVglMYx)PGamkS~AlA8<{Us{dsRs_4Ju`g?>aFw^Hd2vSnhE0b&=ltpUHTVYYZjX(<#?;F@p1MMdMnP<#+U?>5WUAiFP|DN-5&pCE_2kvD9r`vOh z?mY9-kIOhj8Xq%)SX&XFz}!i*Z+Ya9XNS_Dg7e|Qq>5$JHD)sUi`;2lslEzUpn zW6)1Fv2Ax7`SiD)iosk=Rd`OhrVDam?WOY))?AVbdsdPP8AyX9VdE*8u;G+UJf>V* z#be3I#B$_h!bX{A7dFX=gbgwxVRH~A2wMXhVPileY%2{Sgw3U7!WL69iAKvHN22MH zSb~9)SjfhPgBS9#K`mrtgIdT*>bQk0q-4UjQ!;U*5KxGl1R`;RKqPFA#T~-t7?H3! zMkH*moa+f2ElGt*Pgy1VL{WMBGK0hkWk!S(%80CPLMqM4 zRyk?Lw#rHKnN3Ee`OD~~`O4_K@>5Q&yK+{R?8;+Va(`~qM8N($r{(wOJT3p9V)mts zyp+YMefD_hsi|n*mNFpMtCUvGU8S^AW15nd&mHp?zBJ96{{n^hC^Qb}#>3Vu>Od%`&T~U7HR?QIYK8^1V$g-ZjQCUsj%VD!xLJpzT5^}b$`jDU#SG+_gu6PM4 zuBruiUz)U0rUK4_&=Te8x=)uhhNpUL6)M~dmjv_=@(`dK@XRlHo>mOF^*yqTnZ9IC z37Pg?Q4JntZY1ucR&zW(ho@Jz=kEKY?vTm4!{(_Dn}9l8X6aB#qeCZz9xK!Hn8}+1 zr(+J5XX%-i!_X&wLI4wyWxwxB5Ie?fLPO-X_@dGNsq~F89N~f;?IG*z%@c zPWRmt3ID{4WL~Lg;mLRMe9cok<Ey$eUS7Gpuy8tsR(WCHi{JVmSt9U> z7e473>_sp5_hMjzgD8BAuz=CEfZ4UcpmPhB)uo8hwcAG5ZW~>@ZFKE6XUQDP7nz!U zV>Dsq_9$cQQO4Y(%wV5(Itbs?bz|n*liBQrVNSZ1G4?5B?o()x6*@IDU-R@OHS#c5 ze|;0OHROp_&h9hA&F1dRjZs$vE3Y~07_ta2cqPU9;COd|jYI`bj%Ibf1NCZkGbNGnXxLaV<@1M*)H93`-o9CAww)A0(@; zOVhR+mr<>@caa_?L0ZrD1`9HLx4$UZo%5R6tAOnq4rCc`dl7O!)^}oH3FddWXk=gT zmPYb4nJ&qh7FrPx?Ni!LX`D7< z5gQFDAGwTAEtRmvy0o}VR=kEr)K+Z@syPGIN+W8kW#M5XrKsJVWrvTKBP)E7wOCNg zHby4=lrx+?QY0|DH2Z@A%$dz0n~rTIw4}EMcvNTDkR+GOk=IhyOR8LC@QP7AzTm{8 zy}hY|{5U5xi_tOWz?cJL4vaZ4=D?WaBw~*Iu?&wmx;)S@t?rKkq#Mb~m>nJkRFdtZ zp>z=wicIq&T>`Fe76_;vZ*6Q%B|ps9)aJ>p7~?IBw=mwqcnjk#jJF=*x<=Ht1{0mw zWN17n-rA0ay2Kpc%o+X{#T=W06AyOnUX+iUP-}hUtOu$_2w5-6mt&i>>zZ)4xt zM>WH0Yu=WtY!p!I2G0*S-K|Sk+TJqT@^wW$e*y9jy!b~b4fU}~jNX68qp8-txgTt% zQ5Kl8lT6vQZsh)8tNr+j2o3lC^JX$NYsS)jyYp3J*)HDYnlXN!uL=8|h6$5_m+-f~ zwe;&TT3VH6Q-`c$m^Cn(gl&x*_++`f)kqjFJsPw6h!X{IzN2sPX$8K4Zz{TM<*&?c z;*DDJL+z^Qs)B60!dyukb~o?7{I9yIy1n7E(>p3(T{?TLqjmr9fj%4=#?JoV`q0At zzrVq8mwvfvcGpXlHs*g;SEI7?6x5#rc2XeX({0GV5!GYR#4dDekZusk#T{v7ZRN!-AHe`PsS{=1^T@Z=?&d`P1ASq>4-eqotC z_8VQn|DSGH?LWSJ1(x>hPptAh(k@uY&flf`eX;hKm1d2kVQ=}@hy0r^8|N?cEp{4j z{6E`zCu{FbY8!IO*V*Pw+=9d9|4TQ`{#)R0o&8j7$_v*&ru*(@RF$(0s?vGAOoG+M z1?J6CCOAJI*2Ew{M9C{BdGz+GN*+DEnsP@kub||&pJLBY`YK8vy}ioPM~|Q%*=4x%Qcnt-W7)CF`4Dk}i`0;R=|4nREx*w``*C9j<1(dVlw zd35$_${oGDf|3vTCZ_qKpjA3P==M<+a#g@M097GV1q=*O5x62?q@b)SY;KN@8lbEO zU|gWO24Fa#ru@Ih<>HTkHMb8!pD4%qT&xZ`J6xn=>Jg( zRu~1K609&5Ko!7cH-IG*J;0Q-nMvgw_C+fX@PWau5vz>O@5+c({lzZz;u@u|uJrwsNmK!KRR9FS2H8a&P+A9|3r>Psps*HXBEdxjN=qMC52!4Cd~~R; z>@f?Zp!DN;ybfu!W+Fw&D~tGj`5v>k`Em8@rueFJ-nK7 zNB^#()X}@EC->D2I(Ie2j=o(8OQ0VB{)e}3D!kn+miqOR?E`Rj(YRey;zRJ=^ z@2{f#VXV>bD=T^Q_^L`CeZ87;M=!6S&?=xRWU7GkL{tQ>2slSn zR~15DVovhjV61@3P$~mP3#bf*GGM%bs*tGyMhvJ7g)&^R?qJ-2DzU8+7(Jj$Y^ems z5U3Cv3W1S?@+yImgYqhXk%ICnfRTaX3V@M-x+;L+S5@+edi7+_-e?N4amDCgbp^oK zUvULM=dZN*F#=Fe0iv|p`B$hHBLLMEz%439T|iwZbO8fH)P-CZFh)d8NHqb&Mbrg! z3o0uB`U0iJj}AaR1&E_7oEj=Bcl7sa${pRjic&}KuAto6K%#$FR`TfKRh2yYcs1pY zUS2`TuR|KGL7uoHrGnB=ZW4^}6qP&1cWTNVqdOI)jL)NRnFR03L4?2KOA@9oAnzk5acS&CUdY=2weS*LnzP=ILBIwzqW@fb z5d|BvAj_D7OSxq=^#A(5p{cE3XwoZR_IwzZ!8KTjck3io-v=m8NRrEkvBD^6 zoY3fP{GM_35N=|+T)Zq2BKy^HU)=dfogi}gl~=i9AQ%yh2*x9Wu?Q;=kO)WwFN%IBtjA)c@mJ!rrxJ5soJT z$Ib1!5Sxfi#OA4BGuxdHL5ZM5P@Wbj%Pleyorq3E=ZT^--KQ2IiV#JJ9v!09U9Az4 z2uXzG(IHvgb{!#!kVHtH5+rd?NdzZ?^OV5JKEZ-eL?|K@j}JxuRU`x>f)T-ZbTG1y zk|7ikiU`G{Ly>>45UO0!l})4fi%+-t_X_z=S?oi2_vdHFIrujh%b!1g?mxX7JiF&7f7q^N?Yoel0l3jh*?E-;%-#jV+T2Y<{3?~lg0SENQm#?78w?E;i9&`$|xz}%(vHg}k a`EslbBp!I+fd?MPIQ|VA@!6#S5&{4XRz}wV literal 5253 zcmV;06ng6)iwFP!00000|LtAtlH10y-rxHaFe{akk`+bpPP$y@$g+;2Dl2ly^7#=< zVc{~|MN9$+0Pe+`efRVXuDs5G%N3c-2UUp+pl7-<-CuW40~pLL-Tlod6z8(zW~*zq&st|+VOzwB6&c2vrrUkPiV|;XIZ50Y|9amDt>mgP zZiKGNj^3&8kfvxk(C5dV|?F8H8yT^ zn`XPohdYTFOM-Bc%0XvZ%~rElIpG9RezxG71*qGJ)8cIMKq*kN7OP38LR7>t|NF*m5qop@E((LYJ;( z+u1pr+oYQ-FRW5z1GC<_=U-bh&z%I+uOCLsV1iyGB47C00Uz zn@A#Z?2+g2aWADwMOc?|d2BXcMRkM}u5+EFdR04#$}8JSmRUV>x#+26UN{-F$p>|u z4jcBmJ1DuV+?cQj> z%t2p)d@wW}%jyq0LraC+ZgpL=-L)8VoOSYwO+*2C!O^hScdfS5?v7kJb=w} z%Ow95-0=S=DJiwdsOAy7oCwm?FO@Q7y+4#faM~>nbj@dML&Ppus|YG3fF4R=^^9Rj zy@$hJT~s}t=TP?q>V9rdP}g)Bn|y#;M3qOIT~T#+RNfkM6#`YAu)IC1kj2!s{O1q> ze&2LyE&1Bw#V(Yb;FpK2$o@CoS|L=8so&MVVzt=G_|Njcx%a=wr4SbQ# z`o;!#2HzTdk~}sz_EXb1CrWcPpWJXw!IOIafv)1qtK+s3eQ& zuh0t3vyTnjkxpMx4gqo#QIYZ8jPIng1mQ3f3fJ!m@#W=c)nqbF2pWXLFiJ9 z@4gcp#^YCFe>`q%POL;5;v6g@hn+wFw#vQm>`ELxU+nSi*A>>b{p$cW5I3&Tm4!uZWRp=!&WIL`)Mkl>2QQndg1&)klek-F8hDXT*L03*1 zCUPQH>f@5#P6UABSi$`Ae2d*WvxUnOB+I{Ks;vsmQch(rlKtCpZ&tUpBDI8!k8VJE zh&)#+K*W7*%<4jXshNZ}j;^I7@zqocyyP~+#h0BO7TBZ!T;Zv)pUdV z>}EBkC?|La)Xk3IpT;q|PCl{S9X9gm?`|p%b2(KJIn`JfXb3G~3l2ppNB$d0zVxlE?kp-#TO%|l8;b(Qary7S=+o=V` z^>%XaDkfKQ|0>Aj9#)XaUC{Pq=I(tSe*La(`*lLcsnq zr|tKbJZ=A<;`WuCypqS+c<=GhQ&ZWzEfqlRs#03DaFx<3jcH0+-gnG7eAzZ@{tGn{ zqp)>Amu|N1Sz+0B$vfrQT4u>^E3oKY{DBL%RBYLBa=iza#ZKgfT)JO`JWE}yon0y0 z?6_6^i1FE`y5o+NxWe~~L{tiS^KXjwOQ$vr(fZx^`hYC^*&CJB^1T{1t1Z+JT5X}` z`>HnyYDpzp)RIcHP?DVudAaV>6^rSv9)Ajz&V@6J?2qaYWE*tP zFL{x6Ot|$uYZh~R$?g(z>$_$K-O5}l+$lZH@%)_bUfG_z@143s7V8dMr#fr_>Ts2% zLlunpcA3tj>gFSCk zMxAf+IQ$FglV2Wxys44Xee+BqK8Yq-R4O<;`$k=^N$RIu7A9BJ!@vHO4w({j>gPq? zPcFmr)muzE9x#Ldga3p7J4LTm1?kuGTgpzS6ff^$m)BW-X7!Yu&ePnTe7w=C8<@XnJ=&Oiv}y0rrn3*98p?0CTISfbhvV@}vpuZa#@we- zd!I&~tkJ2t`3-kpQYTMy^*6U6e}+6U%GrJ9xcS_jhcV_F99>-DcV{MRobBrq9&q<$onlnGyO;4B4Q#rRzgFKk4Ho3e} zCbgr$qUj7T9F;BVRsJ8PtGbV-tuj|JtzGXTJxPIdv)D@_sK~whMZx-#Z?$TQn3NIhl7_h~Bes_o`AeGYl9d{3R6K#L|aVNE`?;X<%_1VQ% zO6iqtu7RkPKsHjzIBu+ysFX=I(8zdgs-!3sQr1aijJA|o)G93-spK4Ylw;~tV|LKW zWY|52tyQ<#MJy9zON~dZ#Iun~%5hU2T%`=Yfkwt_W6f!M$!RBzjMt7Q4YyLt_^v%? z_;f|FA{Of6o1G3#lG|NA|5~YFDX#qopVrD3zTy}zL#MxYMH}Uc$ zP)b{!{i$2t(n7OO>W1~Uyp>S7RY5yfbbh?+-ubM`uGh*|VqMYCUx@rC(flK|rv7** zX5+u(*3{0$xF7ANp$nAk6iaqHS8ji_*M53KB*u;ZyqU~~hs4r_y7R}xvSqu=hs4Cb zk-bd&8w(Sz11}M;c^m2HVvMva!)A`Ph-ucqX%fB`Zs1$9*_~m9;ml((@6U3gAkLTY zExx57R_skxmp$sMvV8bfEBS8YW6)ItSq+7`k#1Sey#Dik)yJrJGd?@{t@72Sv!~j2 zuK(R>b^E5dw*Ggk+r#?bUtzmXzuYvwZ=^OHi$AMRqw@U};0Ns;z`=7m1~8(q`?pKf{Y-@kfIj`!UUyz@NqE=0)o->2(+aemxsb0%pP zT>kY_{!N#q{a5)CJ6liuKVNz$FLVwI(8vPTs>aa1mfFt> z3(&rn+80Y}L-$(hKASB=`x8>L*ubt)*?RC{WBE6n^N0ir4^Q$+t$I!lx+DEk4 zS^J3i`syB0UqkJSSU(XTnSthY(tH}{p?M88zhikR7=mVouz8U*n1M!Sz}I&}9MDhi z)0BDov2GN7G&caZ{Kz|O0k)9a0?L7~h1?cU9E2sLmViIeF@RzH$Gjg^mjkEZGm z1^L9H4iNug1ao8oFoHR90hjGUKl&Li~09IEDkd;sQ9F$W|G zFowbykU3~+4qQU4@}#{5AW_s;_s9zM)ji8ZoJ3qkxB-|#ZVK=UFonz%Fi(Ub2t&Xe zQD0LCMT?1*Uk!2v7(;0c$QED>g)tysfGK3AfQ$jgP#D8CzZ~QZFo{)@K=uHWSTPCY z5HN@(gFq&sy-6T*(B1@)DQIs3$P6?$0AvFCngGDBtLA}v{dCXYXbQ4%MfR_|0U-C+ z+yD^#wN^he0R0RgO1qu>PW>VS(A@ysqGQ+sY@x6PB!;ks+!l}{!V*$TK)MK9Kv>Y( z01ykbRzCuOeg+Un*O(e=s&~YDJ@t+-ucOux-8IxZA4tS^Z8eVwudC(}HgYdHm&H!xg`EE{MU)WQ95=oX0zRk_`{%-o82ON? zHH#UAlsn#2#TV35fj_Nt?DU`uLfc2^FdL;<=A(A6D9X!b)qk|IPCsDZ%5% zfVUJzEtEtiGAGq;>T+uj+j#xVl1BNm=i{UdZpcHtUnHsdKEiN9E4g}{D4deT35(vv zAGlPHktSxd>8qk3vR^L`#hH)H2_jbuzz8q`j7J7zkyZeZ03-l;bU;=UX+RQ? z1SC%hl9hQXzzJ{yoJR*|UY`RT0Y|{`_;3_u;_>}MUIqs)flJ`>=(rR&uYg5h5m-D4 zEUw+Y1vY_AVDm(pH>2p03-l;0)VVMp9fR|l|bbQp;9~~3seG? zK;_XG#mbE_sPrwuKJOy~>gAOPP@Wbj%Ox^_PM{O$JW+I}>(m0GfG8k(bcj}0wFV>sNkH=GkgP7d4oCu$ zfaEDb5^G8VoB-!3fs=oN1yBSO0mb7(QG68%U<4Qe#-oFgf0PVR1QY?qqeD@AuaLT& zv$@Tp57SR~`S%LNN?H6vcn{}i$JxX;7vttX;=n(9s;xy^zuOhRw5|M_X3Om4$Fk#Q ztJUeBwa&i6wuloeGK@1#w_ooYQ;*HauuC_d`@+Wik8j?N$Jw_ugGCZ9$Qf0glz=dy z&*$b`V8-?1$&EL29g6_Kv2)38>G0>ze53se*pTL* z$HkByY)I2D4P!tLHlX68%^1&vjc5M-YYgcDhqUr3ag6K1#&!9t>=@XC4Q&2_dkpE} zhLmnpVL%T!puEC~0X^V=e*fw