From 488d4ba4968732e617698da5ef2a340f5570ff5e Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:39:19 -0400 Subject: [PATCH 01/17] Point development setup at the groundbolt-dev workspace The README directs contributors to the groundbolt-dev workspace for the development environment, links it wherever it was named, and marks the webapp-only local setup as the exception rather than the default. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 168df55..0cc9327 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Run the driver as its own service, then register it from the running ground app **Global Settings > Meter Drivers > Register driver** by entering the base URL of its HTTP service. Registered drivers become selectable per meter on the meter form. -The groundbolt-dev workspace metarepo runs `sparknet-http` as part of its stack for +The [groundbolt-dev workspace](https://github.com/EarthSpark/groundbolt-dev) runs `sparknet-http` as part of its stack for convenience during development; that is a choice of that stack, not a dependency of this application. From e14ac9072da11ef7d4816f2b25a51e3972b43b83 Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:52:01 -0400 Subject: [PATCH 02/17] Post driver init to /v1/init and stop deriving a gRPC target from the HTTP host HttpCommandClient.init_driver posts to the spec's /v1/init route. GrpcCommandClient.init_driver raises a ValueError naming whichever of heartbeat_period_duration and aes_key the payload lacks, since the ConfigureDriver message has fixed fields. _grpc_target returns only an advertised or saved target; build_command_client and build_event_client raise GrpcTargetUnavailable when gRPC is selected without one instead of guessing host:50051 or silently falling back to HTTP. Docstrings now point at the meter_driver_spec.grpc stubs from the meter-driver-spec wheel. --- sparkmeter/metering/runtime_client.py | 86 ++++++++++--------- .../metering/tests/test_runtime_client.py | 78 +++++++++++++++-- 2 files changed, 117 insertions(+), 47 deletions(-) diff --git a/sparkmeter/metering/runtime_client.py b/sparkmeter/metering/runtime_client.py index 477488c..f2a8bb7 100644 --- a/sparkmeter/metering/runtime_client.py +++ b/sparkmeter/metering/runtime_client.py @@ -4,13 +4,13 @@ 2.0 Open Source Meter Driver Specification — not against any vendor's package. Any driver that implements the spec's required HTTP+SSE contract works here with zero vendor-specific code. A driver that additionally -implements the optional gRPC profile (see sparkmeter/metering/proto/) -works over gRPC too, using the client Thundercloud compiles from its own -spec-owned .proto files. +implements the optional gRPC profile works over gRPC too, through the +stubs the `meter-driver-spec` wheel compiles from the spec's own +meter_driver.proto (`meter_driver_spec.grpc`). -SparkNet-Http gets no special treatment here: it's one compliant driver -instance among however many a deployment configures. Nothing in this -module imports a vendor-published client package. +No driver gets special treatment here: SparkNet-Http is one compliant +driver instance among however many a deployment configures. Nothing in +this module imports a vendor-published client package. """ from __future__ import annotations @@ -21,7 +21,6 @@ import uuid from decimal import Decimal, InvalidOperation from typing import Any -from urllib.parse import urlparse import grpc import httpx @@ -43,6 +42,17 @@ logger = logging.getLogger(__name__) +# The gRPC profile's ConfigureDriver message has fixed fields (spec section 7, +# meter_driver.proto), so the gRPC init path needs these two whatever the +# driver advertised on /v1/requirements. The HTTP path has no such list: it +# posts exactly the discovered fields. +_GRPC_INIT_REQUIRED_FIELDS = ("heartbeat_period_duration", "aes_key") + + +class GrpcTargetUnavailable(ValueError): + """gRPC was selected for a driver whose contract advertises no grpc target.""" + + # Lowercase behavior verb -> the spec's ElectricalMeterCommandName. Verbs with # no spec command ("none", "enter_unprovisioned") return None; callers skip # building a configure command for those rather than guess a wire value. @@ -116,10 +126,7 @@ def __init__(self, base_url: str, client_id: str): ) async def init_driver(self, payload: dict[str, Any]) -> None: - # The spec's own worked examples show POST /v1/init; the reference - # driver (SparkNet-Http-New) actually serves this at - # /v1/sparknet/init. Targeting the endpoint that's actually live. - response = await self._client.post("/v1/sparknet/init", json=payload) + response = await self._client.post("/v1/init", json=payload) response.raise_for_status() async def register_node(self, req: RegisterNodeRequest) -> None: @@ -150,8 +157,9 @@ async def close(self) -> None: class GrpcCommandClient(MeteringCommandClient): """Command client backed by the standard TC 2.0 meter driver gRPC - profile, compiled from Thundercloud's own proto (sparkmeter/metering/proto/) - — not imported from any driver vendor's package. + profile, using the `meter_driver_spec.grpc` stubs the `meter-driver-spec` + wheel compiles from the spec's meter_driver.proto — not imported from + any driver vendor's package. """ transport_name = "grpc" @@ -161,6 +169,12 @@ def __init__(self, target: str): self._stub = pb2_grpc.MeterDriverControlStub(self._channel) async def init_driver(self, payload: dict[str, Any]) -> None: + missing = [name for name in _GRPC_INIT_REQUIRED_FIELDS if payload.get(name) in (None, "")] + if missing: + raise ValueError( + "gRPC ConfigureDriver requires the init fields {}; the stored driver " + "config is missing: {}".format(", ".join(_GRPC_INIT_REQUIRED_FIELDS), ", ".join(missing)) + ) request = pb2.ConfigureDriver( heartbeat_period_duration=int(payload["heartbeat_period_duration"]), aes_key=_aes_key_bytes(payload["aes_key"]), @@ -205,7 +219,7 @@ async def close(self) -> None: class GrpcEventClient: """Event streaming client backed by the standard TC 2.0 gRPC profile's - SubscribeEvents stream, compiled from Thundercloud's own proto. + SubscribeEvents stream, through the `meter_driver_spec.grpc` stubs. """ transport_name = "grpc-stream" @@ -236,37 +250,37 @@ def _selected_interface_details(provider, provider_details): def _grpc_target(provider, provider_details) -> str | None: + """Return the grpc `target` the driver advertised in x-meter-driver, or None. + + The live contract's selected-interface details win; the target saved at + registration is the fallback. Nothing is derived from the HTTP base URL: + the spec has no default gRPC port, so a driver that advertises no grpc + interface has no target. + """ selected_details = _selected_interface_details(provider, provider_details) target = selected_details.get("target") or selected_details.get("address") target = str(target or "").strip() if not target: target = str((provider or {}).get("selected_interface_target") or "").strip() + return target or None + + +def _require_grpc_target(provider, provider_details) -> str: + target = _grpc_target(provider, provider_details) if not target: - base_url = str((provider or {}).get("base_url") or "").strip() - parsed = urlparse(base_url) - hostname = parsed.hostname or "" - if hostname: - target = "{}:50051".format(hostname) - logger.warning( - "provider %s selected gRPC but no target metadata is available; " - "falling back to derived target %s", - base_url, - target, + raise GrpcTargetUnavailable( + "provider {} selected gRPC but its contract advertises no grpc interface target".format( + (provider or {}).get("base_url") ) - return target or None + ) + return target def build_command_client(provider, client_id: str, provider_details=None) -> MeteringCommandClient: """Create the command transport for the selected provider interface.""" selected_interface = str((provider or {}).get("selected_interface") or "http").strip().lower() if selected_interface == "grpc": - target = _grpc_target(provider, provider_details) - if target: - return GrpcCommandClient(target) - logger.warning( - "provider %s selected gRPC but no grpc target is available; falling back to HTTP commands", - (provider or {}).get("base_url"), - ) + return GrpcCommandClient(_require_grpc_target(provider, provider_details)) return HttpCommandClient(str((provider or {}).get("base_url") or ""), client_id) @@ -274,13 +288,7 @@ def build_event_client(provider, client_id: str, provider_details=None): """Create the event transport for the selected provider interface.""" selected_interface = str((provider or {}).get("selected_interface") or "http").strip().lower() if selected_interface == "grpc": - target = _grpc_target(provider, provider_details) - if target: - return GrpcEventClient(target) - logger.warning( - "provider %s selected gRPC but no grpc target is available; falling back to HTTP SSE events", - (provider or {}).get("base_url"), - ) + return GrpcEventClient(_require_grpc_target(provider, provider_details)) return HttpEventClient(str((provider or {}).get("base_url") or ""), client_id) diff --git a/sparkmeter/metering/tests/test_runtime_client.py b/sparkmeter/metering/tests/test_runtime_client.py index 631e8e7..057c9df 100644 --- a/sparkmeter/metering/tests/test_runtime_client.py +++ b/sparkmeter/metering/tests/test_runtime_client.py @@ -23,13 +23,22 @@ def test_grpc_target_falls_back_to_saved_target(): assert runtime_client._grpc_target(provider, provider_details=None) == "127.0.0.1:50051" -def test_grpc_target_derives_default_from_base_url_host(): +def test_grpc_target_prefers_advertised_interface_details(): + provider = {"base_url": "http://127.0.0.1:18080", "selected_interface_target": "stale:1"} + details = {"selected_interface_details": {"type": "grpc", "target": "h:50051"}} + + assert runtime_client._grpc_target(provider, provider_details=details) == "h:50051" + + +def test_grpc_target_is_none_when_no_grpc_interface_is_advertised(): + # The spec fixes no gRPC port, so nothing is derived from the HTTP host. provider = { "base_url": "http://127.0.0.1:18080", "selected_interface_target": "", } - assert runtime_client._grpc_target(provider, provider_details=None) == "127.0.0.1:50051" + assert runtime_client._grpc_target(provider, provider_details=None) is None + assert runtime_client._grpc_target(provider, provider_details={"selected_interface_details": {}}) is None @pytest.mark.asyncio @@ -273,7 +282,7 @@ async def test_all_endpoints_and_close(self, monkeypatch): calls = client._client.calls methods_and_paths = [(method, path) for method, path, _ in calls] assert methods_and_paths == [ - ("POST", "/v1/sparknet/init"), + ("POST", "/v1/init"), ("POST", "/v1/nodes/register"), ("POST", "/v1/meters/configure"), ("POST", "/v1/nodes/7/balance-and-flags"), @@ -370,6 +379,27 @@ async def test_init_driver_omits_channel_when_absent(self, monkeypatch): request = holder["stub"].calls["InitDriver"][0] assert request.HasField("channel") is False + @pytest.mark.asyncio + @pytest.mark.parametrize( + "payload, missing", + [ + ({"aes_key": "00" * 16}, "heartbeat_period_duration"), + ({"heartbeat_period_duration": 60}, "aes_key"), + ({"heartbeat_period_duration": 60, "aes_key": ""}, "aes_key"), + ], + ) + async def test_init_driver_names_the_fixed_fields_it_is_missing(self, monkeypatch, payload, missing): + # ConfigureDriver has fixed proto fields; a driver whose /v1/requirements + # omitted one cannot be initialized over gRPC, and the error says which. + client, holder, _ = self._client_with_stub(monkeypatch) + + with pytest.raises(ValueError) as exc: + await client.init_driver(payload) + + assert "gRPC ConfigureDriver requires" in str(exc.value) + assert missing in str(exc.value) + assert "InitDriver" not in holder["stub"].calls + @pytest.mark.asyncio async def test_command_methods_delegate_to_stub(self, monkeypatch): client, holder, _ = self._client_with_stub(monkeypatch) @@ -440,9 +470,36 @@ def test_build_command_client_uses_grpc_when_target_available(self, monkeypatch) client = runtime_client.build_command_client(provider, "cid") assert isinstance(client, runtime_client.GrpcCommandClient) - def test_build_command_client_falls_back_to_http(self, monkeypatch): - # gRPC selected but no target resolvable -> HTTP fallback. - provider = {"selected_interface": "grpc", "base_url": ""} + def test_build_command_client_uses_advertised_grpc_target(self, monkeypatch): + holder = {} + + def factory(channel): + holder["stub"] = _RecordingStub(channel) + return holder["stub"] + + targets = [] + monkeypatch.setattr( + runtime_client.grpc.aio, "insecure_channel", lambda target: targets.append(target) + ) + monkeypatch.setattr(runtime_client.pb2_grpc, "MeterDriverControlStub", factory) + provider = {"selected_interface": "grpc", "base_url": "http://driver:18080"} + details = {"selected_interface_details": {"type": "grpc", "target": "h:50051"}} + + client = runtime_client.build_command_client(provider, "cid", provider_details=details) + + assert isinstance(client, runtime_client.GrpcCommandClient) + assert targets == ["h:50051"] + + def test_build_command_client_errors_when_grpc_selected_without_target(self): + # gRPC selected but the contract advertises no grpc interface: no + # derived host:50051 and no silent HTTP fallback. + provider = {"selected_interface": "grpc", "base_url": "http://driver:18080"} + with pytest.raises(runtime_client.GrpcTargetUnavailable) as exc: + runtime_client.build_command_client(provider, "cid") + assert "advertises no grpc interface target" in str(exc.value) + + def test_build_command_client_defaults_to_http(self): + provider = {"base_url": "http://driver:18080"} client = runtime_client.build_command_client(provider, "cid") assert isinstance(client, runtime_client.HttpCommandClient) @@ -456,8 +513,13 @@ def test_build_event_client_uses_grpc_when_target_available(self, monkeypatch): client = runtime_client.build_event_client(provider, "cid") assert isinstance(client, runtime_client.GrpcEventClient) - def test_build_event_client_falls_back_to_http(self): - provider = {"selected_interface": "grpc", "base_url": ""} + def test_build_event_client_errors_when_grpc_selected_without_target(self): + provider = {"selected_interface": "grpc", "base_url": "http://driver:18080"} + with pytest.raises(runtime_client.GrpcTargetUnavailable): + runtime_client.build_event_client(provider, "cid") + + def test_build_event_client_defaults_to_http(self): + provider = {"base_url": "http://driver:18080"} client = runtime_client.build_event_client(provider, "cid") assert isinstance(client, runtime_client.HttpEventClient) From c50e4f58379f71bcd2fdf8f3127f721d62552250 Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:02:52 -0400 Subject: [PATCH 03/17] Validate driver contracts against the spec's required routes and x-meter-driver block validate_contract requires every route the Meter Driver Specification v1.4.0 marks required (/v1/requirements, /v1/init, /v1/nodes/register, /v1/nodes/{node_id}, /v1/nodes/{node_id}/configure-meter, /v1/meters/configure, /v1/events, /v1/status, /v1/healthz) and reads interfaces from the x-meter-driver block, synthesizing an http interface when the block is absent. Init fields come from GET /v1/requirements, which is now mandatory and raises ProviderRegistrationError on failure; each name is typed from the document's InitRequest schema (the /v1/init request body, else components.schemas.InitRequest), with a oneOf such as AesKeyInput reduced to its string alternative, and undocumented names typed as string. The /v1/commands configure_provider vendor-option schema remains as an optional extension whose fields follow the required ones with required=False; validate_provider_config_payload no longer demands values for such fields and omits blank optional values from the init payload. get_runtime_status probes only /v1/healthz; the legacy /health leg is gone. Tests use spec-shaped documents plus meter_driver_spec_openapi.json, the spec's own openapi/meter-driver.yaml converted to JSON. --- sparkmeter/config/provider_settings.py | 310 ++-- .../tests/meter_driver_spec_openapi.json | 1644 +++++++++++++++++ .../config/tests/test_provider_settings.py | 878 ++++++--- 3 files changed, 2410 insertions(+), 422 deletions(-) create mode 100644 sparkmeter/config/tests/meter_driver_spec_openapi.json diff --git a/sparkmeter/config/provider_settings.py b/sparkmeter/config/provider_settings.py index 5e40262..158bd36 100644 --- a/sparkmeter/config/provider_settings.py +++ b/sparkmeter/config/provider_settings.py @@ -30,6 +30,24 @@ class DriverInitializationError(ValueError): _METER_DRIVER_CONFIG_DIR = _REPO_ROOT / "meter_driver_configs" logger = logging.getLogger(__name__) +# The routes the Meter Driver Specification (v1.4.0, docs/spec/index.md +# section 4) marks Required, other than /openapi.json, which is the document +# being checked. A driver's /openapi.json must list every one of these. +REQUIRED_CONTRACT_PATHS = ( + "/v1/requirements", + "/v1/init", + "/v1/nodes/register", + "/v1/nodes/{node_id}", + "/v1/nodes/{node_id}/configure-meter", + "/v1/meters/configure", + "/v1/events", + "/v1/status", + "/v1/healthz", +) + +# The spec's interface-discovery extension block on /openapi.json (section 5.1). +DISCOVERY_EXTENSION = "x-meter-driver" + def _resolve_local_ref(spec, ref): """Resolve a local JSON Pointer reference within an OpenAPI document.""" @@ -53,6 +71,16 @@ def _resolve_schema(spec, schema): return schema +# --------------------------------------------------------------------------- +# Optional extension: /v1/commands configure_provider vendor options +# +# Not part of the spec. A driver may additionally document a /v1/commands +# route whose configure_provider command carries a vendor_options object; +# its fields are offered as optional extras after the spec-discovered +# required fields. Nothing below is required for registration. +# --------------------------------------------------------------------------- + + def _command_type_values(spec, schema): """Extract the possible command_type discriminator values from a schema.""" resolved = _resolve_schema(spec, schema) @@ -127,117 +155,115 @@ def _vendor_option_field_map(spec): return {field["name"]: field for field in _extract_vendor_option_fields(spec)} +# --------------------------------------------------------------------------- +# Init-field discovery: GET /v1/requirements typed by the InitRequest schema +# (spec sections 2, 5.2, 5.3 and 9) +# --------------------------------------------------------------------------- + + def _requirements_url(service_url): """Build the requirements endpoint URL from a service URL.""" return normalize_base_url(service_url) + "/v1/requirements" def _fetch_requirements_payload(service_url, timeout=10.0): - """Fetch the optional driver requirements payload.""" - response = httpx.get(_requirements_url(service_url), timeout=timeout) - response.raise_for_status() - payload = response.json() + """Fetch the driver's GET /v1/requirements response (a RequirementsResponse object).""" + try: + response = httpx.get(_requirements_url(service_url), timeout=timeout) + response.raise_for_status() + except httpx.HTTPError as exc: + raise ProviderRegistrationError("could not fetch driver requirements from /v1/requirements") from exc + try: + payload = response.json() + except ValueError as exc: + raise ProviderRegistrationError("driver requirements response is not valid JSON") from exc if not isinstance(payload, dict): raise ProviderRegistrationError("driver requirements response must be a JSON object") return payload -def _iter_candidate_requirement_schemas(spec): - """Yield object schemas that may describe driver requirement fields.""" - - def _walk(schema, depth=0): - if depth > 6: - return - resolved = _resolve_schema(spec, schema) - if not isinstance(resolved, dict): - return - properties = resolved.get("properties") or {} - if properties: - yield resolved - # Requirement fields are frequently nested (e.g. under - # vendor_options), so descend into object-typed properties too. - for prop_schema in properties.values(): - yield from _walk(prop_schema, depth + 1) +def _required_field_names_from_requirements(payload): + """Return the `required_fields` names from a RequirementsResponse, in the driver's order.""" + names = payload.get("required_fields") + if not isinstance(names, list): + raise ProviderRegistrationError("driver requirements response must list required_fields") + normalized = [] + for name in names: + text = str(name).strip() + if text and text not in normalized: + normalized.append(text) + return normalized - components = (spec.get("components") or {}).get("schemas") or {} - for schema in components.values(): - yield from _walk(schema) - for path_item in (spec.get("paths") or {}).values(): - if not isinstance(path_item, dict): - continue - for operation in path_item.values(): - if not isinstance(operation, dict): - continue - schema = ( - ((operation.get("requestBody") or {}).get("content") or {}) - .get("application/json", {}) - .get("schema", {}) - ) - yield from _walk(schema) +def _init_request_schema(spec): + """Return the document's InitRequest schema. + The POST /v1/init request-body schema is authoritative; a document that + documents init fields only under components.schemas.InitRequest is + read from there. + """ + schema = ( + (((spec.get("paths") or {}).get("/v1/init") or {}).get("post") or {}) + .get("requestBody", {}) + .get("content", {}) + .get("application/json", {}) + .get("schema", {}) + ) + resolved = _resolve_schema(spec, schema) + if resolved.get("properties"): + return resolved + components = (spec.get("components") or {}).get("schemas") or {} + return _resolve_schema(spec, components.get("InitRequest") or {}) -def _best_matching_requirements_schema(spec, required_fields): - """Find the schema whose properties best match the reported required fields.""" - required_names = set(required_fields or []) - best_schema = {} - best_score = 0 - for schema in _iter_candidate_requirement_schemas(spec): - properties = set((schema.get("properties") or {}).keys()) - score = len(required_names & properties) - if score > best_score: - best_schema = schema - best_score = score - return best_schema if best_score else {} +def _scalar_schema(spec, schema): + """Resolve a property schema to the alternative describing its scalar wire form. -# Type hints for the spec's standard driver init fields, used when the -# driver's OpenAPI does not describe a required field's schema itself. -_STANDARD_INIT_FIELD_TYPES = { - "aes_key": "string", - "channel": "integer", - "heartbeat_period_duration": "integer", -} + A oneOf/anyOf without its own type (the spec's AesKeyInput: 32-hex + string or 16-byte array) reduces to its first string-typed alternative, + else its first alternative, so the form layer sees one type and pattern. + """ + resolved = _resolve_schema(spec, schema) + alternatives = resolved.get("oneOf") or resolved.get("anyOf") or [] + if resolved.get("type") or not alternatives: + return resolved + resolved_alternatives = [_resolve_schema(spec, alternative) for alternative in alternatives] + for alternative in resolved_alternatives: + if alternative.get("type") == "string": + return alternative + return resolved_alternatives[0] def _extract_fields_from_requirements(spec, required_fields): - """Build normalized field specs from a requirements list plus OpenAPI schema hints.""" - schema = _best_matching_requirements_schema(spec, required_fields) - properties = schema.get("properties") or {} - schema_required = set(schema.get("required") or []) - fields = [] - for name in required_fields: - resolved = dict(_resolve_schema(spec, properties.get(name) or {})) - if not resolved.get("type") and name in _STANDARD_INIT_FIELD_TYPES: - resolved["type"] = _STANDARD_INIT_FIELD_TYPES[name] - fields.append( - _field_spec( - name, - resolved, - name in schema_required or name in set(required_fields), - ) - ) - return fields + """Build field specs for the advertised init fields, typed from InitRequest. + A field the document does not describe is typed as a string. + """ + properties = _init_request_schema(spec).get("properties") or {} + return [ + _field_spec(name, _scalar_schema(spec, properties.get(name) or {}), True) for name in required_fields + ] -def _extract_driver_requirement_fields(base_url, spec, timeout=10.0): - """Discover required driver fields from /v1/requirements, else OpenAPI.""" - vendor_option_fields = _extract_vendor_option_fields(spec) - if not vendor_option_fields: - # The driver declares no configurable requirements, so there is - # nothing to enrich and no reason to probe /v1/requirements. - return [] - try: - payload = _fetch_requirements_payload(base_url, timeout=timeout) - required_fields = payload.get("required_fields") or [] - if isinstance(required_fields, list) and required_fields: - normalized = [str(name).strip() for name in required_fields if str(name).strip()] - if normalized: - return _extract_fields_from_requirements(spec, normalized) - except (httpx.HTTPError, ValueError, ProviderRegistrationError): - pass - return vendor_option_fields +def _extract_driver_requirement_fields(base_url, spec, timeout=10.0): + """Discover the driver's init fields. + + GET /v1/requirements is required and its list order is kept; each name + is typed from the document's InitRequest schema. Fields from the + optional /v1/commands vendor-option extension, if the document has + one, follow as optional extras. + """ + payload = _fetch_requirements_payload(base_url, timeout=timeout) + fields = _extract_fields_from_requirements(spec, _required_field_names_from_requirements(payload)) + known = {field["name"] for field in fields} + for vendor_field in _extract_vendor_option_fields(spec): + if vendor_field["name"] in known: + continue + extra = dict(vendor_field) + extra["required"] = False + fields.append(extra) + known.add(extra["name"]) + return fields def _get_parameter(name): @@ -492,13 +518,20 @@ def parse_provider_config_text(config_text): def _required_field_names(payload): - """Return the required field names from a config payload.""" + """Return the names a config payload must supply values for. + + Entries are the field specs written at registration (dicts) or bare + names. A dict entry marked `"required": false` is an optional extra + and is not demanded. + """ names = payload.get("required_fields") or [] if not isinstance(names, list): raise DriverConfigError("required_fields must be a list") normalized = [] for entry in names: if isinstance(entry, dict): + if entry.get("required") is False: + continue name = str(entry.get("name") or "").strip() else: name = str(entry).strip() @@ -557,7 +590,11 @@ def _coerce_field_value(name, value, spec): def validate_provider_config_payload(payload): - """Validate required field presence in a driver config payload.""" + """Validate a driver config payload and return the typed init values. + + Every required field needs a value. An optional field left blank is + omitted from the returned `field_values` rather than sent as "". + """ required_fields = _required_field_names(payload) required_field_specs = _required_field_specs(payload) field_values = _field_values(payload) @@ -566,10 +603,11 @@ def validate_provider_config_payload(payload): ] if missing: raise DriverConfigError("required fields are missing values: {}".format(", ".join(missing))) - coerced_field_values = { - name: _coerce_field_value(name, value, required_field_specs.get(name)) - for name, value in field_values.items() - } + coerced_field_values = {} + for name, value in field_values.items(): + if value in (None, "") and name not in required_fields: + continue + coerced_field_values[name] = _coerce_field_value(name, value, required_field_specs.get(name)) return { "required_fields": required_fields, "field_values": coerced_field_values, @@ -756,8 +794,12 @@ def get_openapi_url(service_url): def _normalize_interface_metadata(base_url, spec): - """Extract the advertised interface inventory from a provider contract.""" - extension = spec.get("x-open-thunder") or {} + """Extract the advertised interface inventory from the contract's x-meter-driver block. + + Without the block, or without an http entry in it, one http interface + at the registered base URL is synthesized and becomes the default. + """ + extension = spec.get(DISCOVERY_EXTENSION) or {} interface_entries = extension.get("interfaces") or [] interfaces = [] @@ -863,7 +905,12 @@ def _fallback_interface_metadata(base_url, selected_interface=None): def validate_contract(service_url, timeout=10.0): - """Fetch and validate the provider's OpenAPI contract.""" + """Fetch and validate the driver's OpenAPI contract, then discover its init fields. + + Follows the spec's integration sequence (section 2): GET /openapi.json, + check it lists every required route, read x-meter-driver, then GET + /v1/requirements. A driver missing any of that is not registrable. + """ base_url = normalize_base_url(service_url) openapi_url = get_openapi_url(service_url) try: @@ -879,8 +926,7 @@ def validate_contract(service_url, timeout=10.0): info = spec.get("info") or {} paths = spec.get("paths") or {} - required_paths = ("/v1/commands", "/v1/events") - missing_paths = [path for path in required_paths if path not in paths] + missing_paths = [path for path in REQUIRED_CONTRACT_PATHS if path not in paths] if missing_paths: raise ProviderRegistrationError( "driver contract missing required paths: {}".format(", ".join(missing_paths)) @@ -925,46 +971,40 @@ def get_live_interface_details(service_url, selected_interface=None, timeout=2.0 def get_runtime_status(service_url, timeout=2.0, include_gateway_status=True): - """Check whether the provider service is currently reachable.""" + """Check driver liveness on GET /v1/healthz and, optionally, gateway state on GET /v1/status.""" base_url = normalize_base_url(service_url) healthz_url = base_url.rstrip("/") + "/v1/healthz" - legacy_health_url = base_url.rstrip("/") + "/health" status_url = base_url.rstrip("/") + "/v1/status" - urls = (healthz_url, legacy_health_url) - last_error = None - for url in urls: - try: - response = httpx.get(url, timeout=timeout) - response.raise_for_status() - status = { - "online": True, - "message": "online", - "checked_url": url, - "gateway_checked": bool(include_gateway_status), - } - if not include_gateway_status: - status["gateway_active"] = False - status["gateway_type"] = None - return status - try: - gateway_response = httpx.get(status_url, timeout=timeout) - gateway_response.raise_for_status() - gateway_data = gateway_response.json() - status["gateway_active"] = bool(gateway_data.get("connected")) - status["gateway_type"] = gateway_data.get("gateway_type") - except (httpx.HTTPError, ValueError): - status["gateway_checked"] = True - status["gateway_active"] = False - status["gateway_type"] = None - return status - except httpx.HTTPError as exc: - last_error = exc + try: + response = httpx.get(healthz_url, timeout=timeout) + response.raise_for_status() + except httpx.HTTPError as exc: + return { + "online": False, + "message": str(exc) or "unreachable", + "checked_url": healthz_url, + "gateway_checked": bool(include_gateway_status), + "gateway_active": False, + "gateway_type": None, + } - return { - "online": False, - "message": str(last_error) if last_error is not None else "unreachable", + status = { + "online": True, + "message": "online", "checked_url": healthz_url, "gateway_checked": bool(include_gateway_status), - "gateway_active": False, - "gateway_type": None, } + if not include_gateway_status: + status["gateway_active"] = False + status["gateway_type"] = None + return status + try: + gateway_response = httpx.get(status_url, timeout=timeout) + gateway_response.raise_for_status() + gateway_data = gateway_response.json() + status["gateway_active"] = bool(gateway_data.get("connected")) + status["gateway_type"] = gateway_data.get("gateway_type") + except (httpx.HTTPError, ValueError): + status["gateway_active"] = False + status["gateway_type"] = None + return status diff --git a/sparkmeter/config/tests/meter_driver_spec_openapi.json b/sparkmeter/config/tests/meter_driver_spec_openapi.json new file mode 100644 index 0000000..76a3e83 --- /dev/null +++ b/sparkmeter/config/tests/meter_driver_spec_openapi.json @@ -0,0 +1,1644 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Meter Driver API", + "version": "1.4.0", + "description": "Normative OpenAPI document for the required HTTP+SSE contract of the Meter Driver Specification (see docs/spec/index.md, sections 4-6). Any compliant driver's own /openapi.json response must be a valid OpenAPI document describing this same contract, extended with the x-meter-driver interface-discovery block. This file is the machine-readable reference other implementations validate against; it is illustrative of the required surface, not a specific driver's actual response.\n" + }, + "externalDocs": { + "description": "Full specification (prose, data conventions, event definitions, gRPC profile, compliance checklist)", + "url": "../docs/spec/index.md" + }, + "x-meter-driver": { + "interfaces": [ + { + "type": "http", + "label": "HTTP API", + "base_url": "http://127.0.0.1:18080" + }, + { + "type": "grpc", + "label": "gRPC", + "target": "127.0.0.1:50051" + } + ], + "default_interface": "http" + }, + "paths": { + "/v1/requirements": { + "get": { + "summary": "Discover required initialization fields", + "operationId": "getRequirements", + "tags": [ + "system" + ], + "responses": { + "200": { + "description": "Fields the caller must supply to /v1/init.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RequirementsResponse" + } + } + } + } + } + } + }, + "/v1/init": { + "post": { + "summary": "Reset driver runtime state and apply configuration", + "operationId": "initDriver", + "tags": [ + "system" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InitRequest" + } + } + } + }, + "responses": { + "202": { + "description": "Configuration accepted; a configuration-applied event follows on /v1/events.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AcceptedResponse" + } + } + } + } + } + } + }, + "/v1/nodes/register": { + "post": { + "summary": "Register a meter/node with the driver", + "operationId": "registerNode", + "tags": [ + "nodes" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterNodeRequest" + } + } + } + }, + "responses": { + "202": { + "description": "Accepted; node_registered or node_already_registered follows on /v1/events.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AcceptedResponse" + } + } + } + } + } + } + }, + "/v1/nodes/{node_id}": { + "delete": { + "summary": "Unregister a meter/node", + "operationId": "unregisterNode", + "tags": [ + "nodes" + ], + "parameters": [ + { + "$ref": "#/components/parameters/NodeIdPathParam" + } + ], + "responses": { + "202": { + "description": "Accepted; node_unregistered or node_to_unregister_unknown follows on /v1/events.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AcceptedResponse" + } + } + } + } + } + } + }, + "/v1/nodes/{node_id}/configure-meter": { + "post": { + "summary": "Apply operational configuration to one meter", + "operationId": "configureElectricalMeter", + "tags": [ + "meters" + ], + "parameters": [ + { + "$ref": "#/components/parameters/NodeIdPathParam" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ConfigureElectricalMeterPathRequest" + } + } + } + }, + "responses": { + "202": { + "description": "Accepted; electrical_meter_configuration_accepted then electrical_meter_configuration_applied follow on /v1/events (or invalid_electrical_meter_configuration on rejection).\n", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AcceptedResponse" + } + } + } + } + } + } + }, + "/v1/meters/configure": { + "post": { + "summary": "Body-node_id compatibility form of configure-meter", + "operationId": "configureElectricalMeterCompat", + "tags": [ + "meters" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ConfigureElectricalMeterCompatRequest" + } + } + } + }, + "responses": { + "202": { + "description": "Same semantics as POST /v1/nodes/{node_id}/configure-meter.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AcceptedResponse" + } + } + } + } + } + } + }, + "/v1/nodes/{node_id}/balance-and-flags": { + "post": { + "summary": "Update cached prepaid balance and low-balance flag", + "operationId": "setBalanceAndFlags", + "tags": [ + "meters" + ], + "parameters": [ + { + "$ref": "#/components/parameters/NodeIdPathParam" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SetBalanceAndFlagsRequest" + } + } + } + }, + "responses": { + "202": { + "description": "Accepted; electrical_meter_balance_and_flags_accepted follows on /v1/events.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AcceptedResponse" + } + } + } + } + } + } + }, + "/v1/events": { + "get": { + "summary": "Live event stream (Server-Sent Events)", + "operationId": "subscribeEvents", + "tags": [ + "events" + ], + "responses": { + "200": { + "description": "text/event-stream. Each message is an Event: a \"type\" naming the event and a \"data\" payload whose shape is selected by that type. See docs/spec/index.md section 6 for when each event is emitted.\n", + "content": { + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/Event" + } + } + } + } + } + } + }, + "/v1/status": { + "get": { + "summary": "Operational status for UI and diagnostics", + "operationId": "getStatus", + "tags": [ + "system" + ], + "responses": { + "200": { + "description": "Current driver/gateway status.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + } + } + } + } + }, + "/v1/healthz": { + "get": { + "summary": "Process liveness check", + "operationId": "healthz", + "tags": [ + "system" + ], + "responses": { + "200": { + "description": "Process is alive.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HealthResponse" + } + } + } + } + } + } + }, + "/v1/shutdown": { + "post": { + "summary": "Orderly remote shutdown", + "operationId": "shutdown", + "tags": [ + "system" + ], + "responses": { + "202": { + "description": "Shutdown accepted.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AcceptedResponse" + } + } + } + } + } + } + } + }, + "components": { + "parameters": { + "NodeIdPathParam": { + "name": "node_id", + "in": "path", + "required": true, + "description": "Application-visible meter/node identifier.", + "schema": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + } + }, + "schemas": { + "AcceptedResponse": { + "type": "object", + "required": [ + "accepted" + ], + "properties": { + "accepted": { + "type": "boolean" + } + } + }, + "HealthResponse": { + "type": "object", + "required": [ + "ok" + ], + "properties": { + "ok": { + "type": "boolean" + } + } + }, + "ErrorResponse": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "additionalProperties": true + }, + "RequirementsResponse": { + "type": "object", + "required": [ + "required_fields" + ], + "properties": { + "required_fields": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Field names the caller must supply to /v1/init. Driver-defined; discoverable, not hardcoded by clients." + } + } + }, + "Version": { + "type": "object", + "required": [ + "major", + "minor", + "patch" + ], + "properties": { + "major": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "minor": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "patch": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + }, + "Decimal": { + "type": "object", + "required": [ + "sign", + "coef", + "exp" + ], + "description": "value = sign * coef * 10^exp", + "properties": { + "sign": { + "type": "integer", + "format": "int32" + }, + "coef": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "exp": { + "type": "integer", + "format": "int32" + } + } + }, + "AesKeyInput": { + "oneOf": [ + { + "type": "string", + "pattern": "^[A-Fa-f0-9]{32}$", + "description": "16-byte AES key as a 32-character hex string." + }, + { + "type": "array", + "minItems": 16, + "maxItems": 16, + "items": { + "type": "integer", + "minimum": 0, + "maximum": 255 + }, + "description": "16-byte AES key as an array of byte values." + } + ] + }, + "InitRequest": { + "type": "object", + "description": "Reference fields shown here match the reference radio-network driver; a driver with different init fields must still document them here and make them discoverable via /v1/requirements.\n", + "required": [ + "heartbeat_period_duration", + "aes_key" + ], + "properties": { + "heartbeat_period_duration": { + "type": "integer", + "format": "uint32", + "minimum": 0, + "description": "Heartbeat/reading interval in seconds." + }, + "channel": { + "type": "integer", + "format": "uint32" + }, + "aes_key": { + "$ref": "#/components/schemas/AesKeyInput" + } + } + }, + "NodeTypeName": { + "type": "string", + "description": "Driver-defined meter model/family name." + }, + "RegisterNodeRequest": { + "type": "object", + "required": [ + "node_id", + "node_type" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "node_type": { + "$ref": "#/components/schemas/NodeTypeName" + }, + "mac": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "firmware_version": { + "$ref": "#/components/schemas/Version" + }, + "balance": { + "$ref": "#/components/schemas/Decimal" + }, + "low_balance_flag": { + "type": "boolean" + }, + "request_phased_readings": { + "type": "boolean" + } + } + }, + "ElectricalMeterCommandName": { + "type": "string", + "enum": [ + "ElectricalMeterCommandEnable", + "ElectricalMeterCommandDisable", + "ElectricalMeterCommandReboot", + "ElectricalMeterCommandCalibrateStart", + "ElectricalMeterCommandCalibrateFinish", + "ElectricalMeterCommandEnableSts" + ] + }, + "ElectricalMeterConfiguration": { + "type": "object", + "required": [ + "power_limit", + "current_limit", + "startup_delay", + "throttle_on_time", + "throttle_off_time", + "throttle_count_limit" + ], + "properties": { + "power_limit": { + "type": "number", + "format": "float" + }, + "current_limit": { + "type": "number", + "format": "float" + }, + "startup_delay": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "throttle_on_time": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "throttle_off_time": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "throttle_count_limit": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + }, + "ConfigureElectricalMeterPathRequest": { + "type": "object", + "required": [ + "command", + "configuration" + ], + "properties": { + "command": { + "$ref": "#/components/schemas/ElectricalMeterCommandName" + }, + "configuration": { + "$ref": "#/components/schemas/ElectricalMeterConfiguration" + } + } + }, + "ConfigureElectricalMeterCompatRequest": { + "type": "object", + "required": [ + "node_id", + "command", + "configuration" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "command": { + "$ref": "#/components/schemas/ElectricalMeterCommandName" + }, + "configuration": { + "$ref": "#/components/schemas/ElectricalMeterConfiguration" + } + } + }, + "SetBalanceAndFlagsRequest": { + "type": "object", + "required": [ + "balance", + "low_balance_flag" + ], + "properties": { + "balance": { + "$ref": "#/components/schemas/Decimal" + }, + "low_balance_flag": { + "type": "boolean" + } + } + }, + "StatusResponse": { + "type": "object", + "required": [ + "connected", + "firmware_version", + "messages_sent", + "messages_received", + "gateway_type" + ], + "properties": { + "connected": { + "type": "boolean" + }, + "firmware_version": { + "$ref": "#/components/schemas/Version" + }, + "messages_sent": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "messages_received": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "gateway_type": { + "type": "string" + }, + "gateway_firmware_raw": { + "type": [ + "string", + "null" + ] + }, + "gateway_bootloader_raw": { + "type": [ + "string", + "null" + ] + } + } + }, + "ElectricalMeterState": { + "type": "integer", + "format": "int32", + "enum": [ + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13 + ], + "description": "Meter operational state. -1 = Unknown, 0 = Off, 1 = On, 2 = Start, 3 = Error, 4 = Poweron, 5 = Startup, 6 = Throttle, 7 = ThrottleCheck, 8 = ThrottleError, 9 = Protect, 10 = MeterCheck, 11 = MeterDisabled, 12 = Calibrate, 13 = Tamper.\n" + }, + "RegisterSourceType": { + "type": "integer", + "format": "int32", + "enum": [ + 0, + 1 + ], + "description": "0 = Unknown, 1 = Manual." + }, + "Statistics": { + "type": "object", + "required": [ + "count", + "last_value", + "max", + "min", + "avg" + ], + "properties": { + "count": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "last_value": { + "type": "number", + "format": "double" + }, + "max": { + "type": "number", + "format": "double" + }, + "min": { + "type": "number", + "format": "double" + }, + "avg": { + "type": "number", + "format": "double" + } + } + }, + "Phases": { + "type": "object", + "required": [ + "a", + "b", + "c" + ], + "properties": { + "a": { + "type": "boolean" + }, + "b": { + "type": "boolean" + }, + "c": { + "type": "boolean" + } + } + }, + "HeartbeatReadHop": { + "type": "object", + "required": [ + "mac", + "rssi", + "retry" + ], + "properties": { + "mac": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "rssi": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "retry": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + }, + "DriverConfigurationApplied": { + "type": "object", + "required": [ + "masked_aes_key", + "channel", + "heartbeat_period_duration" + ], + "properties": { + "masked_aes_key": { + "type": "string" + }, + "channel": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "heartbeat_period_duration": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + }, + "NodeRegistered": { + "type": "object", + "required": [ + "node_id", + "source_type" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "source_type": { + "$ref": "#/components/schemas/RegisterSourceType" + } + } + }, + "NodeAlreadyRegistered": { + "type": "object", + "required": [ + "node_id" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + } + }, + "NodeUnregistered": { + "type": "object", + "required": [ + "node_id" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + } + }, + "NodeToUnregisterUnknown": { + "type": "object", + "required": [ + "node_id" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + } + }, + "ElectricalMeterReading": { + "type": "object", + "required": [ + "node_id", + "period_start", + "period_end", + "state", + "frequency", + "current_avg", + "current_min", + "current_max", + "voltage_avg", + "voltage_min", + "voltage_max", + "true_power_avg", + "true_power_inst", + "apparent_power_avg", + "power_factor_avg", + "energy", + "uptime_secs", + "user_power_limit" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "period_start": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "period_end": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "state": { + "$ref": "#/components/schemas/ElectricalMeterState" + }, + "frequency": { + "type": "number", + "format": "float" + }, + "current_avg": { + "type": "number", + "format": "float" + }, + "current_min": { + "type": "number", + "format": "float" + }, + "current_max": { + "type": "number", + "format": "float" + }, + "voltage_avg": { + "type": "number", + "format": "float" + }, + "voltage_min": { + "type": "number", + "format": "float" + }, + "voltage_max": { + "type": "number", + "format": "float" + }, + "true_power_avg": { + "type": "number", + "format": "float" + }, + "true_power_inst": { + "type": "number", + "format": "float" + }, + "apparent_power_avg": { + "type": "number", + "format": "float" + }, + "power_factor_avg": { + "type": "number", + "format": "float" + }, + "energy": { + "type": "number", + "format": "double" + }, + "uptime_secs": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "user_power_limit": { + "type": "number", + "format": "float" + } + } + }, + "ElectricalMeterReadingPhased": { + "allOf": [ + { + "$ref": "#/components/schemas/ElectricalMeterReading" + }, + { + "type": "object", + "required": [ + "phases", + "computed_fields_version" + ], + "properties": { + "frequency_a": { + "type": "number", + "format": "float" + }, + "frequency_b": { + "type": "number", + "format": "float" + }, + "frequency_c": { + "type": "number", + "format": "float" + }, + "current_avg_a": { + "type": "number", + "format": "float" + }, + "current_avg_b": { + "type": "number", + "format": "float" + }, + "current_avg_c": { + "type": "number", + "format": "float" + }, + "current_min_a": { + "type": "number", + "format": "float" + }, + "current_min_b": { + "type": "number", + "format": "float" + }, + "current_min_c": { + "type": "number", + "format": "float" + }, + "current_max_a": { + "type": "number", + "format": "float" + }, + "current_max_b": { + "type": "number", + "format": "float" + }, + "current_max_c": { + "type": "number", + "format": "float" + }, + "voltage_avg_a": { + "type": "number", + "format": "float" + }, + "voltage_avg_b": { + "type": "number", + "format": "float" + }, + "voltage_avg_c": { + "type": "number", + "format": "float" + }, + "voltage_min_a": { + "type": "number", + "format": "float" + }, + "voltage_min_b": { + "type": "number", + "format": "float" + }, + "voltage_min_c": { + "type": "number", + "format": "float" + }, + "voltage_max_a": { + "type": "number", + "format": "float" + }, + "voltage_max_b": { + "type": "number", + "format": "float" + }, + "voltage_max_c": { + "type": "number", + "format": "float" + }, + "true_power_avg_a": { + "type": "number", + "format": "float" + }, + "true_power_avg_b": { + "type": "number", + "format": "float" + }, + "true_power_avg_c": { + "type": "number", + "format": "float" + }, + "true_power_inst_a": { + "type": "number", + "format": "float" + }, + "true_power_inst_b": { + "type": "number", + "format": "float" + }, + "true_power_inst_c": { + "type": "number", + "format": "float" + }, + "apparent_power_avg_a": { + "type": "number", + "format": "float" + }, + "apparent_power_avg_b": { + "type": "number", + "format": "float" + }, + "apparent_power_avg_c": { + "type": "number", + "format": "float" + }, + "power_factor_avg_a": { + "type": "number", + "format": "float" + }, + "power_factor_avg_b": { + "type": "number", + "format": "float" + }, + "power_factor_avg_c": { + "type": "number", + "format": "float" + }, + "phases": { + "$ref": "#/components/schemas/Phases" + }, + "computed_fields_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + } + ] + }, + "InvalidElectricalMeterConfiguration": { + "type": "object", + "required": [ + "invalid_configuration" + ], + "properties": { + "invalid_configuration": { + "$ref": "#/components/schemas/ConfigureElectricalMeterCompatRequest" + } + } + }, + "ElectricalMeterConfigurationAccepted": { + "type": "object", + "required": [ + "node_id" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + } + }, + "ElectricalMeterConfigurationApplied": { + "type": "object", + "required": [ + "node_id", + "configuration" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "configuration": { + "$ref": "#/components/schemas/ElectricalMeterConfiguration" + } + } + }, + "NodeFirmwareVersionChanged": { + "type": "object", + "required": [ + "node_id", + "firmware_version" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "firmware_version": { + "$ref": "#/components/schemas/Version" + } + } + }, + "ElectricalMeterBalanceAndFlagsAccepted": { + "type": "object", + "required": [ + "node_id" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + } + }, + "HeartbeatStatistics": { + "type": "object", + "required": [ + "timestamp", + "total_registered_nodes", + "total_packets_sent", + "total_packets_received", + "nodes_reached_out_to_in_current_heartbeat", + "nodes_heard_from_in_current_heartbeat", + "packets_sent_in_current_heartbeat", + "packets_received_in_current_heartbeat", + "millisecond_read_reply_stats", + "millisecond_set_config_reply_stats" + ], + "properties": { + "timestamp": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "total_registered_nodes": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "total_packets_sent": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "total_packets_received": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "nodes_reached_out_to_in_current_heartbeat": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "nodes_heard_from_in_current_heartbeat": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "packets_sent_in_current_heartbeat": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "packets_received_in_current_heartbeat": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "millisecond_read_reply_stats": { + "$ref": "#/components/schemas/Statistics" + }, + "millisecond_set_config_reply_stats": { + "$ref": "#/components/schemas/Statistics" + } + } + }, + "GatewayStatus": { + "type": "object", + "required": [ + "connected", + "firmware_version", + "messages_sent", + "messages_received", + "gateway_type" + ], + "properties": { + "connected": { + "type": "boolean" + }, + "firmware_version": { + "$ref": "#/components/schemas/Version" + }, + "messages_sent": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "messages_received": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "gateway_type": { + "type": "string" + }, + "gateway_firmware_raw": { + "type": "string" + }, + "gateway_bootloader_raw": { + "type": "string" + } + } + }, + "HeartbeatReadHops": { + "type": "object", + "required": [ + "node_id", + "mac", + "src_ttl", + "ttl", + "last_hop", + "hop_list_length", + "hop_list_overflow", + "hops" + ], + "properties": { + "node_id": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "mac": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "src_ttl": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "ttl": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "last_hop": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "hop_list_length": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "hop_list_overflow": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "hops": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HeartbeatReadHop" + } + } + } + }, + "DriverConfigurationAppliedEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "driver_configuration_applied" + }, + "data": { + "$ref": "#/components/schemas/DriverConfigurationApplied" + } + } + }, + "NodeRegisteredEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "node_registered" + }, + "data": { + "$ref": "#/components/schemas/NodeRegistered" + } + } + }, + "NodeAlreadyRegisteredEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "node_already_registered" + }, + "data": { + "$ref": "#/components/schemas/NodeAlreadyRegistered" + } + } + }, + "NodeUnregisteredEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "node_unregistered" + }, + "data": { + "$ref": "#/components/schemas/NodeUnregistered" + } + } + }, + "NodeToUnregisterUnknownEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "node_to_unregister_unknown" + }, + "data": { + "$ref": "#/components/schemas/NodeToUnregisterUnknown" + } + } + }, + "ElectricalMeterReadingEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "electrical_meter_reading" + }, + "data": { + "$ref": "#/components/schemas/ElectricalMeterReading" + } + } + }, + "ElectricalMeterReadingPhasedEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "electrical_meter_reading_phased" + }, + "data": { + "$ref": "#/components/schemas/ElectricalMeterReadingPhased" + } + } + }, + "InvalidElectricalMeterConfigurationEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "invalid_electrical_meter_configuration" + }, + "data": { + "$ref": "#/components/schemas/InvalidElectricalMeterConfiguration" + } + } + }, + "ElectricalMeterConfigurationAcceptedEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "electrical_meter_configuration_accepted" + }, + "data": { + "$ref": "#/components/schemas/ElectricalMeterConfigurationAccepted" + } + } + }, + "ElectricalMeterConfigurationAppliedEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "electrical_meter_configuration_applied" + }, + "data": { + "$ref": "#/components/schemas/ElectricalMeterConfigurationApplied" + } + } + }, + "NodeFirmwareVersionChangedEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "node_firmware_version_changed" + }, + "data": { + "$ref": "#/components/schemas/NodeFirmwareVersionChanged" + } + } + }, + "ElectricalMeterBalanceAndFlagsAcceptedEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "electrical_meter_balance_and_flags_accepted" + }, + "data": { + "$ref": "#/components/schemas/ElectricalMeterBalanceAndFlagsAccepted" + } + } + }, + "HeartbeatStatisticsEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "heartbeat_statistics" + }, + "data": { + "$ref": "#/components/schemas/HeartbeatStatistics" + } + } + }, + "GatewayStatusEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "gateway_status" + }, + "data": { + "$ref": "#/components/schemas/GatewayStatus" + } + } + }, + "HeartbeatReadHopsEvent": { + "type": "object", + "required": [ + "type", + "data" + ], + "properties": { + "type": { + "type": "string", + "const": "heartbeat_read_hops" + }, + "data": { + "$ref": "#/components/schemas/HeartbeatReadHops" + } + } + }, + "Event": { + "description": "One SSE message: a per-event envelope { type, data }. The event name also appears in the SSE `event:` line.", + "oneOf": [ + { + "$ref": "#/components/schemas/DriverConfigurationAppliedEvent" + }, + { + "$ref": "#/components/schemas/NodeRegisteredEvent" + }, + { + "$ref": "#/components/schemas/NodeAlreadyRegisteredEvent" + }, + { + "$ref": "#/components/schemas/NodeUnregisteredEvent" + }, + { + "$ref": "#/components/schemas/NodeToUnregisterUnknownEvent" + }, + { + "$ref": "#/components/schemas/ElectricalMeterReadingEvent" + }, + { + "$ref": "#/components/schemas/ElectricalMeterReadingPhasedEvent" + }, + { + "$ref": "#/components/schemas/InvalidElectricalMeterConfigurationEvent" + }, + { + "$ref": "#/components/schemas/ElectricalMeterConfigurationAcceptedEvent" + }, + { + "$ref": "#/components/schemas/ElectricalMeterConfigurationAppliedEvent" + }, + { + "$ref": "#/components/schemas/NodeFirmwareVersionChangedEvent" + }, + { + "$ref": "#/components/schemas/ElectricalMeterBalanceAndFlagsAcceptedEvent" + }, + { + "$ref": "#/components/schemas/HeartbeatStatisticsEvent" + }, + { + "$ref": "#/components/schemas/GatewayStatusEvent" + }, + { + "$ref": "#/components/schemas/HeartbeatReadHopsEvent" + } + ] + } + } + }, + "tags": [ + { + "name": "system" + }, + { + "name": "nodes" + }, + { + "name": "meters" + }, + { + "name": "events" + } + ] +} diff --git a/sparkmeter/config/tests/test_provider_settings.py b/sparkmeter/config/tests/test_provider_settings.py index 990317e..8f1e29c 100644 --- a/sparkmeter/config/tests/test_provider_settings.py +++ b/sparkmeter/config/tests/test_provider_settings.py @@ -2,6 +2,7 @@ """Meter driver settings tests.""" import json +from pathlib import Path import httpx import pytest @@ -9,6 +10,19 @@ from sparkmeter.config import provider_settings from sparkmeter.metering.provider_config import configured_provider_url +# The Meter Driver Specification's own openapi/meter-driver.yaml (v1.4.0), +# converted to JSON: exactly what a driver serving nothing but the spec +# answers on GET /openapi.json (its x-meter-driver block lists http + grpc). +_SPEC_DOCUMENT_PATH = Path(__file__).with_name("meter_driver_spec_openapi.json") + +# The reference driver's /v1/requirements list (spec section 5.2 example). +_REFERENCE_REQUIRED_FIELDS = ("aes_key", "channel", "heartbeat_period_duration") + + +def load_spec_document(): + """Return a fresh copy of the spec's OpenAPI document.""" + return json.loads(_SPEC_DOCUMENT_PATH.read_text()) + class FakeResponse(object): """Minimal HTTPX-like response test double.""" @@ -24,239 +38,457 @@ def json(self): return self._payload -def _fake_openapi_get(url, timeout): - """A minimal valid driver OpenAPI, for tests that only need a saved provider.""" - return FakeResponse( - { - "info": {"title": "SparkNet-Http", "version": "1.2.3"}, - "paths": {"/v1/commands": {}, "/v1/events": {}}, - "x-open-thunder": {"default_interface": "http", "interfaces": []}, - } +def _spec_document(**overrides): + """A minimal document listing the spec's required routes with an http x-meter-driver block.""" + document = { + "openapi": "3.1.0", + "info": {"title": "Spec Driver", "version": "1.2.3"}, + "paths": {path: {} for path in provider_settings.REQUIRED_CONTRACT_PATHS}, + "x-meter-driver": { + "default_interface": "http", + "interfaces": [{"type": "http", "label": "HTTP API", "base_url": "http://127.0.0.1:18080"}], + }, + } + document.update(overrides) + return document + + +def _http_error(url, status_code=404): + request = httpx.Request("GET", url) + return httpx.HTTPStatusError( + "failed", request=request, response=httpx.Response(status_code, request=request) ) -def test_validate_contract_discovers_interfaces(monkeypatch): +def _fake_driver( + document=None, required_fields=("heartbeat_period_duration", "aes_key"), requirements_error=None +): + """Return an httpx.get double for a driver serving /openapi.json and /v1/requirements. + + Every URL it answers is recorded on `fake_get.calls`; anything other + than those two routes is an assertion failure, so a test sees any + stray probe (a legacy /health, a vendor init route) immediately. + """ + document = _spec_document() if document is None else document + calls = [] + def fake_get(url, timeout): - assert url == "http://127.0.0.1:18080/openapi.json" - assert timeout == 10.0 - return FakeResponse( - { - "info": { - "title": "SparkNet-Http", - "version": "1.2.3", - }, - "paths": { - "/v1/commands": {}, - "/v1/events": {}, - }, - "x-open-thunder": { - "default_interface": "grpc", - "interfaces": [ - { - "type": "grpc", - "label": "gRPC", - "target": "127.0.0.1:19090", - }, - ], - }, - } - ) + calls.append(url) + if url.endswith("/v1/requirements"): + if requirements_error is not None: + raise requirements_error + return FakeResponse({"required_fields": list(required_fields)}) + if url.endswith("/openapi.json"): + return FakeResponse(document) + raise AssertionError("unexpected GET {}".format(url)) - monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + fake_get.calls = calls + return fake_get - details = provider_settings.validate_contract("http://127.0.0.1:18080") - assert details["name"] == "SparkNet-Http" - assert details["base_url"] == "http://127.0.0.1:18080" - assert details["openapi_url"] == "http://127.0.0.1:18080/openapi.json" - assert details["default_interface"] == "grpc" - assert [interface["type"] for interface in details["interfaces"]] == ["http", "grpc"] +def _fake_openapi_get(url, timeout): + """A spec-only driver, for tests that only need a saved provider.""" + return _fake_driver()(url, timeout) -def test_validate_contract_discovers_vendor_option_requirements(monkeypatch): - def fake_get(url, timeout): - if url.endswith("/v1/requirements"): - return FakeResponse( - { - "required_fields": ["aes_key", "channel", "heartbeat_period_duration"], +def _vendor_options_extension(**properties): + """The optional /v1/commands configure_provider extension a reference driver adds.""" + return { + "paths": { + "/v1/commands": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [{"$ref": "#/components/schemas/ConfigureProviderCommand"}] + } + } + } + } } - ) - return FakeResponse( - { - "openapi": "3.1.0", - "info": { - "title": "SparkNet-Http", - "version": "1.2.3", - }, - "paths": { - "/v1/commands": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "oneOf": [ - {"$ref": "#/components/schemas/ConfigureProviderCommand"}, - ], - }, - }, - }, - }, - }, - }, - "/v1/events": {}, - }, - "components": { - "schemas": { - "ConfigureProviderCommand": { + } + }, + "components": { + "schemas": { + "ConfigureProviderCommand": { + "type": "object", + "properties": { + "command_type": {"type": "string", "enum": ["configure_provider"]}, + "vendor_options": { "type": "object", - "properties": { - "command_type": { - "type": "string", - "enum": ["configure_provider"], - }, - "vendor_options": { - "type": "object", - "required": ["aes_key", "channel"], - "properties": { - "aes_key": { - "type": "string", - "title": "AES key", - "description": "32-character hex network key.", - "pattern": "[0-9a-fA-F]{32}", - }, - "channel": { - "type": "integer", - "title": "Channel", - "description": "Radio channel to configure.", - "minimum": 11, - "maximum": 26, - }, - }, - }, - }, + "required": list(properties), + "properties": properties, }, }, - }, - "x-open-thunder": { - "default_interface": "http", - "interfaces": [], - }, + } } - ) + }, + } + +def _with_vendor_options(document, **properties): + """Return `document` extended with a /v1/commands vendor-option schema.""" + extension = _vendor_options_extension(**properties) + document = dict(document) + document["paths"] = {**document.get("paths", {}), **extension["paths"]} + document["components"] = { + "schemas": { + **((document.get("components") or {}).get("schemas") or {}), + **extension["components"]["schemas"], + } + } + return document + + +# --------------------------------------------------------------------------- +# validate_contract against the spec's own document +# --------------------------------------------------------------------------- + + +def test_spec_document_fixture_is_the_spec(monkeypatch): + document = load_spec_document() + assert document["info"]["version"] == "1.4.0" + assert document["x-meter-driver"]["default_interface"] == "http" + assert set(provider_settings.REQUIRED_CONTRACT_PATHS) <= set(document["paths"]) + # The spec's required-route list, and nothing the reference driver adds. + assert "/v1/commands" not in document["paths"] + + +def test_validate_contract_accepts_a_spec_only_driver(monkeypatch): + # A driver serving exactly the spec document, with the reference + # /v1/requirements list. This is the meter-driver-emulator case. + fake_get = _fake_driver(load_spec_document(), required_fields=_REFERENCE_REQUIRED_FIELDS) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) details = provider_settings.validate_contract("http://127.0.0.1:18080") + assert details["name"] == "Meter Driver API" + assert details["service_version"] == "1.4.0" + assert [interface["type"] for interface in details["interfaces"]] == ["http", "grpc"] + assert details["default_interface"] == "http" + # Requirements come from GET /v1/requirements, in the driver's order. + assert [field["name"] for field in details["driver_requirement_fields"]] == list( + _REFERENCE_REQUIRED_FIELDS + ) + assert all(field["required"] for field in details["driver_requirement_fields"]) + # ...typed by the document's InitRequest schema. fields = details["driver_requirement_field_map"] - assert fields["aes_key"]["required"] is True - assert fields["aes_key"]["pattern"] == "[0-9a-fA-F]{32}" - assert fields["channel"]["required"] is True - assert fields["channel"]["minimum"] == 11 - assert fields["channel"]["maximum"] == 26 - assert fields["heartbeat_period_duration"]["required"] is True assert fields["heartbeat_period_duration"]["type"] == "integer" + assert fields["heartbeat_period_duration"]["minimum"] == 0 + assert fields["channel"]["type"] == "integer" + assert fields["aes_key"]["type"] == "string" + assert fields["aes_key"]["pattern"] == "^[A-Fa-f0-9]{32}$" + # No /v1/commands, so no optional extras. + assert details["vendor_option_fields"] == [] + # Exactly the spec's two discovery calls, nothing else. + assert fake_get.calls == [ + "http://127.0.0.1:18080/openapi.json", + "http://127.0.0.1:18080/v1/requirements", + ] -def test_validate_contract_falls_back_to_openapi_requirements(monkeypatch): - def fake_get(url, timeout): - if url.endswith("/v1/requirements"): - raise httpx.HTTPStatusError( - "missing", - request=httpx.Request("GET", url), - response=httpx.Response(404, request=httpx.Request("GET", url)), - ) - return FakeResponse( - { - "openapi": "3.1.0", - "info": { - "title": "SparkNet-Http", - "version": "1.2.3", - }, - "paths": { - "/v1/commands": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "oneOf": [ - {"$ref": "#/components/schemas/ConfigureProviderCommand"}, - ], - }, - }, - }, - }, - }, - }, - "/v1/events": {}, - }, - "components": { - "schemas": { - "ConfigureProviderCommand": { - "type": "object", - "properties": { - "command_type": { - "type": "string", - "enum": ["configure_provider"], - }, - "vendor_options": { - "type": "object", - "required": ["aes_key", "channel"], - "properties": { - "aes_key": { - "type": "string", - "pattern": "[0-9a-fA-F]{32}", - }, - "channel": { - "type": "integer", - "minimum": 11, - "maximum": 26, - }, - }, - }, - }, - }, - }, - }, - "x-open-thunder": { - "default_interface": "http", - "interfaces": [], - }, +def test_validate_contract_accepts_the_spec_document_from_its_openapi_url(monkeypatch): + monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(load_spec_document())) + + details = provider_settings.validate_contract("http://127.0.0.1:18080/openapi.json") + + assert details["base_url"] == "http://127.0.0.1:18080" + assert details["openapi_url"] == "http://127.0.0.1:18080/openapi.json" + + +# --------------------------------------------------------------------------- +# validate_contract: interfaces +# --------------------------------------------------------------------------- + + +def test_validate_contract_discovers_grpc_interface(monkeypatch): + document = _spec_document( + **{ + "x-meter-driver": { + "default_interface": "grpc", + "interfaces": [ + {"type": "http", "label": "HTTP API", "base_url": "http://127.0.0.1:18080"}, + {"type": "grpc", "label": "gRPC", "target": "h:50051"}, + ], } - ) + } + ) + fake_get = _fake_driver(document) + monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + assert fake_get.calls[0] == "http://127.0.0.1:18080/openapi.json" + assert details["name"] == "Spec Driver" + assert details["default_interface"] == "grpc" + by_type = {interface["type"]: interface for interface in details["interfaces"]} + assert set(by_type) == {"http", "grpc"} + assert by_type["grpc"]["target"] == "h:50051" + assert by_type["grpc"]["address"] == "h:50051" + + +def test_validate_contract_synthesizes_http_when_discovery_block_is_absent(monkeypatch): + document = _spec_document() + del document["x-meter-driver"] + monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(document)) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + assert details["interfaces"] == [ + { + "type": "http", + "label": "HTTP API", + "base_url": "http://127.0.0.1:18080", + "address": "http://127.0.0.1:18080", + } + ] + assert details["default_interface"] == "http" + + +def test_validate_contract_ignores_non_spec_discovery_blocks(monkeypatch): + # A block under any other name is not the spec's; it is not read. The + # reference driver's pre-spec block name is assembled here so that no + # source line in sparkmeter/ carries it verbatim. + document = _spec_document() + del document["x-meter-driver"] + document["-".join(["x", "open", "thunder"])] = { + "default_interface": "grpc", + "interfaces": [{"type": "grpc", "target": "h:50051"}], + } + monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(document)) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + assert [interface["type"] for interface in details["interfaces"]] == ["http"] + assert details["default_interface"] == "http" + +# --------------------------------------------------------------------------- +# validate_contract: required routes +# --------------------------------------------------------------------------- + + +def test_required_contract_paths_are_the_spec_required_routes(): + assert provider_settings.REQUIRED_CONTRACT_PATHS == ( + "/v1/requirements", + "/v1/init", + "/v1/nodes/register", + "/v1/nodes/{node_id}", + "/v1/nodes/{node_id}/configure-meter", + "/v1/meters/configure", + "/v1/events", + "/v1/status", + "/v1/healthz", + ) + + +def test_validate_contract_rejects_missing_paths_naming_them(monkeypatch): + document = _spec_document() + del document["paths"]["/v1/requirements"] + del document["paths"]["/v1/init"] + del document["paths"]["/v1/healthz"] + fake_get = _fake_driver(document) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + with pytest.raises(provider_settings.ProviderRegistrationError) as exc: + provider_settings.validate_contract("http://127.0.0.1:18080") + + assert "missing required paths" in str(exc.value) + assert "/v1/requirements, /v1/init, /v1/healthz" in str(exc.value) + # Rejected on the document alone; requirements are never probed. + assert fake_get.calls == ["http://127.0.0.1:18080/openapi.json"] + + +def test_validate_contract_does_not_require_vendor_routes(monkeypatch): + # /v1/commands is a reference-driver extension, not a spec route. + document = _spec_document() + assert "/v1/commands" not in document["paths"] + monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(document)) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + assert details["name"] == "Spec Driver" + + +def test_validate_contract_rejects_a_document_with_only_vendor_routes(monkeypatch): + document = _spec_document(paths={"/v1/commands": {}, "/v1/events": {}}) + monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(document)) + + with pytest.raises(provider_settings.ProviderRegistrationError) as exc: + provider_settings.validate_contract("http://127.0.0.1:18080") + + assert "/v1/requirements" in str(exc.value) + assert "/v1/init" in str(exc.value) + + +# --------------------------------------------------------------------------- +# validate_contract: requirements probe +# --------------------------------------------------------------------------- + + +def test_validate_contract_returns_requirements_in_driver_order(monkeypatch): + fake_get = _fake_driver(required_fields=("zeta", "alpha", "heartbeat_period_duration")) + monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + assert [field["name"] for field in details["driver_requirement_fields"]] == [ + "zeta", + "alpha", + "heartbeat_period_duration", + ] + assert fake_get.calls == [ + "http://127.0.0.1:18080/openapi.json", + "http://127.0.0.1:18080/v1/requirements", + ] + + +def test_validate_contract_types_undocumented_requirements_as_string(monkeypatch): + # The minimal document has no InitRequest schema, so every field is a string. + monkeypatch.setattr( + provider_settings.httpx, "get", _fake_driver(required_fields=("aes_key", "site_token")) + ) + details = provider_settings.validate_contract("http://127.0.0.1:18080") fields = details["driver_requirement_field_map"] - assert fields["aes_key"]["required"] is True - assert fields["channel"]["required"] is True + assert fields["aes_key"]["type"] == "string" + assert fields["site_token"]["type"] == "string" + assert fields["site_token"]["required"] is True + + +def test_validate_contract_types_requirements_from_the_init_request_schema(monkeypatch): + document = load_spec_document() + # A driver with different init fields documents them in InitRequest + # (spec section 5.3) and lists them on /v1/requirements. + document["components"]["schemas"]["InitRequest"]["properties"]["site_token"] = { + "type": "string", + "title": "Site token", + "pattern": "^[a-z]+$", + } + document["components"]["schemas"]["InitRequest"]["properties"]["poll_seconds"] = { + "type": "integer", + "minimum": 5, + "maximum": 3600, + "default": 60, + } + fake_get = _fake_driver(document, required_fields=("site_token", "poll_seconds")) + monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + fields = details["driver_requirement_field_map"] + assert fields["site_token"] == { + "name": "site_token", + "label": "Site token", + "type": "string", + "required": True, + "description": "", + "pattern": "^[a-z]+$", + "minimum": None, + "maximum": None, + "default": None, + } + assert fields["poll_seconds"]["type"] == "integer" + assert fields["poll_seconds"]["minimum"] == 5 + assert fields["poll_seconds"]["maximum"] == 3600 + assert fields["poll_seconds"]["default"] == 60 + + +def test_validate_contract_requires_the_requirements_probe(monkeypatch): + url = "http://127.0.0.1:18080/v1/requirements" + monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(requirements_error=_http_error(url))) + with pytest.raises(provider_settings.ProviderRegistrationError) as exc: + provider_settings.validate_contract("http://127.0.0.1:18080") + + assert "/v1/requirements" in str(exc.value) + + +def test_validate_contract_requires_the_requirements_probe_even_with_vendor_options(monkeypatch): + # The vendor-option schema is not a substitute for /v1/requirements. + document = _with_vendor_options(_spec_document(), aes_key={"type": "string"}) + monkeypatch.setattr( + provider_settings.httpx, + "get", + _fake_driver(document, requirements_error=httpx.ConnectError("down")), + ) + + with pytest.raises(provider_settings.ProviderRegistrationError): + provider_settings.validate_contract("http://127.0.0.1:18080") -def test_validate_contract_rejects_missing_paths(monkeypatch): + +def test_validate_contract_rejects_malformed_requirements(monkeypatch): def fake_get(url, timeout): - return FakeResponse( - { - "info": { - "title": "SparkNet-Http", - }, - "paths": { - "/v1/commands": {}, - }, - } - ) + if url.endswith("/v1/requirements"): + return FakeResponse({"fields": ["aes_key"]}) + return FakeResponse(_spec_document()) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) - try: + with pytest.raises(provider_settings.ProviderRegistrationError) as exc: provider_settings.validate_contract("http://127.0.0.1:18080") - except provider_settings.ProviderRegistrationError as exc: - assert "missing required paths" in str(exc) - else: # pragma: no cover - raise AssertionError("expected ProviderRegistrationError") + + assert "required_fields" in str(exc.value) + + +def test_validate_contract_accepts_a_driver_requiring_no_init_fields(monkeypatch): + monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(required_fields=())) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + assert details["driver_requirement_fields"] == [] + assert details["driver_requirement_field_map"] == {} + + +# --------------------------------------------------------------------------- +# validate_contract: optional /v1/commands vendor options +# --------------------------------------------------------------------------- + + +def test_validate_contract_appends_vendor_options_as_optional_extras(monkeypatch): + document = _with_vendor_options( + load_spec_document(), + aes_key={"type": "string", "title": "AES key", "pattern": "[0-9a-fA-F]{32}"}, + channel={"type": "integer", "title": "Channel", "minimum": 11, "maximum": 26}, + region={"type": "string", "title": "Region", "description": "Radio regulatory region."}, + ) + monkeypatch.setattr( + provider_settings.httpx, + "get", + _fake_driver(document, required_fields=("heartbeat_period_duration", "aes_key")), + ) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + # Required fields first, in /v1/requirements order; vendor extras after, + # optional, and never duplicating a required field. + assert [(field["name"], field["required"]) for field in details["driver_requirement_fields"]] == [ + ("heartbeat_period_duration", True), + ("aes_key", True), + ("channel", False), + ("region", False), + ] + fields = details["driver_requirement_field_map"] + # aes_key keeps the spec's InitRequest typing, not the vendor schema's. + assert fields["aes_key"]["pattern"] == "^[A-Fa-f0-9]{32}$" + assert fields["channel"]["minimum"] == 11 + assert fields["channel"]["maximum"] == 26 + assert fields["region"]["description"] == "Radio regulatory region." + # The raw vendor-option view is still exposed for the form layer. + assert [field["name"] for field in details["vendor_option_fields"]] == ["aes_key", "channel", "region"] + assert set(details["vendor_option_field_map"]) == {"aes_key", "channel", "region"} + + +def test_validate_contract_appends_nothing_without_vendor_options(monkeypatch): + monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(required_fields=("aes_key",))) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + assert [field["name"] for field in details["driver_requirement_fields"]] == ["aes_key"] + assert details["vendor_option_fields"] == [] + assert details["vendor_option_field_map"] == {} def test_configured_provider_url_uses_saved_setting(session, monkeypatch): @@ -315,17 +547,6 @@ def fake_initialize_provider_sync(provider, field_values, provider_details=None) } -def _openapi_spec(**overrides): - """A minimal contract that passes validate_contract's required checks.""" - spec = { - "info": {"title": "SparkNet-Http", "version": "1.2.3"}, - "paths": {"/v1/commands": {}, "/v1/events": {}}, - "x-open-thunder": {"default_interface": "http", "interfaces": []}, - } - spec.update(overrides) - return spec - - def _use_temp_config_root(monkeypatch, tmp_path): """Redirect the module's config directory globals at a temp location.""" monkeypatch.setattr(provider_settings, "_REPO_ROOT", tmp_path) @@ -404,93 +625,104 @@ def test_fetch_requirements_payload_rejects_non_object(monkeypatch): provider_settings._fetch_requirements_payload("http://127.0.0.1:18080") -def test_iter_candidate_requirement_schemas_skips_non_dicts_and_bounds_depth(): - # Nest object schemas deeper than the depth-6 walk bound: the marker - # property sits at nesting depth 7 and must NOT be yielded, while the - # wrapper one level shallower (depth 6) still is. - nested = {"properties": {"deep_marker": {"type": "string"}}} - for level in range(7): - nested = {"properties": {"wrap_{}".format(level): nested}} +def test_fetch_requirements_payload_wraps_transport_and_json_errors(monkeypatch): + def boom(url, timeout): + raise httpx.ConnectError("down") + + monkeypatch.setattr(provider_settings.httpx, "get", boom) + with pytest.raises(provider_settings.ProviderRegistrationError) as exc: + provider_settings._fetch_requirements_payload("http://127.0.0.1:18080") + assert "/v1/requirements" in str(exc.value) + + class BadJSON(FakeResponse): + def json(self): + raise ValueError("bad") + + monkeypatch.setattr(provider_settings.httpx, "get", lambda url, timeout: BadJSON({})) + with pytest.raises(provider_settings.ProviderRegistrationError) as exc: + provider_settings._fetch_requirements_payload("http://127.0.0.1:18080") + assert "not valid JSON" in str(exc.value) + + +def test_required_field_names_from_requirements_keeps_order_and_dedups(): + names = provider_settings._required_field_names_from_requirements( + {"required_fields": ["b", " a ", "", "b", 7]} + ) + assert names == ["b", "a", "7"] + + +def test_required_field_names_from_requirements_rejects_non_list(): + with pytest.raises(provider_settings.ProviderRegistrationError): + provider_settings._required_field_names_from_requirements({"required_fields": "aes_key"}) + with pytest.raises(provider_settings.ProviderRegistrationError): + provider_settings._required_field_names_from_requirements({}) + + +def test_init_request_schema_reads_the_init_request_body(): + schema = provider_settings._init_request_schema(load_spec_document()) + assert set(schema["properties"]) == {"heartbeat_period_duration", "channel", "aes_key"} + assert schema["required"] == ["heartbeat_period_duration", "aes_key"] + +def test_init_request_schema_falls_back_to_components_init_request(): spec = { - "components": {"schemas": {"Root": nested}}, - "paths": { - "/v1/commands": "not-a-dict", # non-dict path item is skipped - "/v1/events": {"post": "not-a-dict"}, # non-dict operation is skipped - }, + "paths": {"/v1/init": {"post": {}}}, + "components": {"schemas": {"InitRequest": {"properties": {"site_token": {"type": "string"}}}}}, } + assert provider_settings._init_request_schema(spec) == {"properties": {"site_token": {"type": "string"}}} + assert provider_settings._init_request_schema({"paths": {}, "components": {}}) == {} - property_sets = [ - set(s.get("properties") or {}) for s in provider_settings._iter_candidate_requirement_schemas(spec) - ] - # The innermost wrapper (holding "wrap_0") is yielded at depth 6. - assert {"wrap_0"} in property_sets - # The schema one level deeper (holding "deep_marker") is beyond the bound. - assert not any("deep_marker" in properties for properties in property_sets) +def test_scalar_schema_reduces_one_of_to_the_string_alternative(): + spec = load_spec_document() + aes_key = spec["components"]["schemas"]["InitRequest"]["properties"]["aes_key"] + resolved = provider_settings._scalar_schema(spec, aes_key) + assert resolved["type"] == "string" + assert resolved["pattern"] == "^[A-Fa-f0-9]{32}$" -def test_iter_candidate_requirement_schemas_skips_ref_to_non_dict(): - # A component schema whose $ref resolves to a non-dict (a list here) must - # be skipped by the walk rather than treated as a properties-bearing schema. - spec = { - "components": {"schemas": {"Bad": {"$ref": "#/x/list"}}}, - "x": {"list": [1, 2, 3]}, - "paths": {}, +def test_scalar_schema_falls_back_to_the_first_alternative_and_passes_plain_schemas(): + assert provider_settings._scalar_schema({}, {"oneOf": [{"type": "integer"}, {"type": "array"}]}) == { + "type": "integer" + } + assert provider_settings._scalar_schema({}, {"anyOf": [{"type": "boolean"}]}) == {"type": "boolean"} + assert provider_settings._scalar_schema({}, {"type": "integer", "oneOf": [{"type": "string"}]}) == { + "type": "integer", + "oneOf": [{"type": "string"}], } - assert list(provider_settings._iter_candidate_requirement_schemas(spec)) == [] + assert provider_settings._scalar_schema({}, {}) == {} -def test_extract_fields_from_requirements_uses_standard_type_hints(): - # No component schema describes these, so the standard hints supply types. - fields = provider_settings._extract_fields_from_requirements( - {"components": {}, "paths": {}}, ["channel", "aes_key"] - ) +def test_extract_fields_from_requirements_types_from_init_request_else_string(): + spec = load_spec_document() + fields = provider_settings._extract_fields_from_requirements(spec, ["channel", "aes_key", "site_token"]) by_name = {field["name"]: field for field in fields} assert by_name["channel"]["type"] == "integer" assert by_name["aes_key"]["type"] == "string" - assert by_name["channel"]["required"] is True + assert by_name["site_token"]["type"] == "string" + assert all(field["required"] for field in fields) -def test_extract_driver_requirement_fields_returns_empty_without_vendor_options(monkeypatch): - # A contract with no configure-provider schema advertises no requirements. - assert provider_settings._extract_driver_requirement_fields("http://x", {"paths": {}}) == [] +def test_extract_driver_requirement_fields_probes_without_vendor_options(monkeypatch): + # No /v1/commands schema: the probe still happens and is the whole answer. + fake_get = _fake_driver(required_fields=("aes_key",)) + monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + fields = provider_settings._extract_driver_requirement_fields("http://127.0.0.1:18080", {"paths": {}}) + + assert [field["name"] for field in fields] == ["aes_key"] + assert fake_get.calls == ["http://127.0.0.1:18080/v1/requirements"] -def test_extract_driver_requirement_fields_falls_back_on_requirements_error(monkeypatch): - spec = { - "paths": { - "/v1/commands": { - "post": { - "requestBody": { - "content": { - "application/json": {"schema": {"oneOf": [{"$ref": "#/components/schemas/Cfg"}]}} - } - } - } - } - }, - "components": { - "schemas": { - "Cfg": { - "properties": { - "command_type": {"const": "configure_provider"}, - "vendor_options": { - "required": ["aes_key"], - "properties": {"aes_key": {"type": "string"}}, - }, - } - } - } - }, - } + +def test_extract_driver_requirement_fields_raises_on_requirements_error(monkeypatch): + spec = _with_vendor_options({"paths": {}}, aes_key={"type": "string"}) def boom(url, timeout): raise httpx.ConnectError("down") monkeypatch.setattr(provider_settings.httpx, "get", boom) - fields = provider_settings._extract_driver_requirement_fields("http://x", spec) - assert [field["name"] for field in fields] == ["aes_key"] + with pytest.raises(provider_settings.ProviderRegistrationError): + provider_settings._extract_driver_requirement_fields("http://x", spec) # --------------------------------------------------------------------------- @@ -521,7 +753,7 @@ def test_get_openapi_url_appends_suffix(): def test_normalize_interface_metadata_injects_http_and_dedups(): spec = { - "x-open-thunder": { + "x-meter-driver": { "default_interface": "grpc", "interfaces": [ "not-a-dict", @@ -545,7 +777,7 @@ def test_normalize_interface_metadata_blank_address_without_target_or_base_url() # An advertised interface that declares neither base_url nor target still # appears, but with an empty address. spec = { - "x-open-thunder": { + "x-meter-driver": { "default_interface": "http", "interfaces": [{"type": "mqtt", "label": "MQTT"}], } @@ -558,11 +790,35 @@ def test_normalize_interface_metadata_blank_address_without_target_or_base_url() def test_normalize_interface_metadata_defaults_to_http_when_unknown(): - spec = {"x-open-thunder": {"default_interface": "carrier-pigeon", "interfaces": []}} + spec = {"x-meter-driver": {"default_interface": "carrier-pigeon", "interfaces": []}} details = provider_settings._normalize_interface_metadata("http://base", spec) assert details["default_interface"] == "http" +def test_normalize_interface_metadata_reads_the_spec_block(): + details = provider_settings._normalize_interface_metadata("http://base", load_spec_document()) + assert details["default_interface"] == "http" + assert details["interfaces"] == [ + { + "type": "http", + "label": "HTTP API", + "base_url": "http://127.0.0.1:18080", + "address": "http://127.0.0.1:18080", + }, + {"type": "grpc", "label": "gRPC", "target": "127.0.0.1:50051", "address": "127.0.0.1:50051"}, + ] + + +def test_normalize_interface_metadata_without_block_synthesizes_http(): + details = provider_settings._normalize_interface_metadata("http://base", {"paths": {}}) + assert details == { + "interfaces": [ + {"type": "http", "label": "HTTP API", "base_url": "http://base", "address": "http://base"} + ], + "default_interface": "http", + } + + def test_apply_selected_interface_falls_back_to_default_when_invalid(): details = { "interfaces": [{"type": "http"}, {"type": "grpc"}], @@ -614,8 +870,7 @@ def json(self): def test_validate_contract_requires_info_title(monkeypatch): - spec = _openapi_spec(info={}) - monkeypatch.setattr(provider_settings.httpx, "get", lambda url, timeout: FakeResponse(spec)) + monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(_spec_document(info={}))) with pytest.raises(provider_settings.ProviderRegistrationError) as exc: provider_settings.validate_contract("http://127.0.0.1:18080") assert "info.title" in str(exc.value) @@ -645,17 +900,23 @@ def test_get_live_interface_details_applies_selection_on_success(monkeypatch): def test_get_runtime_status_reports_gateway_when_connected(monkeypatch): + calls = [] + def fake_get(url, timeout): + calls.append(url) + if url.endswith("/v1/healthz"): + return FakeResponse({"ok": True}) if url.endswith("/v1/status"): return FakeResponse({"connected": True, "gateway_type": "sparknet"}) - return FakeResponse({}) + raise AssertionError("unexpected GET {}".format(url)) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) status = provider_settings.get_runtime_status("http://127.0.0.1:18080") assert status["online"] is True assert status["gateway_active"] is True assert status["gateway_type"] == "sparknet" - assert status["checked_url"].endswith("/v1/healthz") + assert status["checked_url"] == "http://127.0.0.1:18080/v1/healthz" + assert calls == ["http://127.0.0.1:18080/v1/healthz", "http://127.0.0.1:18080/v1/status"] def test_get_runtime_status_can_skip_gateway_probe(monkeypatch): @@ -679,20 +940,24 @@ def fake_get(url, timeout): assert status["gateway_checked"] is True -def test_get_runtime_status_falls_back_to_legacy_health(monkeypatch): +def test_get_runtime_status_is_offline_when_healthz_fails_and_probes_nothing_else(monkeypatch): + # /v1/healthz is the spec's liveness route; a failure means offline. No + # legacy /health probe is attempted and /v1/status is not consulted. + calls = [] + def fake_get(url, timeout): + calls.append(url) if url.endswith("/v1/healthz"): raise httpx.ConnectError("no healthz") - if url.endswith("/health"): - return FakeResponse({}) - if url.endswith("/v1/status"): - return FakeResponse({"connected": False}) - return FakeResponse({}) + return FakeResponse({"connected": True}) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) status = provider_settings.get_runtime_status("http://127.0.0.1:18080") - assert status["online"] is True - assert status["checked_url"].endswith("/health") + assert status["online"] is False + assert status["gateway_active"] is False + assert status["gateway_type"] is None + assert status["checked_url"] == "http://127.0.0.1:18080/v1/healthz" + assert calls == ["http://127.0.0.1:18080/v1/healthz"] def test_get_runtime_status_reports_offline_when_unreachable(monkeypatch): @@ -860,6 +1125,20 @@ def test_required_field_names_accepts_dicts_and_strings(): assert names == ["aes_key", "channel"] +def test_required_field_names_skips_optional_extras(): + # Vendor-option extras are written with required=False and are not demanded. + names = provider_settings._required_field_names( + { + "required_fields": [ + {"name": "aes_key", "required": True}, + {"name": "channel", "required": False}, + {"name": "heartbeat_period_duration"}, + ] + } + ) + assert names == ["aes_key", "heartbeat_period_duration"] + + def test_coerce_field_value_by_type(): assert provider_settings._coerce_field_value("c", "26", {"type": "integer"}) == 26 assert provider_settings._coerce_field_value("r", "1.5", {"type": "number"}) == 1.5 @@ -894,6 +1173,31 @@ def test_validate_provider_config_payload_reports_missing_and_coerces(): assert validated["field_values"]["channel"] == 26 +def test_validate_provider_config_payload_omits_blank_optional_fields(): + validated = provider_settings.validate_provider_config_payload( + { + "required_fields": [ + {"name": "aes_key", "type": "string", "required": True}, + {"name": "channel", "type": "integer", "required": False}, + {"name": "region", "type": "string", "required": False}, + ], + "field_values": {"aes_key": "00" * 16, "channel": "", "region": "eu"}, + } + ) + # A blank optional integer is neither demanded nor coerced; a filled one is kept. + assert validated["required_fields"] == ["aes_key"] + assert validated["field_values"] == {"aes_key": "00" * 16, "region": "eu"} + + +def test_validate_provider_config_payload_accepts_a_driver_with_no_fields(): + assert provider_settings.validate_provider_config_payload( + {"required_fields": [], "field_values": {}} + ) == { + "required_fields": [], + "field_values": {}, + } + + def test_normalize_init_status_defaults_for_bad_input(): assert provider_settings._normalize_init_status({"init_status": "nope"}) == { "has_successful_init": False, From f069d4e024eb25800b925d9668826dfbad65b906 Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:04:02 -0400 Subject: [PATCH 04/17] Build the driver init payload from the discovered required fields _load_driver_init_payload no longer hardcodes aes_key and heartbeat_period_duration. It validates the driver's config file with validate_provider_config_payload, so the payload is exactly the fields the driver reported on GET /v1/requirements (persisted as required_fields, typed from its InitRequest schema) with the stored field_values coerced to those types; blank optional extras are omitted and a driver requiring no fields gets an empty body. A missing required value or an uncoercible value skips init with a warning instead of posting a partial body. Only aes_key, when present as a string, is format-checked as 32 hex characters; a byte-array key passes through. --- sparkmeter/metering/reconcile.py | 57 +++--- sparkmeter/metering/tests/test_reconcile.py | 183 ++++++++++++++------ 2 files changed, 160 insertions(+), 80 deletions(-) diff --git a/sparkmeter/metering/reconcile.py b/sparkmeter/metering/reconcile.py index 077067f..9ae520f 100644 --- a/sparkmeter/metering/reconcile.py +++ b/sparkmeter/metering/reconcile.py @@ -5,7 +5,7 @@ The provider holds no durable per-meter state across restarts. This module re-issues the full sequence on every webapp boot: - configure_provider (heartbeat + vendor net params) + init_driver (POST /v1/init with the driver's discovered init fields) for each meter in DB: register_meter configure_meter (limits + behavior verb) @@ -76,8 +76,8 @@ async def reconcile_all( this Flask instance. `skip_provider_init` is used when the caller has already issued the - vendor-specific provider init for this runtime transition and only - needs the per-meter reconcile sequence. + driver init for this runtime transition and only needs the per-meter + reconcile sequence. """ logger.info("metering reconcile: starting") @@ -134,10 +134,25 @@ async def reconcile_all( def _load_driver_init_payload(flask_app: "Flask") -> dict[str, Any] | None: + """Build the POST /v1/init body from the driver's discovered fields and stored values. + + The field set is whatever the driver reported on GET /v1/requirements + at registration (persisted as `required_fields` in its config file, + typed from its InitRequest schema) and the values are the operator's + `field_values`; nothing about the field set is hardcoded here. The + payload is exactly those fields. Returns None, with a warning, when + the config is absent or incomplete, so reconcile skips init rather + than posting a body the driver will reject. + + `aes_key` is the one field with a format check, and only when present + as a string: the spec's 32-hex-character form (section 5.3). + """ try: from sparkmeter.config.provider_settings import ( + DriverConfigError, get_enabled_provider, load_provider_runtime_settings, + validate_provider_config_payload, ) except ImportError: return None @@ -149,35 +164,25 @@ def _load_driver_init_payload(flask_app: "Flask") -> dict[str, Any] | None: with flask_app.app_context(): enabled_provider = get_enabled_provider() driver_config = load_provider_runtime_settings(enabled_provider) - driver_field_values = ((driver_config or {}).get("field_values")) or {} - aes_key_hex = driver_field_values.get("aes_key") - if not aes_key_hex: + if not driver_config: return None - heartbeat = driver_field_values.get("heartbeat_period_duration") - if heartbeat in (None, ""): + try: + payload: dict[str, Any] = dict(validate_provider_config_payload(driver_config)["field_values"]) + except DriverConfigError as exc: + logger.warning("metering reconcile: skipping driver init; driver config is not usable: %s", exc) return None - payload: dict[str, Any] = { - "heartbeat_period_duration": int(heartbeat), - "aes_key": str(aes_key_hex).strip(), - } - channel = driver_field_values.get("channel") - if channel is not None: - try: - payload["channel"] = int(channel) - except (TypeError, ValueError): + aes_key = payload.get("aes_key") + if isinstance(aes_key, str): + aes_key = aes_key.strip() + if not _AES_KEY_HEX_RE.fullmatch(aes_key): logger.warning( - "metering reconcile: skipping invalid channel %r; expected integer", - channel, + "metering reconcile: skipping driver init; AES key %r is not 32 hex characters", + payload.get("aes_key"), ) - aes_key = payload["aes_key"] - if not _AES_KEY_HEX_RE.fullmatch(aes_key): - logger.warning( - "metering reconcile: skipping invalid driver AES key %r; expected 32 hex characters", - aes_key_hex, - ) - return None + return None + payload["aes_key"] = aes_key return payload diff --git a/sparkmeter/metering/tests/test_reconcile.py b/sparkmeter/metering/tests/test_reconcile.py index 44b6e9d..f78768b 100644 --- a/sparkmeter/metering/tests/test_reconcile.py +++ b/sparkmeter/metering/tests/test_reconcile.py @@ -9,6 +9,7 @@ calls that intentionally invoke them from a fresh thread. """ +import logging import threading from types import SimpleNamespace @@ -209,103 +210,177 @@ def test_load_meters_signature_takes_flask_app(self): with pytest.raises(TypeError): reconcile._load_meters() # type: ignore[call-arg] - def test_load_driver_init_payload_reads_driver_fields(self, app, session, monkeypatch): + @staticmethod + def _use_driver_config(monkeypatch, config): monkeypatch.setattr( provider_settings, "get_enabled_provider", lambda: {"base_url": "http://127.0.0.1:18080", "selected_interface": "http"}, ) - monkeypatch.setattr( - provider_settings, - "load_provider_runtime_settings", - lambda provider: { + monkeypatch.setattr(provider_settings, "load_provider_runtime_settings", lambda provider: config) + + # The field specs registration writes for the reference /v1/requirements + # list, typed from the spec's InitRequest schema. + _REFERENCE_FIELDS = [ + {"name": "aes_key", "type": "string", "required": True}, + {"name": "channel", "type": "integer", "required": True}, + {"name": "heartbeat_period_duration", "type": "integer", "required": True}, + ] + + def test_load_driver_init_payload_builds_from_discovered_fields(self, app, session, monkeypatch): + self._use_driver_config( + monkeypatch, + { + "required_fields": self._REFERENCE_FIELDS, "field_values": { - "aes_key": "00112233445566778899AABBCCDDEEFF", + "aes_key": " 00112233445566778899AABBCCDDEEFF ", "channel": "26", "heartbeat_period_duration": "60", - } + }, }, ) payload = _run_in_fresh_thread(reconcile._load_driver_init_payload, app) + # Exactly the discovered fields, coerced to their InitRequest types. assert payload == { "aes_key": "00112233445566778899AABBCCDDEEFF", "channel": 26, "heartbeat_period_duration": 60, } - def test_load_driver_init_payload_skips_invalid_aes_key(self, app, session, monkeypatch): - monkeypatch.setattr( - provider_settings, - "get_enabled_provider", - lambda: {"base_url": "http://127.0.0.1:18080", "selected_interface": "http"}, + def test_load_driver_init_payload_uses_the_drivers_own_fields(self, app, session, monkeypatch): + # A driver with init fields unlike the reference driver's: the payload + # is whatever it asked for, with no aes_key or heartbeat expected. + self._use_driver_config( + monkeypatch, + { + "required_fields": [ + {"name": "site_token", "type": "string", "required": True}, + {"name": "poll_seconds", "type": "integer", "required": True}, + {"name": "verbose", "type": "boolean", "required": False}, + ], + "field_values": {"site_token": "abc", "poll_seconds": "30", "verbose": "yes"}, + }, ) - monkeypatch.setattr( - provider_settings, - "load_provider_runtime_settings", - lambda provider: { - "field_values": { - "aes_key": "not-hex", - "channel": "12", - "heartbeat_period_duration": "60", - } + + payload = _run_in_fresh_thread(reconcile._load_driver_init_payload, app) + + assert payload == {"site_token": "abc", "poll_seconds": 30, "verbose": True} + + def test_load_driver_init_payload_omits_blank_optional_fields(self, app, session, monkeypatch): + self._use_driver_config( + monkeypatch, + { + "required_fields": [ + {"name": "heartbeat_period_duration", "type": "integer", "required": True}, + {"name": "channel", "type": "integer", "required": False}, + ], + "field_values": {"heartbeat_period_duration": "60", "channel": ""}, }, ) payload = _run_in_fresh_thread(reconcile._load_driver_init_payload, app) + assert payload == {"heartbeat_period_duration": 60} + + def test_load_driver_init_payload_sends_empty_body_for_a_driver_needing_no_fields( + self, app, session, monkeypatch + ): + self._use_driver_config(monkeypatch, {"required_fields": [], "field_values": {}}) + + assert _run_in_fresh_thread(reconcile._load_driver_init_payload, app) == {} + + def test_load_driver_init_payload_none_without_a_config_file(self, app, session, monkeypatch): + self._use_driver_config(monkeypatch, {}) + + assert _run_in_fresh_thread(reconcile._load_driver_init_payload, app) is None + + def test_load_driver_init_payload_skips_invalid_aes_key(self, app, session, monkeypatch, caplog): + self._use_driver_config( + monkeypatch, + { + "required_fields": self._REFERENCE_FIELDS, + "field_values": {"aes_key": "not-hex", "channel": "12", "heartbeat_period_duration": "60"}, + }, + ) + + with caplog.at_level(logging.WARNING): + payload = _run_in_fresh_thread(reconcile._load_driver_init_payload, app) + assert payload is None + assert any("not 32 hex characters" in record.message for record in caplog.records) - def test_load_driver_init_payload_requires_heartbeat(self, app, session, monkeypatch): - monkeypatch.setattr( - provider_settings, "get_enabled_provider", lambda: {"base_url": "http://127.0.0.1:18080"} + def test_load_driver_init_payload_only_checks_aes_key_when_present(self, app, session, monkeypatch): + # No aes_key among the driver's fields: no hex check applies. + self._use_driver_config( + monkeypatch, + { + "required_fields": [ + {"name": "heartbeat_period_duration", "type": "integer", "required": True} + ], + "field_values": {"heartbeat_period_duration": "60"}, + }, ) - monkeypatch.setattr( - provider_settings, - "load_provider_runtime_settings", - lambda provider: {"field_values": {"aes_key": "00112233445566778899AABBCCDDEEFF"}}, + + assert _run_in_fresh_thread(reconcile._load_driver_init_payload, app) == { + "heartbeat_period_duration": 60 + } + + def test_load_driver_init_payload_passes_a_byte_array_aes_key_through(self, app, session, monkeypatch): + # The spec's AesKeyInput also allows a 16-byte integer array; only the + # string form is format-checked. + key = list(range(16)) + self._use_driver_config( + monkeypatch, + { + "required_fields": [{"name": "aes_key", "type": "array", "required": True}], + "field_values": {"aes_key": key}, + }, ) - # No heartbeat period means there is nothing to init the driver with. - assert _run_in_fresh_thread(reconcile._load_driver_init_payload, app) is None + assert _run_in_fresh_thread(reconcile._load_driver_init_payload, app) == {"aes_key": key} - def test_load_driver_init_payload_skips_invalid_channel(self, app, session, monkeypatch): - monkeypatch.setattr( - provider_settings, "get_enabled_provider", lambda: {"base_url": "http://127.0.0.1:18080"} + def test_load_driver_init_payload_skips_when_a_required_field_is_missing( + self, app, session, monkeypatch, caplog + ): + self._use_driver_config( + monkeypatch, + { + "required_fields": self._REFERENCE_FIELDS, + "field_values": {"aes_key": "00112233445566778899AABBCCDDEEFF", "channel": "26"}, + }, ) - monkeypatch.setattr( - provider_settings, - "load_provider_runtime_settings", - lambda provider: { + + with caplog.at_level(logging.WARNING): + payload = _run_in_fresh_thread(reconcile._load_driver_init_payload, app) + + # A required field without a value means the driver cannot be + # initialized; nothing partial is sent. + assert payload is None + assert any("heartbeat_period_duration" in record.message for record in caplog.records) + + def test_load_driver_init_payload_skips_on_uncoercible_value(self, app, session, monkeypatch, caplog): + self._use_driver_config( + monkeypatch, + { + "required_fields": self._REFERENCE_FIELDS, "field_values": { "aes_key": "00112233445566778899AABBCCDDEEFF", "channel": "not-an-int", "heartbeat_period_duration": "60", - } + }, }, ) - payload = _run_in_fresh_thread(reconcile._load_driver_init_payload, app) + with caplog.at_level(logging.WARNING): + payload = _run_in_fresh_thread(reconcile._load_driver_init_payload, app) - # The bad channel is dropped; the rest of the payload still comes through. - assert payload == { - "aes_key": "00112233445566778899AABBCCDDEEFF", - "heartbeat_period_duration": 60, - } + assert payload is None + assert any("channel" in record.message for record in caplog.records) - def test_load_driver_init_payload_requires_aes_key(self, app, session, monkeypatch): - monkeypatch.setattr( - provider_settings, "get_enabled_provider", lambda: {"base_url": "http://127.0.0.1:18080"} - ) - monkeypatch.setattr( - provider_settings, - "load_provider_runtime_settings", - lambda provider: {"field_values": {"heartbeat_period_duration": "60"}}, ) - # No AES key means there is nothing to initialize the driver with. - assert _run_in_fresh_thread(reconcile._load_driver_init_payload, app) is None class _RecordingClient: From af82de8fb613add7b453ef067be35e164af2434f Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:07:54 -0400 Subject: [PATCH 05/17] Subscribe to /v1/events with only the X-Client-Id header The spec's GET /v1/events declares no parameters, so stream_json_events sends no client_id query string; the client identifies itself through the X-Client-Id header alone. --- sparkmeter/metering/http_sse.py | 13 ++++++------- sparkmeter/metering/tests/test_http_sse.py | 20 +++++++------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/sparkmeter/metering/http_sse.py b/sparkmeter/metering/http_sse.py index 883d032..ad84aad 100644 --- a/sparkmeter/metering/http_sse.py +++ b/sparkmeter/metering/http_sse.py @@ -44,15 +44,14 @@ async def stream_json_events( base_url: str, client_id: str, ) -> AsyncIterator[dict[str, Any]]: - """Yield JSON event payloads from the driver's HTTP SSE endpoint.""" + """Yield JSON event payloads from the driver's HTTP SSE endpoint. + + GET /v1/events declares no parameters in the spec; the client + identifies itself only through the X-Client-Id header. + """ headers = {"X-Client-Id": client_id, "Accept": "text/event-stream"} async with httpx.AsyncClient(base_url=base_url.rstrip("/"), timeout=None) as client: - async with client.stream( - "GET", - "/v1/events", - params={"client_id": client_id}, - headers=headers, - ) as response: + async with client.stream("GET", "/v1/events", headers=headers) as response: response.raise_for_status() async for payload in _iter_sse_data_payloads(response): yield json.loads(payload) diff --git a/sparkmeter/metering/tests/test_http_sse.py b/sparkmeter/metering/tests/test_http_sse.py index 2f8a877..15fff58 100644 --- a/sparkmeter/metering/tests/test_http_sse.py +++ b/sparkmeter/metering/tests/test_http_sse.py @@ -40,20 +40,13 @@ async def __aexit__(self, exc_type, exc, tb): del exc_type, exc, tb return False - def stream(self, method, path, params=None, headers=None): - type(self).calls.append( - { - "method": method, - "path": path, - "params": params, - "headers": headers, - } - ) + def stream(self, method, path, **kwargs): + type(self).calls.append({"method": method, "path": path, **kwargs}) return _FakeResponse( [ 'data: {"type":"gateway_status"}', "", - 'data: {"event_type":"meter_reading","meter_id":"42"}', + 'data: {"type":"electrical_meter_reading","data":{"node_id":42}}', "", ] ) @@ -74,13 +67,14 @@ async def test_stream_json_events_uses_streaming_sse_request(monkeypatch): assert events == [ {"type": "gateway_status"}, - {"event_type": "meter_reading", "meter_id": "42"}, + {"type": "electrical_meter_reading", "data": {"node_id": 42}}, ] + # The spec's GET /v1/events takes no parameters: the client id travels + # only in the X-Client-Id header, never as a query string. assert _FakeAsyncClient.calls == [ { "method": "GET", "path": "/v1/events", - "params": {"client_id": "test-client"}, "headers": { "Accept": "text/event-stream", "X-Client-Id": "test-client", @@ -96,7 +90,7 @@ async def test_iter_sse_payloads_skips_comments_and_flushes_trailing_data(): response = _FakeResponse( [ ": keep-alive heartbeat", - "event: meter_reading", + "event: electrical_meter_reading", "data: line-one", "data: line-two", "", From ac991e2139be3de1073916eb00d2cc3d8a75a42e Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:07:54 -0400 Subject: [PATCH 06/17] Recognize only the spec's event type names on the SSE stream The side-channel set in events.py is exactly the Meter Driver Specification's event type constants; the reference driver's own alias for driver_configuration_applied is no longer accepted and is logged as an unknown type like any other non-spec name. --- sparkmeter/metering/events.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sparkmeter/metering/events.py b/sparkmeter/metering/events.py index 5f12b52..0bacf92 100644 --- a/sparkmeter/metering/events.py +++ b/sparkmeter/metering/events.py @@ -55,7 +55,12 @@ } # Envelope "type" values that are observed but not acted on beyond logging. +# These are the spec's event `type` constants (openapi/meter-driver.yaml +# *Event schemas) other than the reading, heartbeat_statistics and +# heartbeat_read_hops types handled above. Any other name, including a +# driver's own alias for one of these, is unknown and logged as such. _SIDE_CHANNEL_TYPES = { + "driver_configuration_applied", "gateway_status", "node_registered", "node_already_registered", @@ -66,13 +71,6 @@ "electrical_meter_configuration_accepted", "electrical_meter_configuration_applied", "electrical_meter_balance_and_flags_accepted", - # "sparknet_configuration_applied" is what SparkNet-Http-New's live HTTP - # SSE stream actually emits for this event (its own server-side naming, - # not something Thundercloud controls) -- kept for as long as that - # specific driver is in use. "driver_configuration_applied" is the - # vendor-neutral name from the meter-driver-spec. - "sparknet_configuration_applied", - "driver_configuration_applied", } # Side-channel types worth an INFO line rather than DEBUG. From 3ec649e8d281437b12859465fd9ae22726c89403 Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:07:55 -0400 Subject: [PATCH 07/17] Test the driver client end to end against a spec-only driver A driver serving the spec's own OpenAPI document is registered from its base URL, its generated config lists the fields from /v1/requirements, and reconcile then sends exactly one POST /v1/init carrying those fields and nothing else. The events tests pin the side-channel set to the spec's event names and check that a vendor alias is reported as unknown; the lifespan tests use a spec event name for their "other event" case. --- sparkmeter/metering/tests/test_events.py | 59 +++++++++++++ sparkmeter/metering/tests/test_lifespan.py | 4 +- sparkmeter/metering/tests/test_reconcile.py | 96 +++++++++++++++++++++ 3 files changed, 157 insertions(+), 2 deletions(-) diff --git a/sparkmeter/metering/tests/test_events.py b/sparkmeter/metering/tests/test_events.py index 17eb46c..64a9974 100644 --- a/sparkmeter/metering/tests/test_events.py +++ b/sparkmeter/metering/tests/test_events.py @@ -134,6 +134,65 @@ async def capture(event): assert captured == [] assert any("side-channel" in rec.message for rec in caplog.records) + def test_side_channel_types_are_exactly_the_spec_event_names(self): + # The spec's *Event `type` constants, less the ones dispatched to handlers + # (electrical_meter_reading, electrical_meter_reading_phased, + # heartbeat_statistics, heartbeat_read_hops). + assert events._SIDE_CHANNEL_TYPES == { + "driver_configuration_applied", + "node_registered", + "node_already_registered", + "node_unregistered", + "node_to_unregister_unknown", + "invalid_electrical_meter_configuration", + "electrical_meter_configuration_accepted", + "electrical_meter_configuration_applied", + "node_firmware_version_changed", + "electrical_meter_balance_and_flags_accepted", + "gateway_status", + } + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "spec_type", + sorted(events._SIDE_CHANNEL_TYPES), + ) + async def test_every_spec_side_channel_type_is_recognized(self, caplog, spec_type): + with caplog.at_level(logging.DEBUG): + await events.dispatch_dict_event({"type": spec_type, "data": {}}, []) + + assert any("side-channel" in rec.message for rec in caplog.records) + assert not any("unknown type" in rec.message for rec in caplog.records) + + @pytest.mark.asyncio + async def test_driver_configuration_applied_is_a_known_side_channel(self, caplog): + raw = { + "type": "driver_configuration_applied", + "data": {"masked_aes_key": "62..22", "channel": 26, "heartbeat_period_duration": 60}, + } + with caplog.at_level(logging.DEBUG): + await events.dispatch_dict_event(raw, []) + + assert any("side-channel" in rec.message for rec in caplog.records) + + @pytest.mark.asyncio + async def test_vendor_alias_for_configuration_applied_is_unknown(self, caplog): + # The reference driver's own name for the spec's + # driver_configuration_applied event is not a spec name and is not + # accepted as one. It is assembled here so that no source line in + # sparkmeter/ carries the non-spec name verbatim. + alias = "_".join(["sparknet", "configuration", "applied"]) + captured: list = [] + + async def capture(event): + captured.append(event) + + with caplog.at_level(logging.WARNING): + await events.dispatch_dict_event({"type": alias, "data": {"channel": 26}}, [capture]) + + assert captured == [] + assert any("unknown type" in rec.message and alias in rec.getMessage() for rec in caplog.records) + @pytest.mark.asyncio async def test_heartbeat_statistics_parsed(self): captured: list = [] diff --git a/sparkmeter/metering/tests/test_lifespan.py b/sparkmeter/metering/tests/test_lifespan.py index 4b263d6..9ebf91c 100644 --- a/sparkmeter/metering/tests/test_lifespan.py +++ b/sparkmeter/metering/tests/test_lifespan.py @@ -379,7 +379,7 @@ class TestObserveGatewayDisconnect: @pytest.mark.asyncio async def test_non_gateway_event_ignored(self): app, gateway_state = _build_app() - await lifespan._observe_gateway_status(app, {"type": "meter_reading"}) + await lifespan._observe_gateway_status(app, {"type": "electrical_meter_reading"}) assert gateway_state["gateway_paused"] is False @pytest.mark.asyncio @@ -625,7 +625,7 @@ async def stream_events(self, client_id): if index == 0: # Connect successfully, deliver one event, then the # stream ends -> "treating provider as restarted". - yield {"type": "meter_reading", "data": {}} + yield {"type": "electrical_meter_reading", "data": {}} return # Second connection attempt breaks mid-stream. raise RuntimeError("stream broke") diff --git a/sparkmeter/metering/tests/test_reconcile.py b/sparkmeter/metering/tests/test_reconcile.py index f78768b..45ef351 100644 --- a/sparkmeter/metering/tests/test_reconcile.py +++ b/sparkmeter/metering/tests/test_reconcile.py @@ -379,8 +379,104 @@ def test_load_driver_init_payload_skips_on_uncoercible_value(self, app, session, assert payload is None assert any("channel" in record.message for record in caplog.records) + +class _AcceptedResponse: + def raise_for_status(self): + return None + + +class _RecordingHttpx: + """httpx.AsyncClient double recording every request the HTTP transport makes.""" + + def __init__(self, **kwargs): + self.kwargs = kwargs + self.calls: list = [] + + async def post(self, path, json=None): + self.calls.append(("POST", path, json)) + return _AcceptedResponse() + + async def delete(self, path): + self.calls.append(("DELETE", path, None)) + return _AcceptedResponse() + + async def get(self, path, **kwargs): + raise AssertionError("unexpected GET {}".format(path)) + + async def aclose(self): + return None + + +class TestSpecOnlyDriverInit: + """A driver serving exactly the spec: registered, configured, initialized.""" + + @pytest.mark.asyncio + async def test_reconcile_posts_exactly_one_init_with_the_discovered_fields( + self, app, session, monkeypatch, tmp_path + ): + import json + from pathlib import Path + + from sparkmeter.metering import runtime_client + + spec_document = json.loads( + (Path(provider_settings.__file__).parent / "tests" / "meter_driver_spec_openapi.json").read_text() ) + required_fields = ["heartbeat_period_duration", "aes_key"] + + class _Response: + def __init__(self, payload): + self.payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self.payload + + def fake_get(url, timeout): + if url.endswith("/openapi.json"): + return _Response(spec_document) + if url.endswith("/v1/requirements"): + return _Response({"required_fields": required_fields}) + raise AssertionError("unexpected GET {}".format(url)) + monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + monkeypatch.setattr(provider_settings, "_REPO_ROOT", tmp_path) + monkeypatch.setattr(provider_settings, "_METER_DRIVER_CONFIG_DIR", tmp_path / "meter_driver_configs") + + # Register the driver from its base URL, as the settings form does. + provider_id = provider_settings.save_provider_settings("http://127.0.0.1:18080", "http") + session.commit() + provider = provider_settings.get_provider(provider_id) + + # The generated config lists the driver's own /v1/requirements fields; + # the operator fills them in. + config = json.loads(provider_settings.load_provider_config_text(provider)) + assert [field["name"] for field in config["required_fields"]] == required_fields + assert config["field_values"] == {"heartbeat_period_duration": "", "aes_key": ""} + config["field_values"] = { + "heartbeat_period_duration": "60", + "aes_key": "00112233445566778899aabbccddeeff", + } + provider_settings.save_provider_config_text(provider, json.dumps(config)) + + monkeypatch.setattr(runtime_client.httpx, "AsyncClient", _RecordingHttpx) + monkeypatch.setattr(reconcile, "_load_meters", lambda flask_app: []) + client = runtime_client.HttpCommandClient(provider["base_url"], "cid") + + await reconcile.reconcile_all(client, app) + + # Exactly one POST /v1/init carrying exactly the discovered fields, + # typed by the spec's InitRequest schema; no vendor init route, no + # legacy /health probe. + assert client._client.calls == [ + ( + "POST", + "/v1/init", + {"heartbeat_period_duration": 60, "aes_key": "00112233445566778899aabbccddeeff"}, + ) + ] class _RecordingClient: From cbfff30151a44326e339b0d3be20542e54c11df9 Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:47:07 -0400 Subject: [PATCH 08/17] Check contract operations, guard document shapes, and split interface discovery from requirements Contract validation now checks the spec's required operations rather than path keys: each of GET /v1/requirements, POST /v1/init, POST /v1/nodes/register, DELETE /v1/nodes/{node_id}, POST /v1/nodes/{node_id}/configure-meter, POST /v1/meters/configure, GET /v1/events, GET /v1/status and GET /v1/healthz must be documented under its path with that method. Path parameters match by position, so /v1/nodes/{id} is accepted; a trailing slash is not. Every document access is type-guarded (document, info, paths, x-meter-driver, $ref targets, oneOf/anyOf alternatives, allOf parts) so a malformed document raises ProviderRegistrationError, never AttributeError. Init-field discovery validates GET /v1/requirements with the wheel's RequirementsResponse model (required_fields is array[string]; nothing is stringified, blank names are rejected), names the transport error in its message, warns for a field absent from InitRequest.properties, follows $ref chains with a cycle guard, merges allOf composition, and reads OpenAPI 3.1 type arrays. validate_contract is split: inspect_contract fetches and validates the document and discovers interfaces in one round trip, and is what get_live_interface_details uses, so an advertised gRPC target is never lost to a slow or failing /v1/requirements; validate_contract adds the requirements round trip for registration. get_live_interface_details takes an optional provider and reports the fields recorded in its config file. save_provider_settings loses the unread aes_key and channel parameters. get_runtime_status is offline unless /v1/healthz answers a JSON object with "ok": true, and when /v1/status answers something other than a JSON object. Config payloads: whitespace-only values count as missing; values are checked against the recorded pattern, minimum and maximum; only fields the driver listed are sent; aes_key must be 32 hex characters or an array of 16 byte values; bare-name required_fields entries from configs written before types were recorded are treated as required strings with a warning pointing at re-registration. The spec-document fixture, a fake response class and a fake_driver factory move to sparkmeter/conftest.py; the fixture's regeneration command is documented there and a test checks its version against the installed meter-driver-spec wheel. --- pyproject.toml | 4 + sparkmeter/config/provider_settings.py | 664 +++++++++--- sparkmeter/config/tests/test_configviews.py | 14 +- .../config/tests/test_provider_settings.py | 995 ++++++++++++++---- sparkmeter/conftest.py | 70 ++ uv.lock | 4 +- 6 files changed, 1367 insertions(+), 384 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 23cee37..e5822f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,10 @@ dependencies = [ "meter-driver-spec @ https://github.com/EarthSpark/meter-driver-spec/releases/download/v1.4.0/meter_driver_spec-1.4.0-py3-none-any.whl", "protobuf", "grpcio", + # sparkmeter.config.provider_settings catches pydantic.ValidationError + # raised by the meter-driver-spec models, so pydantic is a dependency of + # this package and not only of meter-driver-spec. + "pydantic", "click", # dates and times diff --git a/sparkmeter/config/provider_settings.py b/sparkmeter/config/provider_settings.py index 158bd36..98e26d7 100644 --- a/sparkmeter/config/provider_settings.py +++ b/sparkmeter/config/provider_settings.py @@ -3,11 +3,14 @@ import json import logging +import re import uuid from pathlib import Path from urllib.parse import urlparse import httpx +from meter_driver_spec.http.models import RequirementsResponse +from pydantic import ValidationError from sparkmeter.config.configdomain import ConfigParameter from sparkmeter.config.configparameter import ParameterObject @@ -30,32 +33,60 @@ class DriverInitializationError(ValueError): _METER_DRIVER_CONFIG_DIR = _REPO_ROOT / "meter_driver_configs" logger = logging.getLogger(__name__) -# The routes the Meter Driver Specification (v1.4.0, docs/spec/index.md -# section 4) marks Required, other than /openapi.json, which is the document -# being checked. A driver's /openapi.json must list every one of these. -REQUIRED_CONTRACT_PATHS = ( - "/v1/requirements", - "/v1/init", - "/v1/nodes/register", - "/v1/nodes/{node_id}", - "/v1/nodes/{node_id}/configure-meter", - "/v1/meters/configure", - "/v1/events", - "/v1/status", - "/v1/healthz", +# The operations the Meter Driver Specification (v1.4.0, docs/spec/index.md +# section 4) marks Required, other than GET /openapi.json, which is the +# document being checked. A driver's /openapi.json must document every one +# of these under its path with this method; the two Recommended routes +# (/v1/nodes/{node_id}/balance-and-flags, /v1/shutdown) are not demanded. +REQUIRED_CONTRACT_OPERATIONS = ( + ("get", "/v1/requirements"), + ("post", "/v1/init"), + ("post", "/v1/nodes/register"), + ("delete", "/v1/nodes/{node_id}"), + ("post", "/v1/nodes/{node_id}/configure-meter"), + ("post", "/v1/meters/configure"), + ("get", "/v1/events"), + ("get", "/v1/status"), + ("get", "/v1/healthz"), ) # The spec's interface-discovery extension block on /openapi.json (section 5.1). DISCOVERY_EXTENSION = "x-meter-driver" +# The gRPC profile's ConfigureDriver message (spec section 7, +# meter_driver.proto) has fixed fields, so a driver used over gRPC must list +# these on /v1/requirements whatever else it asks for. +GRPC_INIT_REQUIRED_FIELDS = ("heartbeat_period_duration", "aes_key") + +# A path template parameter, e.g. {node_id}; its name is not significant +# when matching a driver's paths against the spec's. +_PATH_PARAM_RE = re.compile(r"\{[^}]*\}") + +# The spec's AesKeyInput (section 3): a 16-byte key as 32 hex characters or +# as 16 byte values. +_AES_KEY_HEX_RE = re.compile(r"^[0-9a-fA-F]{32}$") +_AES_KEY_BYTES = 16 + +# Bound on nested schema composition (allOf parts, $ref chains) followed +# while reading a document, so a self-referential schema cannot recurse +# without end. +_SCHEMA_DEPTH_LIMIT = 8 + + +# --------------------------------------------------------------------------- +# OpenAPI schema helpers +# --------------------------------------------------------------------------- + def _resolve_local_ref(spec, ref): """Resolve a local JSON Pointer reference within an OpenAPI document.""" - if not ref or not ref.startswith("#/"): + if not isinstance(ref, str) or not ref.startswith("#/"): return None node = spec for part in ref[2:].split("/"): + if not isinstance(node, dict): + return None node = node.get(part.replace("~1", "/").replace("~0", "~")) if node is None: return None @@ -63,12 +94,67 @@ def _resolve_local_ref(spec, ref): def _resolve_schema(spec, schema): - """Resolve a schema object or local ref to a plain dict.""" - if not isinstance(schema, dict): - return {} - if "$ref" in schema: - return _resolve_local_ref(spec, schema["$ref"]) or {} - return schema + """Resolve a schema object to a plain dict, following $ref chains. + + Anything that is not a dict, or a $ref that does not resolve to one, + is an empty schema. A reference cycle stops at its first repeat. + """ + seen = set() + while isinstance(schema, dict) and "$ref" in schema: + ref = schema["$ref"] + if not isinstance(ref, str) or ref in seen: + return {} + seen.add(ref) + schema = _resolve_local_ref(spec, ref) + return schema if isinstance(schema, dict) else {} + + +def _schema_type(schema): + """Return a schema's JSON type name, or "" when it declares none. + + An OpenAPI 3.1 type array (["string", "null"]) yields its first non-null + entry. + """ + value = (schema or {}).get("type") + if isinstance(value, list): + for entry in value: + if entry != "null": + return str(entry).strip().lower() + return "" + return str(value or "").strip().lower() + + +def _object_schema(spec, schema, depth=0): + """Resolve an object schema, merging the properties and required lists of its allOf parts.""" + resolved = _resolve_schema(spec, schema) + parts = resolved.get("allOf") + if not isinstance(parts, list) or depth >= _SCHEMA_DEPTH_LIMIT: + return resolved + + merged = {key: value for key, value in resolved.items() if key != "allOf"} + properties = dict(merged["properties"]) if isinstance(merged.get("properties"), dict) else {} + required = list(merged["required"]) if isinstance(merged.get("required"), list) else [] + for part in parts: + part_schema = _object_schema(spec, part, depth + 1) + if isinstance(part_schema.get("properties"), dict): + properties.update(part_schema["properties"]) + if isinstance(part_schema.get("required"), list): + required.extend(name for name in part_schema["required"] if name not in required) + for key, value in part_schema.items(): + if key not in ("properties", "required", "allOf"): + merged.setdefault(key, value) + merged["properties"] = properties + merged["required"] = required + return merged + + +def _dig(node, *keys): + """Walk nested dicts by key; anything missing or not a dict yields {}.""" + for key in keys: + if not isinstance(node, dict): + return {} + node = node.get(key) + return node if isinstance(node, dict) else {} # --------------------------------------------------------------------------- @@ -84,7 +170,7 @@ def _resolve_schema(spec, schema): def _command_type_values(spec, schema): """Extract the possible command_type discriminator values from a schema.""" resolved = _resolve_schema(spec, schema) - properties = resolved.get("properties") or {} + properties = resolved.get("properties") if isinstance(resolved.get("properties"), dict) else {} command_type = _resolve_schema(spec, properties.get("command_type") or {}) values = [] if "const" in command_type: @@ -96,21 +182,18 @@ def _command_type_values(spec, schema): def _find_configure_provider_schema(spec): """Locate the configure-provider command schema in the OpenAPI document.""" - request_schema = ( - (((spec.get("paths") or {}).get("/v1/commands") or {}).get("post") or {}) - .get("requestBody", {}) - .get("content", {}) - .get("application/json", {}) - .get("schema", {}) + request_schema = _dig( + spec, "paths", "/v1/commands", "post", "requestBody", "content", "application/json", "schema" ) request_schema = _resolve_schema(spec, request_schema) - for candidate in request_schema.get("oneOf") or []: + candidates = request_schema.get("oneOf") + for candidate in candidates if isinstance(candidates, list) else []: values = _command_type_values(spec, candidate) if "configure_provider" in values: return _resolve_schema(spec, candidate) - components = (spec.get("components") or {}).get("schemas") or {} + components = _dig(spec, "components", "schemas") for schema in components.values(): values = _command_type_values(spec, schema) if "configure_provider" in values: @@ -120,8 +203,8 @@ def _find_configure_provider_schema(spec): def _field_spec(name, schema, required): - """Normalize a vendor-option field description for the form layer.""" - field_type = str(schema.get("type") or "string").strip().lower() or "string" + """Normalize a field description for the form and config layers.""" + field_type = _schema_type(schema) or "string" return { "name": name, "label": str(schema.get("title") or name.replace("_", " ").title()), @@ -138,14 +221,21 @@ def _field_spec(name, schema, required): def _extract_vendor_option_fields(spec): """Extract configure-provider vendor option requirements from OpenAPI.""" command_schema = _find_configure_provider_schema(spec) - properties = command_schema.get("properties") or {} - vendor_schema = _resolve_schema(spec, properties.get("vendor_options") or {}) + properties = ( + command_schema.get("properties") if isinstance(command_schema.get("properties"), dict) else {} + ) + vendor_schema = _object_schema(spec, properties.get("vendor_options") or {}) fields = [] - required_fields = set(vendor_schema.get("required") or []) - for field_name, field_schema in (vendor_schema.get("properties") or {}).items(): - resolved_field = _resolve_schema(spec, field_schema) - fields.append(_field_spec(field_name, resolved_field, field_name in required_fields)) + required_fields = ( + set(vendor_schema["required"]) if isinstance(vendor_schema.get("required"), list) else set() + ) + vendor_properties = ( + vendor_schema.get("properties") if isinstance(vendor_schema.get("properties"), dict) else {} + ) + for field_name, field_schema in vendor_properties.items(): + resolved_field = _scalar_schema(spec, field_schema) + fields.append(_field_spec(str(field_name), resolved_field, field_name in required_fields)) return fields @@ -166,54 +256,69 @@ def _requirements_url(service_url): return normalize_base_url(service_url) + "/v1/requirements" +def _validation_summary(exc): + """One line naming each pydantic validation error's location and message.""" + return "; ".join( + "{}: {}".format(".".join(str(part) for part in error.get("loc") or ()) or "body", error.get("msg")) + for error in exc.errors() + ) + + +def _required_field_names_from_requirements(payload): + """Return the `required_fields` names from a RequirementsResponse, in the driver's order. + + The payload is validated as the spec's RequirementsResponse model + (`required_fields` is an array of strings). A blank name is rejected; + a repeated name is kept once. + """ + try: + requirements = RequirementsResponse.model_validate(payload) + except ValidationError as exc: + raise ProviderRegistrationError( + "driver requirements response is not a valid RequirementsResponse: {}".format( + _validation_summary(exc) + ) + ) from exc + names = [] + for name in requirements.required_fields: + if not name.strip(): + raise ProviderRegistrationError("driver requirements response lists a blank field name") + if name not in names: + names.append(name) + return names + + def _fetch_requirements_payload(service_url, timeout=10.0): - """Fetch the driver's GET /v1/requirements response (a RequirementsResponse object).""" + """GET /v1/requirements and return the driver's required field names.""" + url = _requirements_url(service_url) try: - response = httpx.get(_requirements_url(service_url), timeout=timeout) + response = httpx.get(url, timeout=timeout) response.raise_for_status() except httpx.HTTPError as exc: - raise ProviderRegistrationError("could not fetch driver requirements from /v1/requirements") from exc + raise ProviderRegistrationError( + "could not fetch driver requirements from /v1/requirements: {}".format( + str(exc) or exc.__class__.__name__ + ) + ) from exc try: payload = response.json() except ValueError as exc: raise ProviderRegistrationError("driver requirements response is not valid JSON") from exc - if not isinstance(payload, dict): - raise ProviderRegistrationError("driver requirements response must be a JSON object") - return payload - - -def _required_field_names_from_requirements(payload): - """Return the `required_fields` names from a RequirementsResponse, in the driver's order.""" - names = payload.get("required_fields") - if not isinstance(names, list): - raise ProviderRegistrationError("driver requirements response must list required_fields") - normalized = [] - for name in names: - text = str(name).strip() - if text and text not in normalized: - normalized.append(text) - return normalized + return _required_field_names_from_requirements(payload) def _init_request_schema(spec): - """Return the document's InitRequest schema. + """Return the document's InitRequest schema as one object schema. The POST /v1/init request-body schema is authoritative; a document that documents init fields only under components.schemas.InitRequest is - read from there. + read from there. $ref chains are followed and allOf parts merged. """ - schema = ( - (((spec.get("paths") or {}).get("/v1/init") or {}).get("post") or {}) - .get("requestBody", {}) - .get("content", {}) - .get("application/json", {}) - .get("schema", {}) - ) - resolved = _resolve_schema(spec, schema) - if resolved.get("properties"): + schema = _dig(spec, "paths", "/v1/init", "post", "requestBody", "content", "application/json", "schema") + resolved = _object_schema(spec, schema) + if isinstance(resolved.get("properties"), dict) and resolved["properties"]: return resolved - components = (spec.get("components") or {}).get("schemas") or {} - return _resolve_schema(spec, components.get("InitRequest") or {}) + return _object_schema(spec, _dig(spec, "components", "schemas").get("InitRequest") or {}) def _scalar_schema(spec, schema): @@ -224,25 +329,40 @@ def _scalar_schema(spec, schema): else its first alternative, so the form layer sees one type and pattern. """ resolved = _resolve_schema(spec, schema) - alternatives = resolved.get("oneOf") or resolved.get("anyOf") or [] - if resolved.get("type") or not alternatives: + alternatives = resolved.get("oneOf") or resolved.get("anyOf") + if _schema_type(resolved) or not isinstance(alternatives, list) or not alternatives: return resolved - resolved_alternatives = [_resolve_schema(spec, alternative) for alternative in alternatives] + # Alternatives that are not schema objects (or $refs to none) are skipped. + resolved_alternatives = [ + resolved_alternative + for resolved_alternative in (_resolve_schema(spec, alternative) for alternative in alternatives) + if resolved_alternative + ] for alternative in resolved_alternatives: - if alternative.get("type") == "string": + if _schema_type(alternative) == "string": return alternative - return resolved_alternatives[0] + return resolved_alternatives[0] if resolved_alternatives else {} def _extract_fields_from_requirements(spec, required_fields): """Build field specs for the advertised init fields, typed from InitRequest. - A field the document does not describe is typed as a string. + A field the document does not describe is typed as a string, with a + warning: the spec (section 5.2) says the names must match the init + request schema in /openapi.json. """ - properties = _init_request_schema(spec).get("properties") or {} - return [ - _field_spec(name, _scalar_schema(spec, properties.get(name) or {}), True) for name in required_fields - ] + init_schema = _init_request_schema(spec) + properties = init_schema.get("properties") if isinstance(init_schema.get("properties"), dict) else {} + fields = [] + for name in required_fields: + if name not in properties: + logger.warning( + "driver requirement field %r is not described by the contract's InitRequest schema; " + "treating it as a string", + name, + ) + fields.append(_field_spec(name, _scalar_schema(spec, properties.get(name) or {}), True)) + return fields def _extract_driver_requirement_fields(base_url, spec, timeout=10.0): @@ -253,8 +373,8 @@ def _extract_driver_requirement_fields(base_url, spec, timeout=10.0): optional /v1/commands vendor-option extension, if the document has one, follow as optional extras. """ - payload = _fetch_requirements_payload(base_url, timeout=timeout) - fields = _extract_fields_from_requirements(spec, _required_field_names_from_requirements(payload)) + names = _fetch_requirements_payload(base_url, timeout=timeout) + fields = _extract_fields_from_requirements(spec, names) known = {field["name"] for field in fields} for vendor_field in _extract_vendor_option_fields(spec): if vendor_field["name"] in known: @@ -517,27 +637,47 @@ def parse_provider_config_text(config_text): return payload -def _required_field_names(payload): - """Return the names a config payload must supply values for. +def _stored_field_specs(payload): + """Return a config payload's field specs keyed by name, in stored order. - Entries are the field specs written at registration (dicts) or bare - names. A dict entry marked `"required": false` is an optional extra - and is not demanded. + Entries in `required_fields` are the specs written at registration + (dicts with name, type, required, pattern, minimum, maximum, ...). A + bare name is a config written before field types were recorded: it is + treated as a required string field, and a warning says that + re-registering the driver records its types. """ - names = payload.get("required_fields") or [] - if not isinstance(names, list): + entries = payload.get("required_fields") or [] + if not isinstance(entries, list): raise DriverConfigError("required_fields must be a list") - normalized = [] - for entry in names: + + specs = {} + untyped = [] + for entry in entries: if isinstance(entry, dict): - if entry.get("required") is False: - continue name = str(entry.get("name") or "").strip() - else: - name = str(entry).strip() + if name: + specs[name] = entry + continue + name = str(entry).strip() if name: - normalized.append(name) - return normalized + specs[name] = {"name": name, "type": "string", "required": True} + untyped.append(name) + if untyped: + logger.warning( + "driver config lists required_fields without types (%s); they are treated as required " + "strings. Re-register the driver to record the types its contract declares.", + ", ".join(untyped), + ) + return specs + + +def _required_field_names(payload): + """Return the names a config payload must supply values for. + + A spec entry marked `"required": false` is an optional extra and is + not demanded. + """ + return [name for name, spec in _stored_field_specs(payload).items() if spec.get("required") is not False] def _field_values(payload): @@ -548,25 +688,39 @@ def _field_values(payload): return values -def _required_field_specs(payload): - """Return required field metadata keyed by field name.""" - names = payload.get("required_fields") or [] - if not isinstance(names, list): - raise DriverConfigError("required_fields must be a list") +def _is_blank(value): + """Whether a stored value counts as not supplied.""" + return value is None or (isinstance(value, str) and not value.strip()) - specs = {} - for entry in names: - if not isinstance(entry, dict): - continue - name = str(entry.get("name") or "").strip() - if name: - specs[name] = entry - return specs + +def _coerce_aes_key(value): + """Return an aes_key value in one of the spec's AesKeyInput forms. + + The wire forms (spec section 3) are 32 hex characters or an array of + exactly 16 integers 0..255. + """ + if isinstance(value, str): + value = value.strip() + if _AES_KEY_HEX_RE.fullmatch(value): + return value + elif ( + isinstance(value, list) + and len(value) == _AES_KEY_BYTES + and all(isinstance(item, int) and not isinstance(item, bool) and 0 <= item <= 255 for item in value) + ): + return list(value) + raise DriverConfigError( + "field 'aes_key' must be 32 hex characters or an array of {} byte values".format(_AES_KEY_BYTES) + ) def _coerce_field_value(name, value, spec): - """Coerce a raw JSON field value to the type required by the driver.""" - field_type = str((spec or {}).get("type") or "string").strip().lower() + """Coerce a raw JSON field value to the type the driver's contract declares.""" + if name == "aes_key": + return _coerce_aes_key(value) + if isinstance(value, str): + value = value.strip() + field_type = _schema_type(spec) or "string" if field_type == "integer": try: return int(value) @@ -586,28 +740,69 @@ def _coerce_field_value(name, value, spec): if normalized in ("false", "0", "no", "off"): return False raise DriverConfigError("field {!r} must be a boolean".format(name)) - return value + if field_type == "array": + if isinstance(value, list): + return list(value) + raise DriverConfigError("field {!r} must be an array".format(name)) + if field_type == "object": + if isinstance(value, dict): + return dict(value) + raise DriverConfigError("field {!r} must be an object".format(name)) + if isinstance(value, (list, dict)): + raise DriverConfigError("field {!r} must be a string".format(name)) + return value if isinstance(value, str) else str(value) + + +def _check_field_constraints(name, value, spec): + """Enforce the pattern, minimum and maximum the driver's contract declared for a field.""" + pattern = str((spec or {}).get("pattern") or "") + if pattern and isinstance(value, str): + try: + matched = re.search(pattern, value) is not None + except re.error: + logger.warning("field %r declares an unusable pattern %r; not checked", name, pattern) + matched = True + if not matched: + raise DriverConfigError("field {!r} must match the pattern {}".format(name, pattern)) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return + minimum = (spec or {}).get("minimum") + if isinstance(minimum, (int, float)) and not isinstance(minimum, bool) and value < minimum: + raise DriverConfigError("field {!r} must be at least {}".format(name, minimum)) + maximum = (spec or {}).get("maximum") + if isinstance(maximum, (int, float)) and not isinstance(maximum, bool) and value > maximum: + raise DriverConfigError("field {!r} must be at most {}".format(name, maximum)) def validate_provider_config_payload(payload): - """Validate a driver config payload and return the typed init values. + """Validate a driver config payload and return the typed init body. - Every required field needs a value. An optional field left blank is - omitted from the returned `field_values` rather than sent as "". + Every required field needs a non-blank value. Values are coerced to + the types the driver's contract declared and checked against its + pattern/minimum/maximum. Only fields the driver asked for are + returned: an optional field left blank is omitted, and a key the + driver never listed is dropped with a warning. """ - required_fields = _required_field_names(payload) - required_field_specs = _required_field_specs(payload) + specs = _stored_field_specs(payload) + required_fields = [name for name, spec in specs.items() if spec.get("required") is not False] field_values = _field_values(payload) - missing = [ - name for name in required_fields if name not in field_values or field_values[name] in (None, "") - ] + missing = [name for name in required_fields if _is_blank(field_values.get(name))] if missing: raise DriverConfigError("required fields are missing values: {}".format(", ".join(missing))) + + unknown = [name for name in field_values if name not in specs] + if unknown: + logger.warning( + "driver config field_values has keys the driver did not ask for; not sent: %s", ", ".join(unknown) + ) + coerced_field_values = {} - for name, value in field_values.items(): - if value in (None, "") and name not in required_fields: + for name, spec in specs.items(): + if name not in field_values or _is_blank(field_values[name]): continue - coerced_field_values[name] = _coerce_field_value(name, value, required_field_specs.get(name)) + value = _coerce_field_value(name, field_values[name], spec) + _check_field_constraints(name, value, spec) + coerced_field_values[name] = value return { "required_fields": required_fields, "field_values": coerced_field_values, @@ -725,14 +920,51 @@ def initialize_configured_providers_on_startup(timeout=10.0): return results -def save_provider_settings( - service_url, selected_interface, enabled=True, provider_id=None, aes_key="", channel="" -): - """Persist a meter driver entry and return its id.""" +def check_grpc_selection(details): + """Raise ProviderRegistrationError unless the contract supports being driven over gRPC. + + The spec fixes no gRPC port, so gRPC needs a grpc interface with a + target in x-meter-driver. Its ConfigureDriver message has fixed + fields, so the driver's /v1/requirements must list + heartbeat_period_duration and aes_key among the required fields. + """ + grpc_interface = next( + (interface for interface in details.get("interfaces") or [] if interface.get("type") == "grpc"), + None, + ) + if grpc_interface is None: + raise ProviderRegistrationError( + "gRPC is not available: the driver's x-meter-driver block advertises no grpc interface" + ) + if not (grpc_interface.get("target") or grpc_interface.get("address")): + raise ProviderRegistrationError( + "gRPC is not available: the driver's grpc interface advertises no target" + ) + required_names = { + field["name"] for field in details.get("driver_requirement_fields") or [] if field.get("required") + } + missing = [name for name in GRPC_INIT_REQUIRED_FIELDS if name not in required_names] + if missing: + raise ProviderRegistrationError( + "gRPC init (ConfigureDriver) needs the init fields {} but the driver's /v1/requirements " + "does not list: {}".format(", ".join(GRPC_INIT_REQUIRED_FIELDS), ", ".join(missing)) + ) + + +def save_provider_settings(service_url, selected_interface, enabled=True, provider_id=None): + """Persist a meter driver entry and return its id. + + A gRPC selection is refused (ProviderRegistrationError) unless the + contract advertises a grpc target and the init fields gRPC needs; any + other interface the contract does not advertise falls back to its + default interface. + """ details = validate_contract(service_url) selected_interface = (selected_interface or details["default_interface"]).strip().lower() valid_interfaces = {interface["type"] for interface in details.get("interfaces") or []} - if selected_interface not in valid_interfaces: + if selected_interface == "grpc": + check_grpc_selection(details) + elif selected_interface not in valid_interfaces: selected_interface = details["default_interface"] selected_interface_details = next( @@ -798,9 +1030,15 @@ def _normalize_interface_metadata(base_url, spec): Without the block, or without an http entry in it, one http interface at the registered base URL is synthesized and becomes the default. + That leniency goes beyond the spec, which says the block lists the + active interfaces. """ - extension = spec.get(DISCOVERY_EXTENSION) or {} - interface_entries = extension.get("interfaces") or [] + extension = spec.get(DISCOVERY_EXTENSION) + if not isinstance(extension, dict): + extension = {} + interface_entries = extension.get("interfaces") + if not isinstance(interface_entries, list): + interface_entries = [] interfaces = [] seen_types = set() @@ -904,12 +1142,36 @@ def _fallback_interface_metadata(base_url, selected_interface=None): ) -def validate_contract(service_url, timeout=10.0): - """Fetch and validate the driver's OpenAPI contract, then discover its init fields. +def _normalize_path_template(path): + """Return a path with every {param} replaced by {} so parameter names do not matter.""" + return _PATH_PARAM_RE.sub("{}", path) - Follows the spec's integration sequence (section 2): GET /openapi.json, - check it lists every required route, read x-meter-driver, then GET - /v1/requirements. A driver missing any of that is not registrable. + +def _missing_required_operations(paths): + """Return "METHOD /path" labels for the spec's required operations a document lacks. + + A path matches regardless of its parameter names (/v1/nodes/{id} is + /v1/nodes/{node_id}); a trailing slash does not match. + """ + by_template = {} + for path, item in paths.items(): + if isinstance(path, str) and isinstance(item, dict): + by_template.setdefault(_normalize_path_template(path), item) + + missing = [] + for method, path in REQUIRED_CONTRACT_OPERATIONS: + item = by_template.get(_normalize_path_template(path), {}) + if not isinstance(item.get(method), dict): + missing.append("{} {}".format(method.upper(), path)) + return missing + + +def _fetch_contract_document(service_url, timeout=10.0): + """GET /openapi.json and check it is a document a compliant driver would serve. + + Returns `(base_url, openapi_url, document)`. The document must be a + JSON object with `info.title`, a `paths` object listing every required + operation, and, when present, an object-valued x-meter-driver block. """ base_url = normalize_base_url(service_url) openapi_url = get_openapi_url(service_url) @@ -923,71 +1185,136 @@ def validate_contract(service_url, timeout=10.0): spec = response.json() except ValueError as exc: raise ProviderRegistrationError("driver returned invalid JSON") from exc + if not isinstance(spec, dict): + raise ProviderRegistrationError("driver OpenAPI document must be a JSON object") - info = spec.get("info") or {} - paths = spec.get("paths") or {} - missing_paths = [path for path in REQUIRED_CONTRACT_PATHS if path not in paths] - if missing_paths: + info = spec.get("info") + if not isinstance(info, dict) or not info.get("title"): + raise ProviderRegistrationError("driver contract missing info.title") + + paths = spec.get("paths") + if not isinstance(paths, dict): + raise ProviderRegistrationError("driver contract missing paths") + missing = _missing_required_operations(paths) + if missing: raise ProviderRegistrationError( - "driver contract missing required paths: {}".format(", ".join(missing_paths)) + "driver contract missing required routes: {}".format(", ".join(missing)) ) - name = info.get("title") - if not name: - raise ProviderRegistrationError("driver contract missing info.title") + extension = spec.get(DISCOVERY_EXTENSION) + if extension is not None and not isinstance(extension, dict): + raise ProviderRegistrationError("driver contract {} must be an object".format(DISCOVERY_EXTENSION)) + + return base_url, openapi_url, spec - driver_requirement_fields = _extract_driver_requirement_fields(base_url, spec, timeout=timeout) +def _contract_details(base_url, openapi_url, spec): + """Build the details dict for a fetched contract, without init-field discovery.""" + info = spec["info"] return { - "name": str(name), + "name": str(info["title"]), "base_url": base_url, "openapi_url": openapi_url, "service_version": str(info.get("version") or ""), - "driver_requirement_fields": driver_requirement_fields, - "driver_requirement_field_map": {field["name"]: field for field in driver_requirement_fields}, + "driver_requirement_fields": [], + "driver_requirement_field_map": {}, "vendor_option_fields": _extract_vendor_option_fields(spec), "vendor_option_field_map": _vendor_option_field_map(spec), **_normalize_interface_metadata(base_url, spec), } -def get_live_interface_details(service_url, selected_interface=None, timeout=2.0): - """Fetch the current interface inventory advertised by the provider.""" +def inspect_contract(service_url, timeout=10.0): + """Fetch and validate the driver's OpenAPI contract and discover its interfaces. + + One round trip (GET /openapi.json). `driver_requirement_fields` is + empty here: init-field discovery is a second round trip that + registration makes through `validate_contract`. + """ + base_url, openapi_url, spec = _fetch_contract_document(service_url, timeout=timeout) + return _contract_details(base_url, openapi_url, spec) + + +def validate_contract(service_url, timeout=10.0): + """Fetch and validate the driver's OpenAPI contract, then discover its init fields. + + Follows the spec's integration sequence (section 2): GET /openapi.json, + check it lists every required operation, read x-meter-driver, then GET + /v1/requirements typed by the document's InitRequest schema. A driver + missing any of that is not registrable. + """ + base_url, openapi_url, spec = _fetch_contract_document(service_url, timeout=timeout) + details = _contract_details(base_url, openapi_url, spec) + driver_requirement_fields = _extract_driver_requirement_fields(base_url, spec, timeout=timeout) + details["driver_requirement_fields"] = driver_requirement_fields + details["driver_requirement_field_map"] = {field["name"]: field for field in driver_requirement_fields} + return details + + +def _stored_driver_fields(provider): + """Return the field specs recorded in a saved provider's config file, if any.""" + if not provider: + return [] + try: + return list(_stored_field_specs(load_provider_runtime_settings(provider)).values()) + except DriverConfigError: + return [] + + +def get_live_interface_details(service_url, selected_interface=None, timeout=2.0, provider=None): + """Fetch the interface inventory the driver advertises right now. + + Only GET /openapi.json is called, so an advertised gRPC target is + never lost to a slow or failing /v1/requirements. When `provider` is + given, `driver_requirement_fields` come from the fields recorded in + its config file at registration. + """ base_url = normalize_base_url(service_url) try: - provider_data = validate_contract(base_url, timeout=timeout) + details = inspect_contract(base_url, timeout=timeout) except ProviderRegistrationError as exc: - details = _fallback_interface_metadata( - base_url, - selected_interface=selected_interface, - ) + details = _fallback_interface_metadata(base_url, selected_interface=selected_interface) details["error"] = str(exc) - return details - return _apply_selected_interface( - provider_data, - selected_interface=selected_interface, - ) + fields = _stored_driver_fields(provider) + details["driver_requirement_fields"] = fields + details["driver_requirement_field_map"] = {field["name"]: field for field in fields} + return _apply_selected_interface(details, selected_interface=selected_interface) def get_runtime_status(service_url, timeout=2.0, include_gateway_status=True): - """Check driver liveness on GET /v1/healthz and, optionally, gateway state on GET /v1/status.""" + """Check driver liveness on GET /v1/healthz and, optionally, gateway state on GET /v1/status. + + Online means /v1/healthz answered 200 with the spec's HealthResponse + `{"ok": true}`. With `include_gateway_status`, /v1/status must answer + a JSON object (its `connected` and `gateway_type` are reported); a + transport failure there leaves the driver online with no gateway. + """ base_url = normalize_base_url(service_url) healthz_url = base_url.rstrip("/") + "/v1/healthz" status_url = base_url.rstrip("/") + "/v1/status" - try: - response = httpx.get(healthz_url, timeout=timeout) - response.raise_for_status() - except httpx.HTTPError as exc: + + def offline(message): return { "online": False, - "message": str(exc) or "unreachable", + "message": message, "checked_url": healthz_url, "gateway_checked": bool(include_gateway_status), "gateway_active": False, "gateway_type": None, } + try: + response = httpx.get(healthz_url, timeout=timeout) + response.raise_for_status() + health = response.json() + except httpx.HTTPError as exc: + return offline(str(exc) or "unreachable") + except ValueError: + return offline("driver /v1/healthz response is not valid JSON") + if not isinstance(health, dict) or health.get("ok") is not True: + return offline('driver /v1/healthz did not answer {"ok": true}') + status = { "online": True, "message": "online", @@ -1002,9 +1329,12 @@ def get_runtime_status(service_url, timeout=2.0, include_gateway_status=True): gateway_response = httpx.get(status_url, timeout=timeout) gateway_response.raise_for_status() gateway_data = gateway_response.json() - status["gateway_active"] = bool(gateway_data.get("connected")) - status["gateway_type"] = gateway_data.get("gateway_type") except (httpx.HTTPError, ValueError): status["gateway_active"] = False status["gateway_type"] = None + return status + if not isinstance(gateway_data, dict): + return offline("driver /v1/status response is not a JSON object") + status["gateway_active"] = bool(gateway_data.get("connected")) + status["gateway_type"] = gateway_data.get("gateway_type") return status diff --git a/sparkmeter/config/tests/test_configviews.py b/sparkmeter/config/tests/test_configviews.py index ceff2c8..75d3e27 100644 --- a/sparkmeter/config/tests/test_configviews.py +++ b/sparkmeter/config/tests/test_configviews.py @@ -123,7 +123,7 @@ def test_meter_driver_with_saved_provider(self, client, config, monkeypatch): monkeypatch.setattr( configviews, "get_live_interface_details", - lambda base_url, selected_interface=None: _interface_details(), + lambda base_url, selected_interface=None, provider=None: _interface_details(), ) seen = {} @@ -163,7 +163,7 @@ def test_meter_driver_add_post_saves(self, client, config, monkeypatch): monkeypatch.setattr( provider_settings, "get_live_interface_details", - lambda service_url, selected_interface=None, timeout=2.0: details, + lambda service_url, selected_interface=None, timeout=2.0, provider=None: details, ) monkeypatch.setattr( provider_settings, @@ -172,9 +172,7 @@ def test_meter_driver_add_post_saves(self, client, config, monkeypatch): ) saved = {} - def fake_save( - service_url, selected_interface, enabled=True, provider_id=None, aes_key="", channel="" - ): + def fake_save(service_url, selected_interface, enabled=True, provider_id=None): saved["service_url"] = service_url saved["selected_interface"] = selected_interface saved["enabled"] = enabled @@ -212,7 +210,7 @@ def test_meter_driver_edit_get(self, client, config, monkeypatch): monkeypatch.setattr( configviews, "get_live_interface_details", - lambda base_url, selected_interface=None: _interface_details(), + lambda base_url, selected_interface=None, provider=None: _interface_details(), ) response = client.get("/config/meter-driver/driver-1/edit") @@ -236,7 +234,7 @@ def test_meter_driver_config_get(self, client, config, monkeypatch): monkeypatch.setattr( configviews, "get_live_interface_details", - lambda base_url, selected_interface=None: _interface_details(), + lambda base_url, selected_interface=None, provider=None: _interface_details(), ) monkeypatch.setattr( provider_settings, @@ -267,7 +265,7 @@ def _patch_config_editor(self, monkeypatch, provider): monkeypatch.setattr( configviews, "get_live_interface_details", - lambda base_url, selected_interface=None: _interface_details(), + lambda base_url, selected_interface=None, provider=None: _interface_details(), ) monkeypatch.setattr( provider_settings, diff --git a/sparkmeter/config/tests/test_provider_settings.py b/sparkmeter/config/tests/test_provider_settings.py index 8f1e29c..cb637a5 100644 --- a/sparkmeter/config/tests/test_provider_settings.py +++ b/sparkmeter/config/tests/test_provider_settings.py @@ -1,8 +1,15 @@ # -*- coding: utf-8 -*- -"""Meter driver settings tests.""" +"""Meter driver settings tests. +The spec-document fixture (`spec_document`, the spec's own +openapi/meter-driver.yaml as JSON) and the `fake_driver` httpx.get double +come from sparkmeter/conftest.py, which also documents how to regenerate +the fixture after a meter-driver-spec wheel bump. +""" + +import importlib.metadata import json -from pathlib import Path +import logging import httpx import pytest @@ -10,40 +17,38 @@ from sparkmeter.config import provider_settings from sparkmeter.metering.provider_config import configured_provider_url -# The Meter Driver Specification's own openapi/meter-driver.yaml (v1.4.0), -# converted to JSON: exactly what a driver serving nothing but the spec -# answers on GET /openapi.json (its x-meter-driver block lists http + grpc). -_SPEC_DOCUMENT_PATH = Path(__file__).with_name("meter_driver_spec_openapi.json") - # The reference driver's /v1/requirements list (spec section 5.2 example). _REFERENCE_REQUIRED_FIELDS = ("aes_key", "channel", "heartbeat_period_duration") - -def load_spec_document(): - """Return a fresh copy of the spec's OpenAPI document.""" - return json.loads(_SPEC_DOCUMENT_PATH.read_text()) - - -class FakeResponse(object): - """Minimal HTTPX-like response test double.""" - - def __init__(self, payload): - self._payload = payload - - def raise_for_status(self): - """Pretend the response was successful.""" - - def json(self): - """Return the configured JSON payload.""" - return self._payload +# The spec's Required operations (docs/spec/index.md section 4), written out +# rather than derived from the module under test. +_SPEC_REQUIRED_OPERATIONS = ( + ("get", "/v1/requirements"), + ("post", "/v1/init"), + ("post", "/v1/nodes/register"), + ("delete", "/v1/nodes/{node_id}"), + ("post", "/v1/nodes/{node_id}/configure-meter"), + ("post", "/v1/meters/configure"), + ("get", "/v1/events"), + ("get", "/v1/status"), + ("get", "/v1/healthz"), +) + + +def _paths(operations): + """An OpenAPI `paths` object with an empty operation for each (method, path).""" + paths = {} + for method, path in operations: + paths.setdefault(path, {})[method] = {} + return paths def _spec_document(**overrides): - """A minimal document listing the spec's required routes with an http x-meter-driver block.""" + """A minimal document with the spec's required operations and an http x-meter-driver block.""" document = { "openapi": "3.1.0", "info": {"title": "Spec Driver", "version": "1.2.3"}, - "paths": {path: {} for path in provider_settings.REQUIRED_CONTRACT_PATHS}, + "paths": _paths(_SPEC_REQUIRED_OPERATIONS), "x-meter-driver": { "default_interface": "http", "interfaces": [{"type": "http", "label": "HTTP API", "base_url": "http://127.0.0.1:18080"}], @@ -56,41 +61,12 @@ def _spec_document(**overrides): def _http_error(url, status_code=404): request = httpx.Request("GET", url) return httpx.HTTPStatusError( - "failed", request=request, response=httpx.Response(status_code, request=request) + "{} for {}".format(status_code, url), + request=request, + response=httpx.Response(status_code, request=request), ) -def _fake_driver( - document=None, required_fields=("heartbeat_period_duration", "aes_key"), requirements_error=None -): - """Return an httpx.get double for a driver serving /openapi.json and /v1/requirements. - - Every URL it answers is recorded on `fake_get.calls`; anything other - than those two routes is an assertion failure, so a test sees any - stray probe (a legacy /health, a vendor init route) immediately. - """ - document = _spec_document() if document is None else document - calls = [] - - def fake_get(url, timeout): - calls.append(url) - if url.endswith("/v1/requirements"): - if requirements_error is not None: - raise requirements_error - return FakeResponse({"required_fields": list(required_fields)}) - if url.endswith("/openapi.json"): - return FakeResponse(document) - raise AssertionError("unexpected GET {}".format(url)) - - fake_get.calls = calls - return fake_get - - -def _fake_openapi_get(url, timeout): - """A spec-only driver, for tests that only need a saved provider.""" - return _fake_driver()(url, timeout) - - def _vendor_options_extension(**properties): """The optional /v1/commands configure_provider extension a reference driver adds.""" return { @@ -141,24 +117,32 @@ def _with_vendor_options(document, **properties): return document +def _use_temp_config_root(monkeypatch, tmp_path): + """Redirect the module's config directory globals at a temp location.""" + monkeypatch.setattr(provider_settings, "_REPO_ROOT", tmp_path) + monkeypatch.setattr(provider_settings, "_METER_DRIVER_CONFIG_DIR", tmp_path / "meter_driver_configs") + + # --------------------------------------------------------------------------- # validate_contract against the spec's own document # --------------------------------------------------------------------------- -def test_spec_document_fixture_is_the_spec(monkeypatch): - document = load_spec_document() - assert document["info"]["version"] == "1.4.0" - assert document["x-meter-driver"]["default_interface"] == "http" - assert set(provider_settings.REQUIRED_CONTRACT_PATHS) <= set(document["paths"]) - # The spec's required-route list, and nothing the reference driver adds. - assert "/v1/commands" not in document["paths"] +def test_spec_document_fixture_is_the_pinned_spec(spec_document): + # The fixture is regenerated from the spec tag the wheel is built from; + # a wheel bump without regeneration fails here. + assert spec_document["info"]["version"] == importlib.metadata.version("meter-driver-spec") + assert spec_document["x-meter-driver"]["default_interface"] == "http" + for method, path in _SPEC_REQUIRED_OPERATIONS: + assert method in spec_document["paths"][path] + # The spec's routes, and nothing the reference driver adds. + assert "/v1/commands" not in spec_document["paths"] -def test_validate_contract_accepts_a_spec_only_driver(monkeypatch): +def test_validate_contract_accepts_a_spec_only_driver(monkeypatch, spec_document, fake_driver): # A driver serving exactly the spec document, with the reference # /v1/requirements list. This is the meter-driver-emulator case. - fake_get = _fake_driver(load_spec_document(), required_fields=_REFERENCE_REQUIRED_FIELDS) + fake_get = fake_driver(spec_document, required_fields=_REFERENCE_REQUIRED_FIELDS) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) details = provider_settings.validate_contract("http://127.0.0.1:18080") @@ -188,8 +172,10 @@ def test_validate_contract_accepts_a_spec_only_driver(monkeypatch): ] -def test_validate_contract_accepts_the_spec_document_from_its_openapi_url(monkeypatch): - monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(load_spec_document())) +def test_validate_contract_accepts_the_spec_document_from_its_openapi_url( + monkeypatch, spec_document, fake_driver +): + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(spec_document)) details = provider_settings.validate_contract("http://127.0.0.1:18080/openapi.json") @@ -197,12 +183,45 @@ def test_validate_contract_accepts_the_spec_document_from_its_openapi_url(monkey assert details["openapi_url"] == "http://127.0.0.1:18080/openapi.json" +def test_validate_contract_accepts_exactly_the_spec_required_operations(monkeypatch, fake_driver): + # A document whose paths are the nine Required routes with their + # methods and nothing else (no Recommended routes) registers. + document = { + "openapi": "3.1.0", + "info": {"title": "Minimal Driver", "version": "0.1.0"}, + "paths": { + "/v1/requirements": {"get": {}}, + "/v1/init": {"post": {}}, + "/v1/nodes/register": {"post": {}}, + "/v1/nodes/{node_id}": {"delete": {}}, + "/v1/nodes/{node_id}/configure-meter": {"post": {}}, + "/v1/meters/configure": {"post": {}}, + "/v1/events": {"get": {}}, + "/v1/status": {"get": {}}, + "/v1/healthz": {"get": {}}, + }, + } + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) + + details = provider_settings.validate_contract("http://127.0.0.1:18080") + + assert details["name"] == "Minimal Driver" + + +def test_validate_contract_does_not_demand_recommended_routes(monkeypatch, spec_document, fake_driver): + del spec_document["paths"]["/v1/nodes/{node_id}/balance-and-flags"] + del spec_document["paths"]["/v1/shutdown"] + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(spec_document)) + + assert provider_settings.validate_contract("http://127.0.0.1:18080")["name"] == "Meter Driver API" + + # --------------------------------------------------------------------------- # validate_contract: interfaces # --------------------------------------------------------------------------- -def test_validate_contract_discovers_grpc_interface(monkeypatch): +def test_validate_contract_discovers_grpc_interface(monkeypatch, fake_driver): document = _spec_document( **{ "x-meter-driver": { @@ -214,7 +233,7 @@ def test_validate_contract_discovers_grpc_interface(monkeypatch): } } ) - fake_get = _fake_driver(document) + fake_get = fake_driver(document) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) details = provider_settings.validate_contract("http://127.0.0.1:18080") @@ -228,10 +247,10 @@ def test_validate_contract_discovers_grpc_interface(monkeypatch): assert by_type["grpc"]["address"] == "h:50051" -def test_validate_contract_synthesizes_http_when_discovery_block_is_absent(monkeypatch): +def test_validate_contract_synthesizes_http_when_discovery_block_is_absent(monkeypatch, fake_driver): document = _spec_document() del document["x-meter-driver"] - monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(document)) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) details = provider_settings.validate_contract("http://127.0.0.1:18080") @@ -246,17 +265,15 @@ def test_validate_contract_synthesizes_http_when_discovery_block_is_absent(monke assert details["default_interface"] == "http" -def test_validate_contract_ignores_non_spec_discovery_blocks(monkeypatch): - # A block under any other name is not the spec's; it is not read. The - # reference driver's pre-spec block name is assembled here so that no - # source line in sparkmeter/ carries it verbatim. +def test_validate_contract_ignores_non_spec_discovery_blocks(monkeypatch, fake_driver): + # A block under any other name is not the spec's; it is not read. document = _spec_document() del document["x-meter-driver"] - document["-".join(["x", "open", "thunder"])] = { + document["x-vendor-extension"] = { "default_interface": "grpc", "interfaces": [{"type": "grpc", "target": "h:50051"}], } - monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(document)) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) details = provider_settings.validate_contract("http://127.0.0.1:18080") @@ -264,62 +281,144 @@ def test_validate_contract_ignores_non_spec_discovery_blocks(monkeypatch): assert details["default_interface"] == "http" +def test_validate_contract_rejects_a_non_object_discovery_block(monkeypatch, fake_driver): + document = _spec_document(**{"x-meter-driver": ["http"]}) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) + + with pytest.raises(provider_settings.ProviderRegistrationError, match="x-meter-driver must be an object"): + provider_settings.validate_contract("http://127.0.0.1:18080") + + # --------------------------------------------------------------------------- # validate_contract: required routes # --------------------------------------------------------------------------- -def test_required_contract_paths_are_the_spec_required_routes(): - assert provider_settings.REQUIRED_CONTRACT_PATHS == ( - "/v1/requirements", - "/v1/init", - "/v1/nodes/register", - "/v1/nodes/{node_id}", - "/v1/nodes/{node_id}/configure-meter", - "/v1/meters/configure", - "/v1/events", - "/v1/status", - "/v1/healthz", - ) +def test_required_contract_operations_are_the_spec_required_routes(): + assert provider_settings.REQUIRED_CONTRACT_OPERATIONS == _SPEC_REQUIRED_OPERATIONS -def test_validate_contract_rejects_missing_paths_naming_them(monkeypatch): +def test_validate_contract_rejects_missing_routes_naming_them(monkeypatch, fake_driver): document = _spec_document() del document["paths"]["/v1/requirements"] del document["paths"]["/v1/init"] del document["paths"]["/v1/healthz"] - fake_get = _fake_driver(document) + fake_get = fake_driver(document) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) with pytest.raises(provider_settings.ProviderRegistrationError) as exc: provider_settings.validate_contract("http://127.0.0.1:18080") - assert "missing required paths" in str(exc.value) - assert "/v1/requirements, /v1/init, /v1/healthz" in str(exc.value) + assert "missing required routes" in str(exc.value) + assert "GET /v1/requirements, POST /v1/init, GET /v1/healthz" in str(exc.value) # Rejected on the document alone; requirements are never probed. assert fake_get.calls == ["http://127.0.0.1:18080/openapi.json"] -def test_validate_contract_does_not_require_vendor_routes(monkeypatch): +def test_validate_contract_checks_the_method_not_just_the_path(monkeypatch, fake_driver): + document = _spec_document() + document["paths"]["/v1/init"] = {"get": {}} # the spec's init is a POST + document["paths"]["/v1/nodes/{node_id}"] = {"post": {}} # the spec's unregister is a DELETE + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) + + with pytest.raises(provider_settings.ProviderRegistrationError) as exc: + provider_settings.validate_contract("http://127.0.0.1:18080") + + assert "POST /v1/init" in str(exc.value) + assert "DELETE /v1/nodes/{node_id}" in str(exc.value) + assert "/v1/healthz" not in str(exc.value) + + +def test_validate_contract_matches_path_templates_by_position_not_parameter_name(monkeypatch, fake_driver): + document = _spec_document() + document["paths"]["/v1/nodes/{id}"] = document["paths"].pop("/v1/nodes/{node_id}") + document["paths"]["/v1/nodes/{meter}/configure-meter"] = document["paths"].pop( + "/v1/nodes/{node_id}/configure-meter" + ) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) + + assert provider_settings.validate_contract("http://127.0.0.1:18080")["name"] == "Spec Driver" + + +def test_validate_contract_rejects_trailing_slashes(monkeypatch, fake_driver): + document = _spec_document() + document["paths"]["/v1/init/"] = document["paths"].pop("/v1/init") + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) + + with pytest.raises(provider_settings.ProviderRegistrationError, match="POST /v1/init"): + provider_settings.validate_contract("http://127.0.0.1:18080") + + +def test_validate_contract_does_not_require_vendor_routes(monkeypatch, fake_driver): # /v1/commands is a reference-driver extension, not a spec route. document = _spec_document() assert "/v1/commands" not in document["paths"] - monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(document)) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) details = provider_settings.validate_contract("http://127.0.0.1:18080") assert details["name"] == "Spec Driver" -def test_validate_contract_rejects_a_document_with_only_vendor_routes(monkeypatch): - document = _spec_document(paths={"/v1/commands": {}, "/v1/events": {}}) - monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(document)) +def test_validate_contract_rejects_a_document_with_only_vendor_routes(monkeypatch, fake_driver): + document = _spec_document(paths={"/v1/commands": {"post": {}}, "/v1/events": {"get": {}}}) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) with pytest.raises(provider_settings.ProviderRegistrationError) as exc: provider_settings.validate_contract("http://127.0.0.1:18080") - assert "/v1/requirements" in str(exc.value) - assert "/v1/init" in str(exc.value) + assert "GET /v1/requirements" in str(exc.value) + assert "POST /v1/init" in str(exc.value) + + +# --------------------------------------------------------------------------- +# validate_contract: malformed documents never raise anything but +# ProviderRegistrationError +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "document, message", + [ + (["not", "an", "object"], "must be a JSON object"), + (_spec_document(paths=["/v1/init"]), "missing paths"), + (_spec_document(info="Spec Driver"), "info.title"), + (_spec_document(info={"version": "1"}), "info.title"), + ], +) +def test_validate_contract_rejects_malformed_documents(monkeypatch, fake_driver, document, message): + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) + + with pytest.raises(provider_settings.ProviderRegistrationError, match=message): + provider_settings.validate_contract("http://127.0.0.1:18080") + + +def test_validate_contract_tolerates_non_dict_path_items_and_operations(monkeypatch, fake_driver): + document = _spec_document() + document["paths"]["/v1/shutdown"] = "not-a-dict" + document["paths"]["/v1/status"] = {"get": "not-a-dict"} + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) + + with pytest.raises(provider_settings.ProviderRegistrationError) as exc: + provider_settings.validate_contract("http://127.0.0.1:18080") + + assert str(exc.value).endswith("missing required routes: GET /v1/status") + + +def test_validate_contract_tolerates_refs_to_non_dicts(monkeypatch, spec_document, fake_driver): + spec_document["components"]["schemas"]["InitRequest"]["properties"]["aes_key"] = {"$ref": "#/tags"} + spec_document["components"]["schemas"]["InitRequest"]["properties"]["channel"] = { + "oneOf": ["integer", 7, {"type": "integer"}] + } + monkeypatch.setattr( + provider_settings.httpx, "get", fake_driver(spec_document, required_fields=("aes_key", "channel")) + ) + + fields = provider_settings.validate_contract("http://127.0.0.1:18080")["driver_requirement_field_map"] + + # A $ref to a list is an empty schema (string); non-dict alternatives are skipped. + assert fields["aes_key"]["type"] == "string" + assert fields["channel"]["type"] == "integer" # --------------------------------------------------------------------------- @@ -327,15 +426,15 @@ def test_validate_contract_rejects_a_document_with_only_vendor_routes(monkeypatc # --------------------------------------------------------------------------- -def test_validate_contract_returns_requirements_in_driver_order(monkeypatch): - fake_get = _fake_driver(required_fields=("zeta", "alpha", "heartbeat_period_duration")) +def test_validate_contract_returns_requirements_in_driver_order(monkeypatch, spec_document, fake_driver): + fake_get = fake_driver(spec_document, required_fields=("channel", "aes_key", "heartbeat_period_duration")) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) details = provider_settings.validate_contract("http://127.0.0.1:18080") assert [field["name"] for field in details["driver_requirement_fields"]] == [ - "zeta", - "alpha", + "channel", + "aes_key", "heartbeat_period_duration", ] assert fake_get.calls == [ @@ -344,39 +443,51 @@ def test_validate_contract_returns_requirements_in_driver_order(monkeypatch): ] -def test_validate_contract_types_undocumented_requirements_as_string(monkeypatch): - # The minimal document has no InitRequest schema, so every field is a string. +def test_validate_contract_types_undocumented_requirements_as_string_with_a_warning( + monkeypatch, caplog, fake_driver +): + # The minimal document has no InitRequest schema, so every field is a + # string, and each is reported: the spec says the names must match the + # init request schema. monkeypatch.setattr( - provider_settings.httpx, "get", _fake_driver(required_fields=("aes_key", "site_token")) + provider_settings.httpx, + "get", + fake_driver(_spec_document(), required_fields=("aes_key", "site_token")), ) - details = provider_settings.validate_contract("http://127.0.0.1:18080") + with caplog.at_level(logging.WARNING): + details = provider_settings.validate_contract("http://127.0.0.1:18080") fields = details["driver_requirement_field_map"] assert fields["aes_key"]["type"] == "string" assert fields["site_token"]["type"] == "string" assert fields["site_token"]["required"] is True + warned = [record.getMessage() for record in caplog.records if "not described" in record.getMessage()] + assert any("'site_token'" in message for message in warned) + assert any("'aes_key'" in message for message in warned) -def test_validate_contract_types_requirements_from_the_init_request_schema(monkeypatch): - document = load_spec_document() +def test_validate_contract_types_requirements_from_the_init_request_schema( + monkeypatch, caplog, spec_document, fake_driver +): # A driver with different init fields documents them in InitRequest # (spec section 5.3) and lists them on /v1/requirements. - document["components"]["schemas"]["InitRequest"]["properties"]["site_token"] = { + spec_document["components"]["schemas"]["InitRequest"]["properties"]["site_token"] = { "type": "string", "title": "Site token", "pattern": "^[a-z]+$", } - document["components"]["schemas"]["InitRequest"]["properties"]["poll_seconds"] = { + spec_document["components"]["schemas"]["InitRequest"]["properties"]["poll_seconds"] = { "type": "integer", "minimum": 5, "maximum": 3600, "default": 60, } - fake_get = _fake_driver(document, required_fields=("site_token", "poll_seconds")) + fake_get = fake_driver(spec_document, required_fields=("site_token", "poll_seconds")) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) - details = provider_settings.validate_contract("http://127.0.0.1:18080") + with caplog.at_level(logging.WARNING): + details = provider_settings.validate_contract("http://127.0.0.1:18080") fields = details["driver_requirement_field_map"] assert fields["site_token"] == { @@ -394,47 +505,63 @@ def test_validate_contract_types_requirements_from_the_init_request_schema(monke assert fields["poll_seconds"]["minimum"] == 5 assert fields["poll_seconds"]["maximum"] == 3600 assert fields["poll_seconds"]["default"] == 60 + assert not any("not described" in record.getMessage() for record in caplog.records) -def test_validate_contract_requires_the_requirements_probe(monkeypatch): +def test_validate_contract_requires_the_requirements_probe_and_names_the_cause(monkeypatch, fake_driver): url = "http://127.0.0.1:18080/v1/requirements" - monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(requirements_error=_http_error(url))) + monkeypatch.setattr( + provider_settings.httpx, "get", fake_driver(_spec_document(), requirements_error=_http_error(url)) + ) with pytest.raises(provider_settings.ProviderRegistrationError) as exc: provider_settings.validate_contract("http://127.0.0.1:18080") assert "/v1/requirements" in str(exc.value) + # The underlying transport error is part of the message the form shows. + assert "404 for http://127.0.0.1:18080/v1/requirements" in str(exc.value) -def test_validate_contract_requires_the_requirements_probe_even_with_vendor_options(monkeypatch): +def test_validate_contract_requires_the_requirements_probe_even_with_vendor_options(monkeypatch, fake_driver): # The vendor-option schema is not a substitute for /v1/requirements. document = _with_vendor_options(_spec_document(), aes_key={"type": "string"}) monkeypatch.setattr( provider_settings.httpx, "get", - _fake_driver(document, requirements_error=httpx.ConnectError("down")), + fake_driver(document, requirements_error=httpx.ConnectError("down")), ) - with pytest.raises(provider_settings.ProviderRegistrationError): + with pytest.raises(provider_settings.ProviderRegistrationError, match="down"): provider_settings.validate_contract("http://127.0.0.1:18080") -def test_validate_contract_rejects_malformed_requirements(monkeypatch): +@pytest.mark.parametrize( + "payload", + [ + {"fields": ["aes_key"]}, + {"required_fields": "aes_key"}, + {"required_fields": ["aes_key", 7]}, + {"required_fields": [None]}, + {"required_fields": ["aes_key", " "]}, + ["aes_key"], + ], +) +def test_validate_contract_rejects_malformed_requirements(monkeypatch, fake_json_response, payload): def fake_get(url, timeout): if url.endswith("/v1/requirements"): - return FakeResponse({"fields": ["aes_key"]}) - return FakeResponse(_spec_document()) + return fake_json_response(payload) + return fake_json_response(_spec_document()) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) with pytest.raises(provider_settings.ProviderRegistrationError) as exc: provider_settings.validate_contract("http://127.0.0.1:18080") - assert "required_fields" in str(exc.value) + assert "requirements response" in str(exc.value) -def test_validate_contract_accepts_a_driver_requiring_no_init_fields(monkeypatch): - monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(required_fields=())) +def test_validate_contract_accepts_a_driver_requiring_no_init_fields(monkeypatch, fake_driver): + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document(), required_fields=())) details = provider_settings.validate_contract("http://127.0.0.1:18080") @@ -442,14 +569,102 @@ def test_validate_contract_accepts_a_driver_requiring_no_init_fields(monkeypatch assert details["driver_requirement_field_map"] == {} +# --------------------------------------------------------------------------- +# validate_contract: InitRequest schema composition +# --------------------------------------------------------------------------- + + +def test_validate_contract_reads_init_request_composed_with_all_of(monkeypatch, spec_document, fake_driver): + schemas = spec_document["components"]["schemas"] + schemas["BaseInit"] = { + "type": "object", + "required": ["site_token"], + "properties": {"site_token": {"type": "string", "pattern": "^[a-z]+$"}}, + } + schemas["InitRequest"] = { + "allOf": [ + {"$ref": "#/components/schemas/BaseInit"}, + {"type": "object", "properties": {"poll_seconds": {"type": "integer", "minimum": 5}}}, + ] + } + monkeypatch.setattr( + provider_settings.httpx, + "get", + fake_driver(spec_document, required_fields=("site_token", "poll_seconds")), + ) + + fields = provider_settings.validate_contract("http://127.0.0.1:18080")["driver_requirement_field_map"] + + assert fields["site_token"]["pattern"] == "^[a-z]+$" + assert fields["poll_seconds"]["type"] == "integer" + assert fields["poll_seconds"]["minimum"] == 5 + + +def test_validate_contract_follows_ref_chains(monkeypatch, spec_document, fake_driver): + schemas = spec_document["components"]["schemas"] + schemas["InitAlias"] = {"$ref": "#/components/schemas/InitRequest"} + schemas["InitAliasAlias"] = {"$ref": "#/components/schemas/InitAlias"} + spec_document["paths"]["/v1/init"]["post"]["requestBody"]["content"]["application/json"]["schema"] = { + "$ref": "#/components/schemas/InitAliasAlias" + } + schemas["HexAlias"] = {"$ref": "#/components/schemas/AesKeyInput"} + schemas["InitRequest"]["properties"]["aes_key"] = {"$ref": "#/components/schemas/HexAlias"} + monkeypatch.setattr( + provider_settings.httpx, + "get", + fake_driver(spec_document, required_fields=("heartbeat_period_duration", "aes_key")), + ) + + fields = provider_settings.validate_contract("http://127.0.0.1:18080")["driver_requirement_field_map"] + + assert fields["heartbeat_period_duration"]["type"] == "integer" + assert fields["aes_key"]["pattern"] == "^[A-Fa-f0-9]{32}$" + + +def test_validate_contract_stops_at_ref_cycles(monkeypatch, spec_document, fake_driver): + schemas = spec_document["components"]["schemas"] + schemas["Loop"] = {"$ref": "#/components/schemas/Loop"} + schemas["InitRequest"]["properties"]["aes_key"] = {"$ref": "#/components/schemas/Loop"} + schemas["InitRequest"]["allOf"] = [{"$ref": "#/components/schemas/InitRequest"}] + monkeypatch.setattr( + provider_settings.httpx, "get", fake_driver(spec_document, required_fields=("aes_key",)) + ) + + fields = provider_settings.validate_contract("http://127.0.0.1:18080")["driver_requirement_field_map"] + + assert fields["aes_key"]["type"] == "string" + + +def test_validate_contract_reads_openapi_31_type_arrays(monkeypatch, spec_document, fake_driver): + schemas = spec_document["components"]["schemas"] + schemas["InitRequest"]["properties"]["channel"] = {"type": ["null", "integer"], "minimum": 11} + schemas["InitRequest"]["properties"]["region"] = {"type": ["string", "null"]} + schemas["InitRequest"]["properties"]["aes_key"] = { + "oneOf": [{"type": ["array"]}, {"type": ["string", "null"], "pattern": "^[a-f]+$"}] + } + monkeypatch.setattr( + provider_settings.httpx, + "get", + fake_driver(spec_document, required_fields=("channel", "region", "aes_key")), + ) + + fields = provider_settings.validate_contract("http://127.0.0.1:18080")["driver_requirement_field_map"] + + assert fields["channel"]["type"] == "integer" + assert fields["channel"]["minimum"] == 11 + assert fields["region"]["type"] == "string" + assert fields["aes_key"]["type"] == "string" + assert fields["aes_key"]["pattern"] == "^[a-f]+$" + + # --------------------------------------------------------------------------- # validate_contract: optional /v1/commands vendor options # --------------------------------------------------------------------------- -def test_validate_contract_appends_vendor_options_as_optional_extras(monkeypatch): +def test_validate_contract_appends_vendor_options_as_optional_extras(monkeypatch, spec_document, fake_driver): document = _with_vendor_options( - load_spec_document(), + spec_document, aes_key={"type": "string", "title": "AES key", "pattern": "[0-9a-fA-F]{32}"}, channel={"type": "integer", "title": "Channel", "minimum": 11, "maximum": 26}, region={"type": "string", "title": "Region", "description": "Radio regulatory region."}, @@ -457,7 +672,7 @@ def test_validate_contract_appends_vendor_options_as_optional_extras(monkeypatch monkeypatch.setattr( provider_settings.httpx, "get", - _fake_driver(document, required_fields=("heartbeat_period_duration", "aes_key")), + fake_driver(document, required_fields=("heartbeat_period_duration", "aes_key")), ) details = provider_settings.validate_contract("http://127.0.0.1:18080") @@ -481,8 +696,10 @@ def test_validate_contract_appends_vendor_options_as_optional_extras(monkeypatch assert set(details["vendor_option_field_map"]) == {"aes_key", "channel", "region"} -def test_validate_contract_appends_nothing_without_vendor_options(monkeypatch): - monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(required_fields=("aes_key",))) +def test_validate_contract_appends_nothing_without_vendor_options(monkeypatch, fake_driver): + monkeypatch.setattr( + provider_settings.httpx, "get", fake_driver(_spec_document(), required_fields=("aes_key",)) + ) details = provider_settings.validate_contract("http://127.0.0.1:18080") @@ -491,16 +708,16 @@ def test_validate_contract_appends_nothing_without_vendor_options(monkeypatch): assert details["vendor_option_field_map"] == {} -def test_configured_provider_url_uses_saved_setting(session, monkeypatch): - monkeypatch.setattr(provider_settings.httpx, "get", _fake_openapi_get) +def test_configured_provider_url_uses_saved_setting(session, monkeypatch, fake_driver): + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document())) provider_settings.save_provider_settings("http://127.0.0.1:18080", "http") session.commit() assert configured_provider_url(default="") == "http://127.0.0.1:18080" -def test_configured_provider_url_ignores_env_override(session, monkeypatch): - monkeypatch.setattr(provider_settings.httpx, "get", _fake_openapi_get) +def test_configured_provider_url_ignores_env_override(session, monkeypatch, fake_driver): + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document())) provider_settings.save_provider_settings("http://127.0.0.1:18080", "http") session.commit() monkeypatch.setenv("METERING_PROVIDER_URL", "http://127.0.0.1:28080") @@ -547,12 +764,6 @@ def fake_initialize_provider_sync(provider, field_values, provider_details=None) } -def _use_temp_config_root(monkeypatch, tmp_path): - """Redirect the module's config directory globals at a temp location.""" - monkeypatch.setattr(provider_settings, "_REPO_ROOT", tmp_path) - monkeypatch.setattr(provider_settings, "_METER_DRIVER_CONFIG_DIR", tmp_path / "meter_driver_configs") - - # --------------------------------------------------------------------------- # JSON-pointer / schema resolution helpers # --------------------------------------------------------------------------- @@ -561,6 +772,7 @@ def _use_temp_config_root(monkeypatch, tmp_path): def test_resolve_local_ref_rejects_non_local_refs(): assert provider_settings._resolve_local_ref({}, "") is None assert provider_settings._resolve_local_ref({}, "https://x/y") is None + assert provider_settings._resolve_local_ref({}, 7) is None def test_resolve_local_ref_walks_and_unescapes_tokens(): @@ -569,8 +781,9 @@ def test_resolve_local_ref_walks_and_unescapes_tokens(): assert provider_settings._resolve_local_ref(spec, "#/components/sch~0emas/a~1b") == {"leaf": 1} -def test_resolve_local_ref_returns_none_for_missing_node(): +def test_resolve_local_ref_returns_none_for_missing_or_non_dict_nodes(): assert provider_settings._resolve_local_ref({"a": {}}, "#/a/missing") is None + assert provider_settings._resolve_local_ref({"a": [1, 2]}, "#/a/0") is None def test_resolve_schema_handles_non_dict_ref_and_plain(): @@ -579,6 +792,48 @@ def test_resolve_schema_handles_non_dict_ref_and_plain(): assert provider_settings._resolve_schema(spec, {"$ref": "#/components/schemas/Foo"}) == {"type": "object"} assert provider_settings._resolve_schema({}, {"$ref": "#/nope"}) == {} assert provider_settings._resolve_schema({}, {"type": "string"}) == {"type": "string"} + # A $ref to something that is not a schema object is an empty schema. + assert provider_settings._resolve_schema({"x": [1]}, {"$ref": "#/x"}) == {} + assert provider_settings._resolve_schema({}, {"$ref": 7}) == {} + + +def test_resolve_schema_follows_chains_and_stops_at_cycles(): + spec = { + "a": {"$ref": "#/b"}, + "b": {"$ref": "#/c"}, + "c": {"type": "integer"}, + "loop1": {"$ref": "#/loop2"}, + "loop2": {"$ref": "#/loop1"}, + } + assert provider_settings._resolve_schema(spec, {"$ref": "#/a"}) == {"type": "integer"} + assert provider_settings._resolve_schema(spec, {"$ref": "#/loop1"}) == {} + + +def test_schema_type_reads_strings_and_type_arrays(): + assert provider_settings._schema_type({"type": "Integer"}) == "integer" + assert provider_settings._schema_type({"type": ["null", "string"]}) == "string" + assert provider_settings._schema_type({"type": ["null"]}) == "" + assert provider_settings._schema_type({}) == "" + assert provider_settings._schema_type(None) == "" + + +def test_object_schema_merges_all_of_parts(): + spec = {"components": {"schemas": {"Base": {"required": ["a"], "properties": {"a": {"type": "string"}}}}}} + merged = provider_settings._object_schema( + spec, + { + "allOf": [ + {"$ref": "#/components/schemas/Base"}, + {"required": ["b"], "properties": {"b": {"type": "integer"}}}, + "not-a-schema", + ], + "description": "kept", + }, + ) + assert merged["properties"] == {"a": {"type": "string"}, "b": {"type": "integer"}} + assert merged["required"] == ["a", "b"] + assert merged["description"] == "kept" + assert "allOf" not in merged def test_command_type_values_reads_const_and_enum(): @@ -589,6 +844,7 @@ def test_command_type_values_reads_const_and_enum(): {}, {"properties": {"command_type": {"enum": ["A", " b ", ""]}}} ) assert values == {"a", "b"} + assert provider_settings._command_type_values({}, {"properties": "nope"}) == set() def test_find_configure_provider_schema_falls_back_to_components(): @@ -612,6 +868,7 @@ def test_find_configure_provider_schema_falls_back_to_components(): def test_find_configure_provider_schema_returns_empty_when_absent(): assert provider_settings._find_configure_provider_schema({"paths": {}, "components": {}}) == {} + assert provider_settings._find_configure_provider_schema({"paths": "nope", "components": []}) == {} # --------------------------------------------------------------------------- @@ -619,22 +876,22 @@ def test_find_configure_provider_schema_returns_empty_when_absent(): # --------------------------------------------------------------------------- -def test_fetch_requirements_payload_rejects_non_object(monkeypatch): - monkeypatch.setattr(provider_settings.httpx, "get", lambda url, timeout: FakeResponse(["nope"])) +def test_fetch_requirements_payload_rejects_non_object(monkeypatch, fake_json_response): + monkeypatch.setattr(provider_settings.httpx, "get", lambda url, timeout: fake_json_response(["nope"])) with pytest.raises(provider_settings.ProviderRegistrationError): provider_settings._fetch_requirements_payload("http://127.0.0.1:18080") -def test_fetch_requirements_payload_wraps_transport_and_json_errors(monkeypatch): +def test_fetch_requirements_payload_wraps_transport_and_json_errors(monkeypatch, fake_json_response): def boom(url, timeout): raise httpx.ConnectError("down") monkeypatch.setattr(provider_settings.httpx, "get", boom) with pytest.raises(provider_settings.ProviderRegistrationError) as exc: provider_settings._fetch_requirements_payload("http://127.0.0.1:18080") - assert "/v1/requirements" in str(exc.value) + assert "/v1/requirements: down" in str(exc.value) - class BadJSON(FakeResponse): + class BadJSON(fake_json_response): def json(self): raise ValueError("bad") @@ -644,22 +901,36 @@ def json(self): assert "not valid JSON" in str(exc.value) -def test_required_field_names_from_requirements_keeps_order_and_dedups(): - names = provider_settings._required_field_names_from_requirements( - {"required_fields": ["b", " a ", "", "b", 7]} +def test_fetch_requirements_payload_returns_the_names(monkeypatch, fake_json_response): + monkeypatch.setattr( + provider_settings.httpx, + "get", + lambda url, timeout: fake_json_response({"required_fields": ["b", "a", "b"]}), ) - assert names == ["b", "a", "7"] + assert provider_settings._fetch_requirements_payload("http://127.0.0.1:18080") == ["b", "a"] -def test_required_field_names_from_requirements_rejects_non_list(): - with pytest.raises(provider_settings.ProviderRegistrationError): - provider_settings._required_field_names_from_requirements({"required_fields": "aes_key"}) - with pytest.raises(provider_settings.ProviderRegistrationError): - provider_settings._required_field_names_from_requirements({}) +def test_required_field_names_from_requirements_validates_the_spec_model(): + names = provider_settings._required_field_names_from_requirements({"required_fields": ["b", "a", "b"]}) + assert names == ["b", "a"] + # RequirementsResponse.required_fields is array[string]: nothing is stringified. + for payload in ({"required_fields": "aes_key"}, {"required_fields": [7]}, {}, {"required_fields": [""]}): + with pytest.raises(provider_settings.ProviderRegistrationError): + provider_settings._required_field_names_from_requirements(payload) -def test_init_request_schema_reads_the_init_request_body(): - schema = provider_settings._init_request_schema(load_spec_document()) +def test_required_field_names_error_names_each_invalid_location(): + # The message joins pydantic's error locations and messages so the form + # can show which part of the /v1/requirements payload was wrong. + with pytest.raises(provider_settings.ProviderRegistrationError) as excinfo: + provider_settings._required_field_names_from_requirements({"required_fields": ["a", 7]}) + message = str(excinfo.value) + assert "required_fields.1" in message + assert "string" in message + + +def test_init_request_schema_reads_the_init_request_body(spec_document): + schema = provider_settings._init_request_schema(spec_document) assert set(schema["properties"]) == {"heartbeat_period_duration", "channel", "aes_key"} assert schema["required"] == ["heartbeat_period_duration", "aes_key"] @@ -671,12 +942,12 @@ def test_init_request_schema_falls_back_to_components_init_request(): } assert provider_settings._init_request_schema(spec) == {"properties": {"site_token": {"type": "string"}}} assert provider_settings._init_request_schema({"paths": {}, "components": {}}) == {} + assert provider_settings._init_request_schema({"paths": [], "components": "x"}) == {} -def test_scalar_schema_reduces_one_of_to_the_string_alternative(): - spec = load_spec_document() - aes_key = spec["components"]["schemas"]["InitRequest"]["properties"]["aes_key"] - resolved = provider_settings._scalar_schema(spec, aes_key) +def test_scalar_schema_reduces_one_of_to_the_string_alternative(spec_document): + aes_key = spec_document["components"]["schemas"]["InitRequest"]["properties"]["aes_key"] + resolved = provider_settings._scalar_schema(spec_document, aes_key) assert resolved["type"] == "string" assert resolved["pattern"] == "^[A-Fa-f0-9]{32}$" @@ -691,11 +962,14 @@ def test_scalar_schema_falls_back_to_the_first_alternative_and_passes_plain_sche "oneOf": [{"type": "string"}], } assert provider_settings._scalar_schema({}, {}) == {} + assert provider_settings._scalar_schema({}, {"oneOf": "nope"}) == {"oneOf": "nope"} + assert provider_settings._scalar_schema({}, {"oneOf": ["nope", 3]}) == {} -def test_extract_fields_from_requirements_types_from_init_request_else_string(): - spec = load_spec_document() - fields = provider_settings._extract_fields_from_requirements(spec, ["channel", "aes_key", "site_token"]) +def test_extract_fields_from_requirements_types_from_init_request_else_string(spec_document): + fields = provider_settings._extract_fields_from_requirements( + spec_document, ["channel", "aes_key", "site_token"] + ) by_name = {field["name"]: field for field in fields} assert by_name["channel"]["type"] == "integer" assert by_name["aes_key"]["type"] == "string" @@ -703,9 +977,9 @@ def test_extract_fields_from_requirements_types_from_init_request_else_string(): assert all(field["required"] for field in fields) -def test_extract_driver_requirement_fields_probes_without_vendor_options(monkeypatch): +def test_extract_driver_requirement_fields_probes_without_vendor_options(monkeypatch, fake_driver): # No /v1/commands schema: the probe still happens and is the whole answer. - fake_get = _fake_driver(required_fields=("aes_key",)) + fake_get = fake_driver({}, required_fields=("aes_key",)) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) fields = provider_settings._extract_driver_requirement_fields("http://127.0.0.1:18080", {"paths": {}}) @@ -795,8 +1069,14 @@ def test_normalize_interface_metadata_defaults_to_http_when_unknown(): assert details["default_interface"] == "http" -def test_normalize_interface_metadata_reads_the_spec_block(): - details = provider_settings._normalize_interface_metadata("http://base", load_spec_document()) +def test_normalize_interface_metadata_tolerates_malformed_blocks(): + for spec in ({"x-meter-driver": "http"}, {"x-meter-driver": {"interfaces": "grpc"}}): + details = provider_settings._normalize_interface_metadata("http://base", spec) + assert [interface["type"] for interface in details["interfaces"]] == ["http"] + + +def test_normalize_interface_metadata_reads_the_spec_block(spec_document): + details = provider_settings._normalize_interface_metadata("http://base", spec_document) assert details["default_interface"] == "http" assert details["interfaces"] == [ { @@ -858,8 +1138,8 @@ def boom(url, timeout): assert "could not fetch" in str(exc.value) -def test_validate_contract_rejects_invalid_json(monkeypatch): - class BadJSON(FakeResponse): +def test_validate_contract_rejects_invalid_json(monkeypatch, fake_json_response): + class BadJSON(fake_json_response): def json(self): raise ValueError("bad") @@ -869,8 +1149,8 @@ def json(self): assert "invalid JSON" in str(exc.value) -def test_validate_contract_requires_info_title(monkeypatch): - monkeypatch.setattr(provider_settings.httpx, "get", _fake_driver(_spec_document(info={}))) +def test_validate_contract_requires_info_title(monkeypatch, fake_driver): + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document(info={}))) with pytest.raises(provider_settings.ProviderRegistrationError) as exc: provider_settings.validate_contract("http://127.0.0.1:18080") assert "info.title" in str(exc.value) @@ -893,21 +1173,63 @@ def boom(url, timeout): assert details["selected_interface"] == "grpc" -def test_get_live_interface_details_applies_selection_on_success(monkeypatch): - monkeypatch.setattr(provider_settings.httpx, "get", _fake_openapi_get) +def test_get_live_interface_details_applies_selection_on_success(monkeypatch, fake_driver): + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document())) details = provider_settings.get_live_interface_details("http://127.0.0.1:18080") assert details["selected_interface"] == "http" -def test_get_runtime_status_reports_gateway_when_connected(monkeypatch): +def test_get_live_interface_details_does_not_probe_requirements(monkeypatch, spec_document, fake_driver): + # Interface discovery is one round trip; a slow or failing + # /v1/requirements cannot lose the advertised gRPC target. + fake_get = fake_driver(spec_document, requirements_error=httpx.ConnectError("slow")) + monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + + details = provider_settings.get_live_interface_details( + "http://127.0.0.1:18080", selected_interface="grpc" + ) + + assert "error" not in details + assert details["selected_interface"] == "grpc" + assert details["selected_interface_details"]["target"] == "127.0.0.1:50051" + assert details["driver_requirement_fields"] == [] + assert fake_get.calls == ["http://127.0.0.1:18080/openapi.json"] + + +def test_get_live_interface_details_reads_recorded_fields_for_a_provider(monkeypatch, fake_driver): + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document())) + recorded = [{"name": "aes_key", "type": "string", "required": True}] + monkeypatch.setattr( + provider_settings, "load_provider_runtime_settings", lambda provider: {"required_fields": recorded} + ) + + details = provider_settings.get_live_interface_details("http://127.0.0.1:18080", provider={"id": "abc"}) + + assert details["driver_requirement_fields"] == recorded + assert details["driver_requirement_field_map"] == {"aes_key": recorded[0]} + + +def test_inspect_contract_omits_requirements(monkeypatch, spec_document, fake_driver): + fake_get = fake_driver(spec_document) + monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + + details = provider_settings.inspect_contract("http://127.0.0.1:18080") + + assert details["name"] == "Meter Driver API" + assert details["driver_requirement_fields"] == [] + assert [interface["type"] for interface in details["interfaces"]] == ["http", "grpc"] + assert fake_get.calls == ["http://127.0.0.1:18080/openapi.json"] + + +def test_get_runtime_status_reports_gateway_when_connected(monkeypatch, fake_json_response): calls = [] def fake_get(url, timeout): calls.append(url) if url.endswith("/v1/healthz"): - return FakeResponse({"ok": True}) + return fake_json_response({"ok": True}) if url.endswith("/v1/status"): - return FakeResponse({"connected": True, "gateway_type": "sparknet"}) + return fake_json_response({"connected": True, "gateway_type": "sparknet"}) raise AssertionError("unexpected GET {}".format(url)) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) @@ -919,19 +1241,19 @@ def fake_get(url, timeout): assert calls == ["http://127.0.0.1:18080/v1/healthz", "http://127.0.0.1:18080/v1/status"] -def test_get_runtime_status_can_skip_gateway_probe(monkeypatch): - monkeypatch.setattr(provider_settings.httpx, "get", lambda url, timeout: FakeResponse({})) +def test_get_runtime_status_can_skip_gateway_probe(monkeypatch, fake_json_response): + monkeypatch.setattr(provider_settings.httpx, "get", lambda url, timeout: fake_json_response({"ok": True})) status = provider_settings.get_runtime_status("http://127.0.0.1:18080", include_gateway_status=False) assert status["online"] is True assert status["gateway_active"] is False assert status["gateway_checked"] is False -def test_get_runtime_status_tolerates_gateway_probe_failure(monkeypatch): +def test_get_runtime_status_tolerates_gateway_probe_failure(monkeypatch, fake_json_response): def fake_get(url, timeout): if url.endswith("/v1/status"): raise httpx.ConnectError("no status") - return FakeResponse({}) + return fake_json_response({"ok": True}) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) status = provider_settings.get_runtime_status("http://127.0.0.1:18080") @@ -940,7 +1262,54 @@ def fake_get(url, timeout): assert status["gateway_checked"] is True -def test_get_runtime_status_is_offline_when_healthz_fails_and_probes_nothing_else(monkeypatch): +@pytest.mark.parametrize("health", [{"ok": False}, {"ok": "true"}, {}, ["ok"], "ok", None]) +def test_get_runtime_status_is_offline_unless_healthz_answers_ok_true( + monkeypatch, fake_json_response, health +): + calls = [] + + def fake_get(url, timeout): + calls.append(url) + if url.endswith("/v1/healthz"): + return fake_json_response(health) + raise AssertionError("unexpected GET {}".format(url)) + + monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + status = provider_settings.get_runtime_status("http://127.0.0.1:18080") + assert status["online"] is False + assert '{"ok": true}' in status["message"] + assert status["gateway_active"] is False + assert calls == ["http://127.0.0.1:18080/v1/healthz"] + + +def test_get_runtime_status_is_offline_when_healthz_is_not_json(monkeypatch, fake_json_response): + class BadJSON(fake_json_response): + def json(self): + raise ValueError("bad") + + monkeypatch.setattr(provider_settings.httpx, "get", lambda url, timeout: BadJSON({})) + status = provider_settings.get_runtime_status("http://127.0.0.1:18080") + assert status["online"] is False + assert "not valid JSON" in status["message"] + + +def test_get_runtime_status_is_offline_when_status_is_not_an_object(monkeypatch, fake_json_response): + def fake_get(url, timeout): + if url.endswith("/v1/status"): + return fake_json_response(["connected"]) + return fake_json_response({"ok": True}) + + monkeypatch.setattr(provider_settings.httpx, "get", fake_get) + status = provider_settings.get_runtime_status("http://127.0.0.1:18080") + assert status["online"] is False + assert "/v1/status" in status["message"] + assert status["gateway_active"] is False + assert status["gateway_type"] is None + + +def test_get_runtime_status_is_offline_when_healthz_fails_and_probes_nothing_else( + monkeypatch, fake_json_response +): # /v1/healthz is the spec's liveness route; a failure means offline. No # legacy /health probe is attempted and /v1/status is not consulted. calls = [] @@ -949,7 +1318,7 @@ def fake_get(url, timeout): calls.append(url) if url.endswith("/v1/healthz"): raise httpx.ConnectError("no healthz") - return FakeResponse({"connected": True}) + return fake_json_response({"connected": True}) monkeypatch.setattr(provider_settings.httpx, "get", fake_get) status = provider_settings.get_runtime_status("http://127.0.0.1:18080") @@ -1115,16 +1484,34 @@ def test_required_field_helpers_reject_wrong_types(): with pytest.raises(provider_settings.DriverConfigError): provider_settings._field_values({"field_values": "nope"}) with pytest.raises(provider_settings.DriverConfigError): - provider_settings._required_field_specs({"required_fields": "nope"}) + provider_settings._stored_field_specs({"required_fields": "nope"}) -def test_required_field_names_accepts_dicts_and_strings(): - names = provider_settings._required_field_names( - {"required_fields": [{"name": "aes_key"}, "channel", {"name": ""}, " "]} - ) +def test_required_field_names_accepts_dicts_and_strings(caplog): + with caplog.at_level(logging.WARNING): + names = provider_settings._required_field_names( + {"required_fields": [{"name": "aes_key"}, "channel", {"name": ""}, " "]} + ) assert names == ["aes_key", "channel"] +def test_stored_field_specs_treats_bare_names_as_required_strings_and_says_so(caplog): + # A config written before field types were recorded (spec entries were + # bare names): usable as required strings, with a pointer to re-register. + with caplog.at_level(logging.WARNING): + specs = provider_settings._stored_field_specs( + {"required_fields": ["channel", {"name": "aes_key", "type": "string", "required": True}]} + ) + assert specs == { + "channel": {"name": "channel", "type": "string", "required": True}, + "aes_key": {"name": "aes_key", "type": "string", "required": True}, + } + assert any( + "channel" in record.getMessage() and "Re-register the driver" in record.getMessage() + for record in caplog.records + ) + + def test_required_field_names_skips_optional_extras(): # Vendor-option extras are written with required=False and are not demanded. names = provider_settings._required_field_names( @@ -1141,20 +1528,59 @@ def test_required_field_names_skips_optional_extras(): def test_coerce_field_value_by_type(): assert provider_settings._coerce_field_value("c", "26", {"type": "integer"}) == 26 + assert provider_settings._coerce_field_value("c", " 26 ", {"type": "integer"}) == 26 assert provider_settings._coerce_field_value("r", "1.5", {"type": "number"}) == 1.5 assert provider_settings._coerce_field_value("b", "yes", {"type": "boolean"}) is True assert provider_settings._coerce_field_value("b", "off", {"type": "boolean"}) is False assert provider_settings._coerce_field_value("b", True, {"type": "boolean"}) is True - assert provider_settings._coerce_field_value("s", "kept", {"type": "string"}) == "kept" + assert provider_settings._coerce_field_value("s", " kept ", {"type": "string"}) == "kept" + assert provider_settings._coerce_field_value("s", 7, {"type": "string"}) == "7" + assert provider_settings._coerce_field_value("s", "x", {"type": ["string", "null"]}) == "x" + assert provider_settings._coerce_field_value("l", [1], {"type": "array"}) == [1] + assert provider_settings._coerce_field_value("o", {"a": 1}, {"type": "object"}) == {"a": 1} def test_coerce_field_value_raises_on_bad_input(): - with pytest.raises(provider_settings.DriverConfigError): - provider_settings._coerce_field_value("c", "nope", {"type": "integer"}) - with pytest.raises(provider_settings.DriverConfigError): - provider_settings._coerce_field_value("r", "nope", {"type": "number"}) - with pytest.raises(provider_settings.DriverConfigError): - provider_settings._coerce_field_value("b", "maybe", {"type": "boolean"}) + for value, spec in ( + ("nope", {"type": "integer"}), + ("nope", {"type": "number"}), + ("maybe", {"type": "boolean"}), + ("x", {"type": "array"}), + ("x", {"type": "object"}), + ([1], {"type": "string"}), + ): + with pytest.raises(provider_settings.DriverConfigError): + provider_settings._coerce_field_value("f", value, spec) + + +def test_coerce_aes_key_accepts_the_two_spec_forms_only(): + hex_key = "00112233445566778899aabbccddeeff" + assert ( + provider_settings._coerce_field_value("aes_key", " {} ".format(hex_key), {"type": "string"}) + == hex_key + ) + assert provider_settings._coerce_field_value("aes_key", list(range(16)), {"type": "string"}) == list( + range(16) + ) + for bad in ("not-hex", hex_key[:-1], list(range(15)), list(range(17)), [256] + [0] * 15, [True] * 16, 7): + with pytest.raises(provider_settings.DriverConfigError, match="aes_key"): + provider_settings._coerce_field_value("aes_key", bad, {"type": "string"}) + + +def test_check_field_constraints_enforces_pattern_and_bounds(): + provider_settings._check_field_constraints("s", "abc", {"pattern": "^[a-z]+$"}) + provider_settings._check_field_constraints("c", 11, {"minimum": 11, "maximum": 26}) + provider_settings._check_field_constraints("c", 26, {"minimum": 11, "maximum": 26}) + with pytest.raises(provider_settings.DriverConfigError, match="pattern"): + provider_settings._check_field_constraints("s", "ABC", {"pattern": "^[a-z]+$"}) + with pytest.raises(provider_settings.DriverConfigError, match="at least"): + provider_settings._check_field_constraints("c", 10, {"minimum": 11}) + with pytest.raises(provider_settings.DriverConfigError, match="at most"): + provider_settings._check_field_constraints("c", 27, {"maximum": 26}) + # Booleans and non-numbers are not bounded; an unusable pattern is skipped. + provider_settings._check_field_constraints("b", True, {"minimum": 5}) + provider_settings._check_field_constraints("s", "x", {"minimum": 5}) + provider_settings._check_field_constraints("s", "x", {"pattern": "(["}) def test_validate_provider_config_payload_reports_missing_and_coerces(): @@ -1173,6 +1599,59 @@ def test_validate_provider_config_payload_reports_missing_and_coerces(): assert validated["field_values"]["channel"] == 26 +def test_validate_provider_config_payload_treats_whitespace_as_missing(): + with pytest.raises(provider_settings.DriverConfigError, match="channel"): + provider_settings.validate_provider_config_payload( + {"required_fields": [{"name": "channel", "type": "integer"}], "field_values": {"channel": " "}} + ) + + +def test_validate_provider_config_payload_enforces_recorded_constraints(): + payload = { + "required_fields": [ + {"name": "channel", "type": "integer", "minimum": 11, "maximum": 26}, + {"name": "region", "type": "string", "pattern": "^[a-z]{2}$"}, + ], + "field_values": {"channel": "27", "region": "eu"}, + } + with pytest.raises(provider_settings.DriverConfigError, match="'channel' must be at most 26"): + provider_settings.validate_provider_config_payload(payload) + payload["field_values"] = {"channel": "26", "region": "EUR"} + with pytest.raises(provider_settings.DriverConfigError, match="'region' must match"): + provider_settings.validate_provider_config_payload(payload) + payload["field_values"] = {"channel": "26", "region": "eu"} + assert provider_settings.validate_provider_config_payload(payload)["field_values"] == { + "channel": 26, + "region": "eu", + } + + +def test_validate_provider_config_payload_checks_aes_key_forms(): + payload = { + "required_fields": [{"name": "aes_key", "type": "string", "pattern": "^[A-Fa-f0-9]{32}$"}], + "field_values": {"aes_key": "not-hex"}, + } + with pytest.raises(provider_settings.DriverConfigError, match="32 hex characters"): + provider_settings.validate_provider_config_payload(payload) + payload["field_values"] = {"aes_key": list(range(16))} + assert provider_settings.validate_provider_config_payload(payload)["field_values"] == { + "aes_key": list(range(16)) + } + + +def test_validate_provider_config_payload_drops_keys_the_driver_did_not_ask_for(caplog): + with caplog.at_level(logging.WARNING): + validated = provider_settings.validate_provider_config_payload( + { + "required_fields": [{"name": "channel", "type": "integer", "required": True}], + "field_values": {"channel": "26", "leftover": "x", "aes_key": "00" * 16}, + } + ) + # A hand-edited config's extra keys are not posted to the driver. + assert validated["field_values"] == {"channel": 26} + assert any("leftover" in record.getMessage() for record in caplog.records) + + def test_validate_provider_config_payload_omits_blank_optional_fields(): validated = provider_settings.validate_provider_config_payload( { @@ -1347,6 +1826,47 @@ def fake_init(provider, payload, timeout=10.0): assert results["ok"]["success"] is True +# --------------------------------------------------------------------------- +# gRPC selection checks +# --------------------------------------------------------------------------- + + +def _grpc_details(target="h:50051", required=("heartbeat_period_duration", "aes_key")): + interfaces = [{"type": "http", "address": "http://x"}] + if target is not None: + interfaces.append({"type": "grpc", "target": target, "address": target}) + return { + "interfaces": interfaces, + "driver_requirement_fields": [{"name": name, "required": True} for name in required] + + [{"name": "channel", "required": False}], + } + + +def test_check_grpc_selection_accepts_an_advertised_target_with_the_fixed_init_fields(): + provider_settings.check_grpc_selection(_grpc_details()) + + +def test_check_grpc_selection_rejects_a_missing_grpc_interface(): + with pytest.raises(provider_settings.ProviderRegistrationError, match="advertises no grpc interface"): + provider_settings.check_grpc_selection(_grpc_details(target=None)) + + +def test_check_grpc_selection_rejects_a_grpc_interface_without_target(): + with pytest.raises(provider_settings.ProviderRegistrationError, match="advertises no target"): + provider_settings.check_grpc_selection(_grpc_details(target="")) + + +def test_check_grpc_selection_rejects_requirements_lacking_the_fixed_init_fields(): + with pytest.raises(provider_settings.ProviderRegistrationError) as exc: + provider_settings.check_grpc_selection(_grpc_details(required=("aes_key",))) + assert "does not list: heartbeat_period_duration" in str(exc.value) + # An optional extra does not count: ConfigureDriver needs the value. + details = _grpc_details(required=("heartbeat_period_duration",)) + details["driver_requirement_fields"].append({"name": "aes_key", "required": False}) + with pytest.raises(provider_settings.ProviderRegistrationError, match="does not list: aes_key"): + provider_settings.check_grpc_selection(details) + + # --------------------------------------------------------------------------- # Persistence (DB-backed) # --------------------------------------------------------------------------- @@ -1396,9 +1916,9 @@ def test_get_saved_providers_handles_blank_invalid_and_non_dict(session): assert saved[0]["enabled"] is True -def test_save_and_lookup_providers_roundtrip(session, monkeypatch, tmp_path): +def test_save_and_lookup_providers_roundtrip(session, monkeypatch, tmp_path, fake_driver): _use_temp_config_root(monkeypatch, tmp_path) - monkeypatch.setattr(provider_settings.httpx, "get", _fake_openapi_get) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document())) provider_id = provider_settings.save_provider_settings("http://127.0.0.1:18080", "http") session.flush() @@ -1408,9 +1928,9 @@ def test_save_and_lookup_providers_roundtrip(session, monkeypatch, tmp_path): assert provider_settings.get_enabled_provider()["id"] == provider_id -def test_save_provider_settings_replaces_existing_by_id(session, monkeypatch, tmp_path): +def test_save_provider_settings_replaces_existing_by_id(session, monkeypatch, tmp_path, fake_driver): _use_temp_config_root(monkeypatch, tmp_path) - monkeypatch.setattr(provider_settings.httpx, "get", _fake_openapi_get) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document())) provider_id = provider_settings.save_provider_settings("http://127.0.0.1:18080", "http") session.flush() @@ -1423,9 +1943,9 @@ def test_save_provider_settings_replaces_existing_by_id(session, monkeypatch, tm assert providers[0]["base_url"] == "http://127.0.0.1:28080" -def test_save_provider_settings_preserves_other_providers(session, monkeypatch, tmp_path): +def test_save_provider_settings_preserves_other_providers(session, monkeypatch, tmp_path, fake_driver): _use_temp_config_root(monkeypatch, tmp_path) - monkeypatch.setattr(provider_settings.httpx, "get", _fake_openapi_get) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document())) first_id = provider_settings.save_provider_settings("http://127.0.0.1:18080", "http") session.flush() @@ -1442,11 +1962,70 @@ def test_save_provider_settings_preserves_other_providers(session, monkeypatch, assert providers[second_id]["base_url"] == "http://127.0.0.1:28080" -def test_save_provider_settings_falls_back_for_invalid_interface(session, monkeypatch, tmp_path): +def test_save_provider_settings_falls_back_for_unknown_non_grpc_interface( + session, monkeypatch, tmp_path, fake_driver +): _use_temp_config_root(monkeypatch, tmp_path) - monkeypatch.setattr(provider_settings.httpx, "get", _fake_openapi_get) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document())) - # "grpc" is not advertised by _fake_openapi_get, so it falls back to http. - provider_id = provider_settings.save_provider_settings("http://127.0.0.1:18080", "grpc") + # "mqtt" is not advertised, so the default interface (http) is saved. + provider_id = provider_settings.save_provider_settings("http://127.0.0.1:18080", "mqtt") session.flush() assert provider_settings.get_provider(provider_id)["selected_interface"] == "http" + + +def test_save_provider_settings_refuses_grpc_the_driver_does_not_advertise( + session, monkeypatch, tmp_path, fake_driver +): + _use_temp_config_root(monkeypatch, tmp_path) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(_spec_document())) + + with pytest.raises(provider_settings.ProviderRegistrationError, match="advertises no grpc interface"): + provider_settings.save_provider_settings("http://127.0.0.1:18080", "grpc") + assert provider_settings.get_saved_providers() == [] + + +def test_save_provider_settings_refuses_grpc_without_a_target(session, monkeypatch, tmp_path, fake_driver): + _use_temp_config_root(monkeypatch, tmp_path) + document = _spec_document( + **{ + "x-meter-driver": { + "default_interface": "http", + "interfaces": [{"type": "http", "base_url": "http://127.0.0.1:18080"}, {"type": "grpc"}], + } + } + ) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(document)) + + with pytest.raises(provider_settings.ProviderRegistrationError, match="advertises no target"): + provider_settings.save_provider_settings("http://127.0.0.1:18080", "grpc") + + +def test_save_provider_settings_refuses_grpc_when_requirements_lack_its_fields( + session, monkeypatch, tmp_path, spec_document, fake_driver +): + _use_temp_config_root(monkeypatch, tmp_path) + # The spec document advertises grpc at 127.0.0.1:50051, but this driver's + # /v1/requirements has no aes_key: ConfigureDriver could never be built. + monkeypatch.setattr( + provider_settings.httpx, + "get", + fake_driver(spec_document, required_fields=("heartbeat_period_duration", "site_token")), + ) + + with pytest.raises(provider_settings.ProviderRegistrationError, match="does not list: aes_key"): + provider_settings.save_provider_settings("http://127.0.0.1:18080", "grpc") + + +def test_save_provider_settings_records_the_advertised_grpc_target( + session, monkeypatch, tmp_path, spec_document, fake_driver +): + _use_temp_config_root(monkeypatch, tmp_path) + monkeypatch.setattr(provider_settings.httpx, "get", fake_driver(spec_document)) + + provider_id = provider_settings.save_provider_settings("http://127.0.0.1:18080", "grpc") + session.flush() + + provider = provider_settings.get_provider(provider_id) + assert provider["selected_interface"] == "grpc" + assert provider["selected_interface_target"] == "127.0.0.1:50051" diff --git a/sparkmeter/conftest.py b/sparkmeter/conftest.py index d23578a..b80cf54 100644 --- a/sparkmeter/conftest.py +++ b/sparkmeter/conftest.py @@ -9,9 +9,11 @@ cross-test pollution and tests can run in parallel with pytest-xdist. """ +import json import os import uuid from contextlib import contextmanager +from pathlib import Path import pytest from sqlalchemy import create_engine, text @@ -26,6 +28,74 @@ TEMPLATE_DB_NAME = "test_template" +# The Meter Driver Specification's own openapi/meter-driver.yaml, converted +# to JSON: exactly what a driver serving nothing but the spec answers on +# GET /openapi.json. Regenerate it after bumping the meter-driver-spec wheel: +# +# uv run python -c 'import json, sys, yaml; \ +# json.dump(yaml.safe_load(open(sys.argv[1])), open(sys.argv[2], "w"), indent=2); \ +# open(sys.argv[2], "a").write("\n")' \ +# /openapi/meter-driver.yaml sparkmeter/config/tests/meter_driver_spec_openapi.json +# +# test_provider_settings.py checks its info.version against the installed +# wheel's version, so a bump cannot leave the fixture stale. +SPEC_OPENAPI_DOCUMENT_PATH = Path(__file__).parent / "config" / "tests" / "meter_driver_spec_openapi.json" + + +class FakeJsonResponse(object): + """Minimal httpx-like response double: `raise_for_status()` passes, `json()` returns the payload.""" + + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + """Pretend the response was successful.""" + + def json(self): + """Return the configured JSON payload.""" + return self._payload + + +@pytest.fixture +def fake_json_response(): + """The FakeJsonResponse class, for httpx.get doubles.""" + return FakeJsonResponse + + +@pytest.fixture +def spec_document(): + """A fresh copy of the spec's OpenAPI document (see SPEC_OPENAPI_DOCUMENT_PATH).""" + return json.loads(SPEC_OPENAPI_DOCUMENT_PATH.read_text()) + + +@pytest.fixture +def fake_driver(): + """Factory for httpx.get doubles serving a driver's /openapi.json and /v1/requirements. + + `fake_driver(document, required_fields=..., requirements_error=...)` + returns a callable with the `httpx.get(url, timeout)` signature. Every + URL it answers is recorded on its `.calls`; any other URL raises + AssertionError, so a stray probe fails the test immediately. + """ + + def _factory(document, required_fields=("heartbeat_period_duration", "aes_key"), requirements_error=None): + calls = [] + + def fake_get(url, timeout): + calls.append(url) + if url.endswith("/v1/requirements"): + if requirements_error is not None: + raise requirements_error + return FakeJsonResponse({"required_fields": list(required_fields)}) + if url.endswith("/openapi.json"): + return FakeJsonResponse(document) + raise AssertionError("unexpected GET {}".format(url)) + + fake_get.calls = calls + return fake_get + + return _factory + # --------------------------------------------------------------------------- # Session-scoped: bootstrap the app and create the template database diff --git a/uv.lock b/uv.lock index 80b9f82..ccde076 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.13" resolution-markers = [ "python_full_version >= '3.14'", @@ -1722,6 +1722,7 @@ dependencies = [ { name = "phonenumbers" }, { name = "protobuf" }, { name = "psycopg2-binary" }, + { name = "pydantic" }, { name = "python-dateutil" }, { name = "python-dotenv" }, { name = "requests" }, @@ -1786,6 +1787,7 @@ requires-dist = [ { name = "phonenumbers" }, { name = "protobuf" }, { name = "psycopg2-binary" }, + { name = "pydantic" }, { name = "python-dateutil" }, { name = "python-dotenv" }, { name = "requests" }, From 2c8cc531df884a020ffb7ef199105151a10800ef Mon Sep 17 00:00:00 2001 From: Tristan Escalada <355457+tescalada@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:47:11 -0400 Subject: [PATCH 09/17] Refuse a gRPC selection the contract cannot honor at registration time check_grpc_selection requires an advertised grpc interface with a target and, because the gRPC ConfigureDriver message has fixed fields, heartbeat_period_duration and aes_key among the driver's required /v1/requirements fields. save_provider_settings applies it to a gRPC selection instead of silently falling back to http, and the settings form's validate_selected_interface reports the same message. The driver-field helpers on the forms are named for what they list (driver_field_spec, driver_fields), both templates label the list "Driver fields" and mark each entry required or optional, and the views pass the saved provider so its recorded fields are shown without a /v1/requirements round trip. --- sparkmeter/config/configviews.py | 2 + sparkmeter/config/providerform.py | 45 +++++------- .../config-meter-driver-config-editor.html | 11 ++- .../templates/config-meter-driver-form.html | 6 +- sparkmeter/config/tests/test_providerform.py | 73 ++++++++++++++++--- 5 files changed, 96 insertions(+), 41 deletions(-) diff --git a/sparkmeter/config/configviews.py b/sparkmeter/config/configviews.py index 13f3ecf..62dad16 100644 --- a/sparkmeter/config/configviews.py +++ b/sparkmeter/config/configviews.py @@ -144,6 +144,7 @@ def meter_driver_edit(provider_id): provider_details = get_live_interface_details( provider["base_url"], selected_interface=provider["selected_interface"], + provider=provider, ) form = MeterDriverSettingsForm( @@ -171,6 +172,7 @@ def meter_driver_config(provider_id): provider_details = get_live_interface_details( provider["base_url"], selected_interface=provider["selected_interface"], + provider=provider, ) form = MeterDriverConfigEditorForm( formdata=request.form if request.method == "POST" else None, diff --git a/sparkmeter/config/providerform.py b/sparkmeter/config/providerform.py index 55dedad..d624805 100644 --- a/sparkmeter/config/providerform.py +++ b/sparkmeter/config/providerform.py @@ -43,38 +43,25 @@ def __init__(self, *args, **kwargs): selected_interface=( self.provider["selected_interface"] if self.provider is not None else None ), + provider=self.provider, ) if self.provider_details is not None: self._set_interface_choices(self.provider_details) - self._apply_vendor_option_labels() + self._apply_driver_field_labels() if not self.selected_interface.data: self.selected_interface.data = ( self.provider["selected_interface"] if self.provider is not None else None ) or self._default_selected_interface() - def vendor_option_spec(self, name): - """Return the normalized driver-requirement spec for a known field.""" + def driver_field_spec(self, name): + """Return the normalized spec of one field the driver asks for, if it does.""" provider_data = self.provider_details or {} return (provider_data.get("driver_requirement_field_map") or {}).get(name) - def supports_vendor_option(self, name): - """Whether the validated provider advertises the given vendor option.""" - return self.vendor_option_spec(name) is not None - - def vendor_option_description(self, name): - """Return help text for a vendor option field.""" - spec = self.vendor_option_spec(name) or {} - return spec.get("description") or "" - - def vendor_option_required(self, name): - """Whether the vendor option is required by the contract.""" - spec = self.vendor_option_spec(name) or {} - return bool(spec.get("required")) - - def vendor_option_fields(self): - """Return the contract-advertised driver requirement field list.""" + def driver_fields(self): + """Return the fields the driver asks for: /v1/requirements first, optional extras after.""" provider_data = self.provider_details or {} return provider_data.get("driver_requirement_fields") or [] @@ -114,13 +101,13 @@ def _default_selected_interface(self): return self.selected_interface.choices[0][0] return "http" - def _apply_vendor_option_labels(self): - """Update known vendor-option labels from the validated contract.""" - aes_key_spec = self.vendor_option_spec("aes_key") + def _apply_driver_field_labels(self): + """Update the known field labels from the validated contract.""" + aes_key_spec = self.driver_field_spec("aes_key") if aes_key_spec and aes_key_spec.get("label"): self.aes_key.label.text = aes_key_spec["label"] - channel_spec = self.vendor_option_spec("channel") + channel_spec = self.driver_field_spec("channel") if channel_spec and channel_spec.get("label"): self.channel.label.text = channel_spec["label"] @@ -133,7 +120,7 @@ def validate_service_url(self, field): try: self.provider_details = provider_settings.validate_contract(field.data) self._set_interface_choices(self.provider_details) - self._apply_vendor_option_labels() + self._apply_driver_field_labels() except provider_settings.ProviderRegistrationError as exc: raise ValidationError(str(exc)) @@ -149,6 +136,12 @@ def validate_selected_interface(self, field): if field.data not in valid_interfaces: raise ValidationError(_("Selected interface is not available from this driver.")) + if field.data == "grpc": + try: + provider_settings.check_grpc_selection(provider_data) + except provider_settings.ProviderRegistrationError as exc: + raise ValidationError(str(exc)) + def validate_aes_key(self, field): field.data = "" @@ -208,8 +201,8 @@ def __init__(self, *args, **kwargs): if not self.config_text.data: self.config_text.data = provider_settings.load_provider_config_text(self.provider) - def required_fields(self): - """Return the driver-required field list.""" + def driver_fields(self): + """Return the fields the driver asks for: /v1/requirements first, optional extras after.""" return (self.provider_details or {}).get("driver_requirement_fields") or [] def config_file_path(self): diff --git a/sparkmeter/config/templates/config-meter-driver-config-editor.html b/sparkmeter/config/templates/config-meter-driver-config-editor.html index dd6d397..9b80c70 100644 --- a/sparkmeter/config/templates/config-meter-driver-config-editor.html +++ b/sparkmeter/config/templates/config-meter-driver-config-editor.html @@ -23,12 +23,17 @@

{{ _('Driver') }}: {{ provider_details.name if provider_details and provider_details.name else provider.name }}

{{ _('File') }}: {{ form.config_file_path() }}

- {% if form.required_fields() %} -

{{ _('Required fields') }}:

+ {% if form.driver_fields() %} +

{{ _('Driver fields') }}: