diff --git a/.github/workflows/ci-simulators.yml b/.github/workflows/ci-simulators.yml index cecc2537..aefdf03e 100644 --- a/.github/workflows/ci-simulators.yml +++ b/.github/workflows/ci-simulators.yml @@ -89,5 +89,8 @@ jobs: - name: Audit Dependencies run: uv audit + - name: Run Tests + run: uv run pytest + - name: Smoke test run: uv run python smoke.py diff --git a/CLAUDE.md b/CLAUDE.md index f0072455..d977ede0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,6 +68,7 @@ just test all just test # chat telemetry twin notification socket dashboard frontend # agent ap-collector claims-gateway provisioner registry tenancy # aq-simulator sensor-simulator (assert the ingest-batch fixture) + # ap-simulator (asserts the simulation-start fixture) just test -integration # throwaway DB/broker, composed, then torn down # Go (registry, tenancy): testcontainers behind `-tags=integration` just test integration # full backend acceptance suite @@ -155,8 +156,8 @@ real port/adapter split only on the outbound side. **Routing** (`Caddyfile`, mirrored by `k8s/istio-*.yml`): `/gateway`, `/tenancy`, `/twin`, `/telemetry`, `/notification`, `/chat` (SSE, needs `flush_interval -1`), `/dashboard`, -`/socket.io`, `/agent` (ungated), `/` → frontend. `/telemetry/ingest` is ungated at the edge -and HMAC-verified in-service — it is one exact path, so a sub-path would 401. +`/socket.io`, `/agent` (ungated), `/` → frontend. `/telemetry/ingest` and `/telemetry/collector` +are ungated at the edge and HMAC-verified in-service — exact paths, so a sub-path would 401. `registry` and `provisioner` have no external route. ## Golden rules diff --git a/Caddyfile b/Caddyfile index bd583784..584c2457 100644 --- a/Caddyfile +++ b/Caddyfile @@ -97,6 +97,12 @@ uri strip_prefix /telemetry reverse_proxy telemetry:3000 } + # A collector reads its routers here with the same key, signing a timestamp instead of + # a body. Exact path, and listed in the same Istio no-principal rule, for the same reason. + handle /telemetry/collector { + uri strip_prefix /telemetry + reverse_proxy telemetry:3000 + } handle_path /telemetry/* { import require_gateway_auth reverse_proxy telemetry:3000 diff --git a/api/telemetry.yaml b/api/telemetry.yaml index af13a825..90ce3fd8 100644 --- a/api/telemetry.yaml +++ b/api/telemetry.yaml @@ -28,10 +28,26 @@ components: in: header name: x-signature description: > - Lowercase-hex HMAC-SHA256 over the raw request body, keyed by - TELEMETRY_INGEST_SECRET. A sensor gateway has no user JWT, so this is + Lowercase-hex HMAC-SHA256 over the raw request body, keyed by the + device key of the body's buildingId (or, in dev only, the shared + TELEMETRY_INGEST_SECRET). A sensor gateway has no user JWT, so this is how it authenticates itself. The signature covers the exact bytes on the wire — sign the string you send, never a re-serialised copy. + collectorSignature: + type: apiKey + in: header + name: x-signature + description: > + Lowercase-hex HMAC-SHA256, keyed by the buildingId's device key, over + "GET /collector\n{buildingId}\n{x-timestamp}". A GET has no body, so + the signed building and timestamp are what prove the caller holds the + key; a timestamp more than 300 s from telemetry's clock is refused. + Without buildingId only the dev shared key signs. + collectorTimestamp: + type: apiKey + in: header + name: x-timestamp + description: Unix seconds, decimal — the value the x-signature covers. parameters: SensorType: @@ -270,6 +286,90 @@ security: - gatewayClaims: [] paths: + /device-keys/buildings/{buildingId}: + post: + summary: Issue a building's device key + description: > + Revokes the building's current device key and returns its new one. This + is the only time the key is shown: put it in the collector's `keys` and + store it nowhere else. Keys are derived from the service's master key, + never stored; other buildings' keys are untouched. + tags: [Ingestion] + security: + - gatewayClaims: [] + parameters: + - name: buildingId + in: path + required: true + schema: { type: string } + responses: + "200": + description: The new key; the previous one no longer signs + content: + application/json: + schema: + type: object + required: [buildingId, key] + properties: + buildingId: { type: string, example: "bldg-3f2b4c5d" } + key: { type: string, description: 64 lowercase hex characters } + "403": + description: The caller may not edit this building + "404": + description: The building is not registered + + /collector: + get: + summary: Routers an on-site collector polls + description: > + Every router sensor placed in a room, grouped by building: what an + ap-collector polls and which room each one counts into. Outdoor routers + and buildings with no router are left out. + + Addresses only, never logins: `endpoint` is the router's ubus URL on the + site network, and the credentials to use it stay in the collector's own + configuration. Ungated at the edge like `/ingest`, and signed by the + caller instead. + tags: [Ingestion] + security: + - collectorSignature: [] + collectorTimestamp: [] + parameters: + - name: buildingId + in: query + required: false + description: The building whose routers to read; the signing key must be its device key. + schema: { type: string, example: "bldg-3f2b4c5d" } + responses: + "200": + description: The routers, by building + content: + application/json: + schema: + type: object + required: [buildings] + properties: + buildings: + type: array + items: + type: object + required: [buildingId, routers] + properties: + buildingId: { type: string, example: "bldg-3f2b4c5d" } + routers: + type: array + minItems: 1 + items: + type: object + required: [sensorId, roomId] + properties: + sensorId: { type: string, format: uuid } + roomId: { type: string, example: "room-lab-2" } + driver: { type: string, example: "openwrt-hostapd" } + endpoint: { type: string, example: "http://10.0.4.12/ubus" } + "401": + description: Unsigned, mis-signed, or stamped outside the 300 s window + /ingest: post: summary: Ingest one building tick @@ -590,6 +690,44 @@ paths: "404": description: Unknown sensor type, or the room has no readings + /connected-devices/buildings/{buildingId}: + get: + summary: Every device connected in a building + description: > + The sum of the building's rooms' totalDeviceCount at its newest report. + A router counts the devices it hears wherever they stand, so this + building figure is the one that means what it says. A room whose last + report is older than the newest is left out (its router was removed or + moved), and each room counts once, so a report delivered twice does not + double the total. + tags: [Reads] + security: + - gatewayClaims: [] + parameters: + - name: buildingId + in: path + required: true + schema: { type: string } + responses: + "200": + description: The building's connected devices at its newest report + content: + application/json: + schema: + type: object + required: [buildingId, totalDeviceCount, timestamp] + properties: + buildingId: { type: string, example: "bldg-3f2b4c5d" } + totalDeviceCount: { type: integer, minimum: 0, example: 47 } + timestamp: + type: integer + description: Epoch milliseconds of the newest report summed + example: 1757251200000 + "403": + description: The caller may not read this building + "404": + description: The building never reported a device count + /{sensorType}/entireBuilding: get: summary: Latest reading for every room in a building diff --git a/backend/acceptance/docker-compose.integration.yml b/backend/acceptance/docker-compose.integration.yml index dbf4f6f8..376031ca 100644 --- a/backend/acceptance/docker-compose.integration.yml +++ b/backend/acceptance/docker-compose.integration.yml @@ -60,6 +60,7 @@ services: - DASHBOARD_URL=http://dashboard:3000 - EDGE_URL=http://gateway - TELEMETRY_INGEST_SECRET=${TELEMETRY_INGEST_SECRET} + - TELEMETRY_DEVICE_MASTER_KEY=${TELEMETRY_DEVICE_MASTER_KEY} volumes: # :z relabels the bind mount for SELinux-enforcing hosts (same reason # the root compose file's gateway service does it for its own mounts) diff --git a/backend/acceptance/edge/test_edge_routing.py b/backend/acceptance/edge/test_edge_routing.py index a74b31ce..32cab12d 100644 --- a/backend/acceptance/edge/test_edge_routing.py +++ b/backend/acceptance/edge/test_edge_routing.py @@ -94,3 +94,20 @@ def test_the_ingest_ungate_is_an_exact_path_not_a_prefix(): assert response.status_code == 401 assert EDGE_REJECTION_MARKER in response.text + + +def test_the_collector_read_crosses_the_edge_ungated(): + """A collector carries no user JWT either. /telemetry/collector is the second exact + path both edges let through; telemetry verifies its signed timestamp itself. Unsigned, + it must be refused by telemetry, not by the edge. + """ + with httpx.Client(timeout=10.0) as client: + signed = telemetry.read_collector(client, f"{TELEMETRY_VIA_EDGE}/collector") + unsigned = telemetry.read_collector( + client, f"{TELEMETRY_VIA_EDGE}/collector", signed=False + ) + + assert signed.status_code == 200 + assert "buildings" in signed.json() + assert unsigned.status_code == 401 + assert EDGE_REJECTION_MARKER not in unsigned.text diff --git a/backend/acceptance/run-integration-tests.sh b/backend/acceptance/run-integration-tests.sh index fdef2778..d532b420 100755 --- a/backend/acceptance/run-integration-tests.sh +++ b/backend/acceptance/run-integration-tests.sh @@ -13,11 +13,11 @@ export COMPOSE_BAKE=true cd "$(dirname "${BASH_SOURCE[0]}")/../.." # repo root -# telemetry refuses to boot without an ingest signing key, and both it -# and the test client read the same one out of the repo-root .env that compose -# auto-loads. Generating it here keeps the suite runnable without `just stack env`. +# telemetry refuses to boot without its device master key, and both it and the +# test client read the shared ingest key out of the repo-root .env that compose +# auto-loads. Generating them here keeps the suite runnable without `just stack env`. mise exec -- sh scripts/env/ingest-key.sh || { - echo "::error::could not generate TELEMETRY_INGEST_SECRET" >&2 + echo "::error::could not generate TELEMETRY_INGEST_SECRET / TELEMETRY_DEVICE_MASTER_KEY" >&2 exit 1 } diff --git a/backend/acceptance/support/telemetry.py b/backend/acceptance/support/telemetry.py index 81018040..a7e1a63a 100644 --- a/backend/acceptance/support/telemetry.py +++ b/backend/acceptance/support/telemetry.py @@ -104,3 +104,14 @@ def latest(client: httpx.Client, metric: str, building_id: str, room_id: str) -> def latest_temperature(client: httpx.Client, building_id: str, room_id: str) -> dict: return latest(client, "temperature", building_id, room_id) + + +def read_collector(client: httpx.Client, url: str, *, signed: bool = True) -> httpx.Response: + """GET the collector's router list, signed as ap-collector signs it.""" + if not signed: + return client.get(url) + stamp = str(int(time.time())) + signature = hmac.new( + config.TELEMETRY_INGEST_SECRET.encode(), f"GET /collector\n\n{stamp}".encode(), hashlib.sha256 + ).hexdigest() + return client.get(url, headers={"X-Signature": signature, "X-Timestamp": stamp}) diff --git a/backend/acceptance/uv.lock b/backend/acceptance/uv.lock index 993b7ce2..0f51dcf5 100644 --- a/backend/acceptance/uv.lock +++ b/backend/acceptance/uv.lock @@ -287,11 +287,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] diff --git a/backend/agent/pyproject.toml b/backend/agent/pyproject.toml index 50ed7db6..65ee171d 100644 --- a/backend/agent/pyproject.toml +++ b/backend/agent/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "psycopg[binary]>=3.2", "sqlalchemy[asyncio]>=2.0.35", "alembic>=1.14.0", - "pyjwt[crypto]>=2.9.0", + "pyjwt[crypto]>=2.15.0", "structlog>=24.4.0", "opentelemetry-api>=1.28.0", "opentelemetry-sdk>=1.28.0", @@ -45,7 +45,7 @@ constraint-dependencies = [ "mako>=1.3.12", "python-multipart>=0.0.31", "starlette>=1.3.1", - "urllib3>=2.7.0", + "urllib3>=2.8.0", "aiohttp>=3.14.3", "cryptography>=50.0.0", "joserfc>=1.6.8", diff --git a/backend/agent/uv.lock b/backend/agent/uv.lock index d2758dc5..01d2e9f4 100644 --- a/backend/agent/uv.lock +++ b/backend/agent/uv.lock @@ -16,7 +16,7 @@ constraints = [ { name = "mako", specifier = ">=1.3.12" }, { name = "python-multipart", specifier = ">=0.0.31" }, { name = "starlette", specifier = ">=1.3.1" }, - { name = "urllib3", specifier = ">=2.7.0" }, + { name = "urllib3", specifier = ">=2.8.0" }, ] [[package]] @@ -77,7 +77,7 @@ requires-dist = [ { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.9.0" }, { name = "pydantic-settings", specifier = ">=2.14.2" }, - { name = "pyjwt", extras = ["crypto"], specifier = ">=2.9.0" }, + { name = "pyjwt", extras = ["crypto"], specifier = ">=2.15.0" }, { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.35" }, { name = "structlog", specifier = ">=24.4.0" }, { name = "tiktoken", specifier = ">=0.8.0" }, @@ -1391,11 +1391,11 @@ wheels = [ [[package]] name = "pyjwt" -version = "2.13.0" +version = "2.15.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, ] [package.optional-dependencies] @@ -1957,11 +1957,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] diff --git a/backend/telemetry/CLAUDE.md b/backend/telemetry/CLAUDE.md index 47477a08..9178d515 100644 --- a/backend/telemetry/CLAUDE.md +++ b/backend/telemetry/CLAUDE.md @@ -2,7 +2,7 @@ Rust / Axum / Postgres+Timescale / Kafka / Redis. Ingests sensor readings, owns thresholds, sensors and device actions, fans out to dashboard and raises alerts. Routes `/telemetry/*` -gated at the edge; `/telemetry/ingest` ungated and HMAC-verified in-service. +gated at the edge; `/telemetry/ingest` and `/telemetry/collector` ungated and HMAC-verified in-service. Docs: `documentation/architecture/telemetry-architecture.qd`, `design/telemetry-storage.qd`. @@ -85,10 +85,11 @@ unreachable. **Simulators are told what to simulate at start, and only then.** `PUT /simulation/buildings/{id}` reads the building's sensors and sends each simulator in `SIMULATORS` the ones whose kind it claims (body: `schemas/fixtures/simulation-start.json`). Every simulator is told, even with an -empty list — empty means stop, so a removed sensor stops being simulated. Routers (no simulator -claims them) and outdoor sensors are never sent. Calls run concurrently with a 5 s timeout, all +empty list — empty means stop, so a removed sensor stops being simulated. Kinds no simulator +claims, and outdoor sensors, are never sent. Calls run concurrently with a 5 s timeout, all are tried, then any failure is `502`. `Simulation::new` refuses a kind that is not a device, or -one claimed twice (doubled readings). No simulator configured → `404`. +one claimed twice (doubled readings). No simulator configured → `404`. The body never +carries coordinates: positions are digital-twin's, not telemetry's. **Registration**: telemetry consumes `building-registration-requested` and answers `building-registration-completed` (both from `twin_schema`). `maxTemperature` is read here @@ -104,3 +105,12 @@ just test telemetry-integration # tests/*.rs against a throwaway TimescaleDB, `tests/` covers `api`, `persistence`, `fanout`, `alerts`, `registration`, `architecture`. Migrations live in `migrations/`. + +**Device keys are derived, never stored.** A building's key is `HMAC(TELEMETRY_DEVICE_MASTER_KEY, +"{buildingId}:{epoch}")`; `buildings.device_key_epoch` is the only state, bumped by +`POST /device-keys/buildings/{id}` to revoke. Read uncached, so a rotation holds on every replica. +`TELEMETRY_INGEST_SECRET` is the optional shared key that signs for any building — dev only. + +**A building's connected devices are summed here, never by a client.** `Readings::building_total` +sums a metric over the rooms at the building's newest report, each room once +(`GET /connected-devices/buildings/{id}`). Only a count adds up: never route an average through it. diff --git a/backend/telemetry/docker-compose.yml b/backend/telemetry/docker-compose.yml index 96cd3bb9..05918af8 100644 --- a/backend/telemetry/docker-compose.yml +++ b/backend/telemetry/docker-compose.yml @@ -13,9 +13,10 @@ services: - REDIS_URL=redis://redis:6379 - KAFKA_BROKERS=kafka:9092 - DIGITAL_TWIN_URL=http://digital-twin:3000 - # Which simulator simulates which device kind; kinds no simulator claims (router) are never sent. - - 'SIMULATORS={"sensor-simulator":{"url":"http://sensor-simulator:3000","kinds":["temperature","peopleCount"]},"aq-simulator":{"url":"http://aq-simulator:3000","kinds":["airQuality"]}}' + # Which simulator simulates which device kind; a kind no simulator claims is never sent. + - 'SIMULATORS={"sensor-simulator":{"url":"http://sensor-simulator:3000","kinds":["temperature","peopleCount"]},"aq-simulator":{"url":"http://aq-simulator:3000","kinds":["airQuality"]},"ap-simulator":{"url":"http://ap-simulator:3000","kinds":["router"]}}' - TELEMETRY_INGEST_SECRET=${TELEMETRY_INGEST_SECRET} + - TELEMETRY_DEVICE_MASTER_KEY=${TELEMETRY_DEVICE_MASTER_KEY} - PORT=3000 - RUST_LOG=info healthcheck: diff --git a/backend/telemetry/migrations/0006_device_key_epoch.down.sql b/backend/telemetry/migrations/0006_device_key_epoch.down.sql new file mode 100644 index 00000000..d203b628 --- /dev/null +++ b/backend/telemetry/migrations/0006_device_key_epoch.down.sql @@ -0,0 +1 @@ +alter table buildings drop column device_key_epoch; diff --git a/backend/telemetry/migrations/0006_device_key_epoch.up.sql b/backend/telemetry/migrations/0006_device_key_epoch.up.sql new file mode 100644 index 00000000..08f0bb57 --- /dev/null +++ b/backend/telemetry/migrations/0006_device_key_epoch.up.sql @@ -0,0 +1,2 @@ +-- A building's device key is derived from the master key and this epoch; bumping it revokes the key. +alter table buildings add column device_key_epoch integer not null default 0; diff --git a/backend/telemetry/src/adapters/driven/postgres/buildings.rs b/backend/telemetry/src/adapters/driven/postgres/buildings.rs index c6eb86c3..e92f6364 100644 --- a/backend/telemetry/src/adapters/driven/postgres/buildings.rs +++ b/backend/telemetry/src/adapters/driven/postgres/buildings.rs @@ -1,4 +1,4 @@ -use crate::kernel::ports::BuildingStore; +use crate::kernel::ports::{BuildingStore, DeviceKeyStore}; use crate::types::building::{BuildingNames, RegisteredBuilding}; use async_trait::async_trait; use sqlx::PgPool; @@ -70,3 +70,25 @@ impl BuildingStore for PgBuildings { })) } } + +#[async_trait] +impl DeviceKeyStore for PgBuildings { + async fn epoch(&self, building_id: &str) -> anyhow::Result> { + Ok( + sqlx::query_scalar("select device_key_epoch from buildings where id = $1") + .bind(building_id) + .fetch_optional(&self.pool) + .await?, + ) + } + + async fn rotate(&self, building_id: &str) -> anyhow::Result> { + Ok(sqlx::query_scalar( + "update buildings set device_key_epoch = device_key_epoch + 1 where id = $1 + returning device_key_epoch", + ) + .bind(building_id) + .fetch_optional(&self.pool) + .await?) + } +} diff --git a/backend/telemetry/src/adapters/driven/postgres/readings.rs b/backend/telemetry/src/adapters/driven/postgres/readings.rs index beda8c62..fd929d73 100644 --- a/backend/telemetry/src/adapters/driven/postgres/readings.rs +++ b/backend/telemetry/src/adapters/driven/postgres/readings.rs @@ -4,7 +4,7 @@ use crate::kernel::ports::ReadingStore; use crate::kernel::registry::PluginRegistry; use crate::types::plugin::ENVELOPE_FIELDS; use crate::types::query::Bucket; -use crate::types::reading::Reading; +use crate::types::reading::{BuildingTotal, Reading}; use async_trait::async_trait; use chrono::{DateTime, Utc}; use serde_json::{Map, Value}; @@ -140,6 +140,35 @@ impl ReadingStore for PgReadings { Ok(rows.iter().map(|row| self.construct(row)).collect()) } + async fn building_total( + &self, + building_id: &str, + metric: &str, + ) -> anyhow::Result> { + // distinct on room: a report delivered twice must not double the total. + let row = sqlx::query( + "with newest as ( + select max(ts) as ts from readings where building_id = $1 and metric = $2 + ), + rooms as ( + select distinct on (room_id) r.ts, r.value + from readings r, newest + where r.building_id = $1 and r.metric = $2 and r.ts = newest.ts + order by room_id + ) + select min(ts) as ts, sum(value) as total from rooms", + ) + .bind(building_id) + .bind(metric) + .fetch_one(&self.pool) + .await?; + let ts: Option> = row.get("ts"); + Ok(ts.map(|ts| BuildingTotal { + ts_ms: to_millis(ts), + value: row.get::, _>("total").unwrap_or_default(), + })) + } + async fn series( &self, building_id: &str, diff --git a/backend/telemetry/src/adapters/driven/postgres/sensors.rs b/backend/telemetry/src/adapters/driven/postgres/sensors.rs index ce0418c6..2aba5ea6 100644 --- a/backend/telemetry/src/adapters/driven/postgres/sensors.rs +++ b/backend/telemetry/src/adapters/driven/postgres/sensors.rs @@ -95,6 +95,17 @@ impl SensorStore for PgSensors { Ok(rows.iter().map(as_sensor).collect()) } + async fn of_type(&self, sensor_type: &str) -> anyhow::Result> { + let rows = sqlx::query( + "select building_id, room_id, sensor_id, name, sensor_type, driver, endpoint from sensors + where sensor_type = $1 order by building_id, sensor_id", + ) + .bind(sensor_type) + .fetch_all(&self.pool) + .await?; + Ok(rows.iter().map(as_sensor).collect()) + } + async fn by_room(&self, building_id: &str, room_id: &str) -> anyhow::Result> { let rows = sqlx::query( "select building_id, room_id, sensor_id, name, sensor_type, driver, endpoint from sensors diff --git a/backend/telemetry/src/adapters/driving/http_api/controllers.rs b/backend/telemetry/src/adapters/driving/http_api/controllers.rs index a3d23e73..5e643eb8 100644 --- a/backend/telemetry/src/adapters/driving/http_api/controllers.rs +++ b/backend/telemetry/src/adapters/driving/http_api/controllers.rs @@ -1,12 +1,14 @@ +use crate::adapters::ingest_auth::CollectorQuery; use crate::adapters::metrics; use crate::kernel::authz; use crate::kernel::readings::DashboardQuery; +use crate::plugins::device_count::TOTAL_DEVICE_COUNT; use crate::state::AppState; use crate::types::error::DomainError; use crate::types::identity::GatewayClaims; use crate::types::plugin::ENVELOPE_FIELDS; use crate::types::reading::Reading; -use crate::types::sensor::Command; +use crate::types::sensor::{Command, Sensor}; use crate::types::threshold::{Bounds, TemperatureLimits}; use axum::Json; use axum::extract::{Path, Query, State}; @@ -303,6 +305,77 @@ pub async fn building_sensors( Ok(Json(json!({ "data": data }))) } +/// Every router a collector may poll, by building. Signed by a collector, never by a user. +pub async fn collector( + State(state): State>, + Query(query): Query, +) -> Result, DomainError> { + // The verifier already bound the signature to this buildingId, or to the shared key. + let wanted = |building: &str| query.building_id.as_deref().is_none_or(|id| id == building); + let buildings: Vec = state + .sensors + .for_collector() + .await? + .iter() + .filter(|building| wanted(&building.building_id)) + .map(|building| { + let routers: Vec = building.routers.iter().map(collector_router).collect(); + json!({ "buildingId": building.building_id, "routers": routers }) + }) + .collect(); + Ok(Json(json!({ "buildings": buildings }))) +} + +// Absent, not null, when unset: the collector fills a missing endpoint from its own template. +fn collector_router(router: &Sensor) -> Value { + let mut entry = json!({ "sensorId": router.sensor_id, "roomId": router.room_id }); + for (key, value) in [("driver", &router.driver), ("endpoint", &router.endpoint)] { + if let Some(value) = value { + entry[key] = json!(value); + } + } + entry +} + +/// Every device connected in the building at its newest report, summed over its rooms. +pub async fn connected_devices( + State(state): State>, + Path(building_id): Path, + claims: GatewayClaims, +) -> Result, DomainError> { + read(&state, &claims, &building_id).await?; + let started = Instant::now(); + let total = state + .readings + .building_total(TOTAL_DEVICE_COUNT, &building_id) + .await; + metrics::record_query("connected_devices", started.elapsed()); + let total = total?; + Ok(Json(json!({ + "buildingId": building_id, + // Each room's count is a validated non-negative integer, so their sum is one too. + "totalDeviceCount": total.value.round() as u64, + "timestamp": total.ts_ms, + }))) +} + +/// Rotates the building's device key and returns the new one; the only time it is shown. +pub async fn issue_device_key( + State(state): State>, + Path(building_id): Path, + claims: GatewayClaims, +) -> Result, DomainError> { + edit(&state, &claims, &building_id).await?; + let key = state + .device_keys + .issue(&building_id) + .await? + .ok_or_else(|| { + DomainError::NotFound(format!("building {building_id} is not registered.")) + })?; + Ok(Json(json!({ "buildingId": building_id, "key": key }))) +} + pub async fn room_sensors( State(state): State>, Path((building_id, room_id)): Path<(String, String)>, diff --git a/backend/telemetry/src/adapters/ingest_auth.rs b/backend/telemetry/src/adapters/ingest_auth.rs index 0d99b7cb..cc1958e9 100644 --- a/backend/telemetry/src/adapters/ingest_auth.rs +++ b/backend/telemetry/src/adapters/ingest_auth.rs @@ -1,5 +1,5 @@ use axum::body::Body; -use axum::extract::{Request, State}; +use axum::extract::{Query, Request, State}; use axum::http::StatusCode; use axum::middleware::Next; use axum::response::Response; @@ -9,8 +9,13 @@ use std::fmt::Write; use std::sync::Arc; use crate::adapters::metrics; +use crate::kernel::ports::DeviceKeyStore; +use serde::Deserialize; pub const SIGNATURE_HEADER: &str = "x-signature"; +pub const TIMESTAMP_HEADER: &str = "x-timestamp"; +/// How far a collector's clock may stray; also how long a captured request stays replayable. +pub const MAX_SKEW_S: i64 = 300; const MAX_BODY_BYTES: usize = 1 << 20; const MIN_SECRET_BYTES: usize = 32; @@ -40,6 +45,122 @@ impl IngestKey { } } +/// What a collector signs for `GET /collector`: a GET has no body, so a canonical string. +pub fn collector_message(building_id: Option<&str>, timestamp: &str) -> String { + format!( + "GET /collector\n{}\n{timestamp}", + building_id.unwrap_or_default() + ) +} + +/// Stamped within `MAX_SKEW_S` of `now_s`. +pub fn is_fresh(timestamp: &str, now_s: i64) -> bool { + timestamp + .parse::() + .is_ok_and(|stamped| (now_s - stamped).abs() <= MAX_SKEW_S) +} + +/// Per-building device keys, derived from one master key, plus the shared key where configured. +#[derive(Clone)] +pub struct DeviceKeys { + master: IngestKey, + shared: Option, + epochs: Arc, +} + +impl DeviceKeys { + pub fn new( + master: &str, + shared: Option<&str>, + epochs: Arc, + ) -> anyhow::Result { + Ok(Self { + master: IngestKey::new(master)?, + shared: shared.map(IngestKey::new).transpose()?, + epochs, + }) + } + + /// The building's key at `epoch`: hex HMAC(master, "{building}:{epoch}"). + pub fn derive(&self, building_id: &str, epoch: i32) -> String { + self.master + .sign(format!("{building_id}:{epoch}").as_bytes()) + } + + /// Revokes the building's key and returns its new one; `None` when it is not registered. + pub async fn issue(&self, building_id: &str) -> anyhow::Result> { + let epoch = self.epochs.rotate(building_id).await?; + Ok(epoch.map(|epoch| self.derive(building_id, epoch))) + } + + /// Signed by the shared key, or by `building_id`'s current key. + pub async fn accepts( + &self, + building_id: Option<&str>, + message: &[u8], + signature: &str, + ) -> bool { + let signed_by = |key: &IngestKey| constant_time_eq(&key.sign(message), signature); + if self.shared.as_ref().is_some_and(signed_by) { + return true; + } + let Some(building_id) = building_id else { + return false; + }; + match self.epochs.epoch(building_id).await { + Ok(Some(epoch)) => { + IngestKey::new(&self.derive(building_id, epoch)).is_ok_and(|key| signed_by(&key)) + } + Ok(None) => false, + Err(error) => { + log::warn!("device key epoch unreadable for {building_id}: {error:?}"); + false + } + } + } +} + +#[derive(Deserialize)] +pub struct CollectorQuery { + #[serde(rename = "buildingId")] + pub building_id: Option, +} + +pub async fn verify_collector_request( + State(keys): State, + request: Request, + next: Next, +) -> Result { + let (building, message, signature, fresh) = { + let header = |name: &str| { + request + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned() + }; + let now_s = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs() as i64); + let building = Query::::try_from_uri(request.uri()) + .map_err(|_| StatusCode::BAD_REQUEST)? + .0 + .building_id; + let timestamp = header(TIMESTAMP_HEADER); + let message = collector_message(building.as_deref(), ×tamp); + let fresh = is_fresh(×tamp, now_s); + (building, message, header(SIGNATURE_HEADER), fresh) + }; + let signed = keys + .accepts(building.as_deref(), message.as_bytes(), &signature) + .await; + match fresh && signed { + true => Ok(next.run(request).await), + false => Err(StatusCode::UNAUTHORIZED), + } +} + fn constant_time_eq(a: &str, b: &str) -> bool { a.len() == b.len() && a.bytes() @@ -48,8 +169,14 @@ fn constant_time_eq(a: &str, b: &str) -> bool { == 0 } +#[derive(Deserialize)] +struct Addressed { + #[serde(rename = "buildingId")] + building_id: String, +} + pub async fn verify_signature( - State(key): State, + State(keys): State, request: Request, next: Next, ) -> Result { @@ -68,7 +195,10 @@ pub async fn verify_signature( .await .map_err(|_| StatusCode::PAYLOAD_TOO_LARGE)?; - if !constant_time_eq(&key.sign(&bytes), &signature) { + // Read only to pick the key; the handler still validates the whole body. + let building = serde_json::from_slice::(&bytes).ok(); + let building_id = building.as_ref().map(|b| b.building_id.as_str()); + if !keys.accepts(building_id, &bytes, &signature).await { metrics::record_ingest("unknown", "bad_signature"); return Err(StatusCode::UNAUTHORIZED); } @@ -81,6 +211,7 @@ pub async fn verify_signature( #[cfg(test)] mod tests { use super::*; + use crate::kernel::fakes::FakeDeviceKeys; const SECRET: &str = "0123456789abcdef0123456789abcdef"; @@ -112,6 +243,126 @@ mod tests { } } + const REQUEST_FIXTURE: &str = + include_str!("../../../../schemas/fixtures/collector-request.json"); + + #[tokio::test] + async fn collector_requests_sign_as_the_fixture_pins() { + let fixture: serde_json::Value = serde_json::from_str(REQUEST_FIXTURE).unwrap(); + let shared = IngestKey::new(fixture["secret"].as_str().unwrap()).unwrap(); + assert_eq!(fixture["maxSkewS"], MAX_SKEW_S); + for case in fixture["cases"].as_array().unwrap() { + let building = case["buildingId"].as_str().filter(|b| !b.is_empty()); + let message = collector_message(building, case["timestamp"].as_str().unwrap()); + assert_eq!( + shared.sign(message.as_bytes()), + case["signature"].as_str().unwrap(), + "{}", + case["name"] + ); + } + } + + #[test] + fn a_timestamp_is_fresh_only_within_the_skew() { + assert!(is_fresh("1000", 1000 + MAX_SKEW_S)); + assert!(is_fresh("1000", 1000 - MAX_SKEW_S)); + assert!(!is_fresh("1000", 1001 + MAX_SKEW_S)); + assert!(!is_fresh("1000", 999 - MAX_SKEW_S)); + assert!(!is_fresh("soon", 1000)); + } + + const MASTER: &str = "a-device-master-key-that-is-at-least-32-bytes"; + + fn keys(shared: Option<&str>) -> DeviceKeys { + let epochs = Arc::new(FakeDeviceKeys::registered(&[ + "bldg-3f2b4c5d", + "bldg-9a8b7c6d", + ])); + DeviceKeys::new(MASTER, shared, epochs).unwrap() + } + + #[test] + fn a_building_key_is_hmac_of_its_id_and_epoch_under_the_master_key() { + let keys = keys(None); + assert_eq!( + keys.derive("bldg-3f2b4c5d", 0), + "b42c3c8cd51584926a35e21ad0e106796e41b6b4bd19333c8b02a203ed34aa3a" + ); + assert_eq!( + keys.derive("bldg-3f2b4c5d", 1), + "f7d42c6926e516f7aaded3441173fe0a59c7b9b860838549cb5b07d4b04def6b" + ); + assert_eq!( + keys.derive("bldg-9a8b7c6d", 0), + "c33a7441dbbd659135d8672c14052891ea2451382a0c16532508b9a9d179fb8f" + ); + } + + fn signed_by(key: &str, message: &[u8]) -> String { + IngestKey::new(key).unwrap().sign(message) + } + + #[tokio::test] + async fn a_building_key_signs_for_its_own_building_only() { + let keys = keys(None); + let signature = signed_by(&keys.derive("bldg-3f2b4c5d", 0), b"tick"); + assert!( + keys.accepts(Some("bldg-3f2b4c5d"), b"tick", &signature) + .await + ); + assert!( + !keys + .accepts(Some("bldg-9a8b7c6d"), b"tick", &signature) + .await + ); + assert!(!keys.accepts(None, b"tick", &signature).await); + } + + #[tokio::test] + async fn rotating_a_building_revokes_its_previous_key() { + let epochs = Arc::new(FakeDeviceKeys::registered(&["bldg-3f2b4c5d"])); + let keys = DeviceKeys::new(MASTER, None, epochs.clone()).unwrap(); + let old = signed_by(&keys.derive("bldg-3f2b4c5d", 0), b"tick"); + + let fresh = keys.issue("bldg-3f2b4c5d").await.unwrap().unwrap(); + + assert!(!keys.accepts(Some("bldg-3f2b4c5d"), b"tick", &old).await); + let new = signed_by(&fresh, b"tick"); + assert!(keys.accepts(Some("bldg-3f2b4c5d"), b"tick", &new).await); + } + + #[tokio::test] + async fn an_unregistered_building_has_no_key_to_issue_or_accept() { + let keys = keys(None); + assert_eq!(keys.issue("ghost").await.unwrap(), None); + let forged = signed_by(&keys.derive("ghost", 0), b"tick"); + assert!(!keys.accepts(Some("ghost"), b"tick", &forged).await); + } + + #[tokio::test] + async fn the_shared_key_signs_for_any_building_only_when_configured() { + let with_shared = keys(Some(SECRET)); + let signature = key().sign(b"tick"); + assert!( + with_shared + .accepts(Some("bldg-9a8b7c6d"), b"tick", &signature) + .await + ); + assert!(with_shared.accepts(None, b"tick", &signature).await); + assert!( + !keys(None) + .accepts(Some("bldg-9a8b7c6d"), b"tick", &signature) + .await + ); + } + + #[test] + fn a_short_master_key_is_rejected() { + let epochs = Arc::new(FakeDeviceKeys::default()); + assert!(DeviceKeys::new("too-short", None, epochs).is_err()); + } + #[test] fn a_short_secret_is_rejected() { assert!(IngestKey::new("too-short").is_err()); diff --git a/backend/telemetry/src/kernel/fakes.rs b/backend/telemetry/src/kernel/fakes.rs index 21917e3a..7e5bf05e 100644 --- a/backend/telemetry/src/kernel/fakes.rs +++ b/backend/telemetry/src/kernel/fakes.rs @@ -1,6 +1,6 @@ use crate::kernel::ports::{ - ActionDispatch, Alerts, BuildingDirectory, BuildingStore, Clock, DispatchError, Fanout, - ReadingStore, RegistrationEvents, SensorStore, SimulatorControl, ThresholdStore, + ActionDispatch, Alerts, BuildingDirectory, BuildingStore, Clock, DeviceKeyStore, DispatchError, + Fanout, ReadingStore, RegistrationEvents, SensorStore, SimulatorControl, ThresholdStore, }; use crate::types::building::RegisteredBuilding; use crate::types::event::{AlertPayload, TelemetryEvent}; @@ -8,12 +8,13 @@ use crate::types::plugin::{ ActionSpec, BoundDirection, BoundSpec, FieldKind, FieldSpec, MetricDescriptor, SensorPlugin, }; use crate::types::query::Bucket; -use crate::types::reading::Reading; +use crate::types::reading::{BuildingTotal, Reading}; use crate::types::sensor::{Command, Sensor, SensorChanges}; use crate::types::simulation::SimulatedSensor; use crate::types::threshold::{Bounds, RoomTemperatureLimit, TemperatureLimits}; use async_trait::async_trait; use serde_json::{Map, Value}; +use std::collections::HashMap; use std::sync::Mutex; static FAKE_DESCRIPTOR: MetricDescriptor = MetricDescriptor { @@ -192,6 +193,29 @@ impl ReadingStore for FakeReadings { Ok(newest) } + async fn building_total( + &self, + building_id: &str, + metric: &str, + ) -> anyhow::Result> { + if self.refuse { + anyhow::bail!("readings refused"); + } + let rows: Vec<&Reading> = self + .rows + .iter() + .filter(|r| r.building_id == building_id && r.metric == metric) + .collect(); + let Some(ts_ms) = rows.iter().map(|r| r.ts_ms).max() else { + return Ok(None); + }; + let mut rooms: Vec<&Reading> = rows.into_iter().filter(|r| r.ts_ms == ts_ms).collect(); + rooms.sort_by(|a, b| a.room_id.cmp(&b.room_id)); + rooms.dedup_by(|a, b| a.room_id == b.room_id); + let value = rooms.iter().map(|r| r.value).sum(); + Ok(Some(BuildingTotal { ts_ms, value })) + } + async fn series( &self, building_id: &str, @@ -395,6 +419,20 @@ impl SensorStore for FakeSensors { .collect()) } + async fn of_type(&self, sensor_type: &str) -> anyhow::Result> { + if self.refuse { + anyhow::bail!("sensors refused"); + } + Ok(self + .registered + .lock() + .unwrap() + .iter() + .filter(|s| s.sensor_type == sensor_type) + .cloned() + .collect()) + } + async fn by_room(&self, building_id: &str, room_id: &str) -> anyhow::Result> { if self.refuse { anyhow::bail!("sensors refused"); @@ -627,3 +665,36 @@ impl SimulatorControl for FakeSimulators { Ok(self.running.iter().any(|name| name == simulator)) } } + +/// Epochs per registered building; an unknown building has none. +#[derive(Default)] +pub struct FakeDeviceKeys { + pub epochs: Mutex>, +} + +impl FakeDeviceKeys { + pub fn registered(buildings: &[&str]) -> Self { + Self { + epochs: Mutex::new(buildings.iter().map(|b| ((*b).to_owned(), 0)).collect()), + } + } +} + +#[async_trait] +impl DeviceKeyStore for FakeDeviceKeys { + async fn epoch(&self, building_id: &str) -> anyhow::Result> { + Ok(self.epochs.lock().unwrap().get(building_id).copied()) + } + + async fn rotate(&self, building_id: &str) -> anyhow::Result> { + Ok(self + .epochs + .lock() + .unwrap() + .get_mut(building_id) + .map(|epoch| { + *epoch += 1; + *epoch + })) + } +} diff --git a/backend/telemetry/src/kernel/ports.rs b/backend/telemetry/src/kernel/ports.rs index 572969c9..6eaa8250 100644 --- a/backend/telemetry/src/kernel/ports.rs +++ b/backend/telemetry/src/kernel/ports.rs @@ -1,7 +1,7 @@ use crate::types::building::{BuildingNames, RegisteredBuilding}; use crate::types::event::{AlertPayload, TelemetryEvent}; use crate::types::query::Bucket; -use crate::types::reading::Reading; +use crate::types::reading::{BuildingTotal, Reading}; use crate::types::sensor::{Command, Sensor, SensorChanges}; use crate::types::simulation::SimulatedSensor; use crate::types::threshold::{Bounds, TemperatureLimits}; @@ -24,6 +24,13 @@ pub trait ReadingStore: Send + Sync { metric: &str, ) -> anyhow::Result>; + /// Sum over rooms at the building's newest timestamp for `metric`, each room once. + async fn building_total( + &self, + building_id: &str, + metric: &str, + ) -> anyhow::Result>; + async fn series( &self, building_id: &str, @@ -76,6 +83,8 @@ pub trait SensorStore: Send + Sync { async fn apply(&self, building_id: &str, changes: &SensorChanges) -> anyhow::Result<()>; async fn by_building(&self, building_id: &str) -> anyhow::Result>; async fn by_room(&self, building_id: &str, room_id: &str) -> anyhow::Result>; + /// Every building's sensors of one device kind. + async fn of_type(&self, sensor_type: &str) -> anyhow::Result>; } #[async_trait] @@ -110,6 +119,15 @@ pub trait BuildingStore: Send + Sync { async fn names_of(&self, building_id: &str) -> anyhow::Result>; } +/// The per-building counter a device key is derived with; bumping it revokes the old key. +#[async_trait] +pub trait DeviceKeyStore: Send + Sync { + /// The building's current epoch, or `None` when it is not registered. + async fn epoch(&self, building_id: &str) -> anyhow::Result>; + /// Moves the building to a new epoch and returns it; `None` when it is not registered. + async fn rotate(&self, building_id: &str) -> anyhow::Result>; +} + #[async_trait] pub trait RegistrationEvents: Send + Sync { async fn publish_completed( diff --git a/backend/telemetry/src/kernel/readings.rs b/backend/telemetry/src/kernel/readings.rs index e141cf0e..4baac620 100644 --- a/backend/telemetry/src/kernel/readings.rs +++ b/backend/telemetry/src/kernel/readings.rs @@ -2,7 +2,7 @@ use crate::kernel::ports::{Clock, ReadingStore}; use crate::kernel::registry::PluginRegistry; use crate::types::error::DomainError; use crate::types::query::{AggMode, Bucket, TimeRange}; -use crate::types::reading::Reading; +use crate::types::reading::{BuildingTotal, Reading}; use std::sync::Arc; pub struct DashboardQuery<'a> { @@ -55,6 +55,21 @@ impl Readings { Ok(self.store.latest_per_room(building_id, metric).await?) } + /// `metric` summed over the building's rooms at its newest report; only a count adds up. + pub async fn building_total( + &self, + metric: &str, + building_id: &str, + ) -> Result { + self.known(metric)?; + self.store + .building_total(building_id, metric) + .await? + .ok_or_else(|| { + DomainError::NotFound(format!("no {metric} data found for {building_id}")) + }) + } + pub async fn dashboard(&self, query: DashboardQuery<'_>) -> Result, DomainError> { self.known(query.metric)?; let now_ms = self.clock.now_ms(); @@ -100,6 +115,55 @@ mod tests { Harness { store, readings } } + #[tokio::test] + async fn a_building_total_sums_every_room_at_the_newest_report_only() { + let h = harness(FakeReadings { + rows: vec![ + reading("fake", "b1", "r1", 10, 5.0), + reading("fake", "b1", "r1", 40, 3.0), + reading("fake", "b1", "r2", 40, 4.0), + reading("fake", "b1", "gone", 10, 9.0), + reading("fake", "b2", "r9", 50, 7.0), + ], + ..Default::default() + }); + + let total = h.readings.building_total("fake", "b1").await.unwrap(); + + assert_eq!( + total, + BuildingTotal { + ts_ms: 40, + value: 7.0 + } + ); + } + + #[tokio::test] + async fn a_report_delivered_twice_counts_each_room_once() { + let h = harness(FakeReadings { + rows: vec![ + reading("fake", "b1", "r1", 40, 3.0), + reading("fake", "b1", "r1", 40, 3.0), + reading("fake", "b1", "r2", 40, 4.0), + ], + ..Default::default() + }); + + let total = h.readings.building_total("fake", "b1").await.unwrap(); + + assert_eq!(total.value, 7.0); + } + + #[tokio::test] + async fn a_building_that_never_reported_has_no_total() { + let h = harness(FakeReadings::default()); + let missing = h.readings.building_total("fake", "b1").await.unwrap_err(); + let unknown = h.readings.building_total("nope", "b1").await.unwrap_err(); + assert!(matches!(missing, DomainError::NotFound(_))); + assert!(matches!(unknown, DomainError::NotFound(_))); + } + fn query(metric: &'static str) -> DashboardQuery<'static> { DashboardQuery { metric, diff --git a/backend/telemetry/src/kernel/sensors.rs b/backend/telemetry/src/kernel/sensors.rs index cca70ef0..22c1ed39 100644 --- a/backend/telemetry/src/kernel/sensors.rs +++ b/backend/telemetry/src/kernel/sensors.rs @@ -1,12 +1,13 @@ use crate::kernel::devices::DeviceCatalog; use crate::kernel::ports::{BuildingStore, SensorStore}; use crate::types::error::{DomainError, ItemError}; -use crate::types::sensor::{Sensor, SensorChanges, SensorUpdate}; +use crate::types::sensor::{CollectorBuilding, Sensor, SensorChanges, SensorUpdate}; use serde_json::{Map, Value}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; pub const MAX_BATCH_SENSORS: usize = 200; +pub const ROUTER: &str = "router"; pub struct Sensors { pub devices: Arc, @@ -172,6 +173,29 @@ impl Sensors { ) -> Result, DomainError> { Ok(self.store.by_room(building_id, room_id).await?) } + + /// Every router in a room, grouped by building; an outdoor one has no zone to count into. + pub async fn for_collector(&self) -> Result, DomainError> { + let mut by_building: BTreeMap> = BTreeMap::new(); + for router in self.store.of_type(ROUTER).await? { + if router.room_id.is_some() { + by_building + .entry(router.building_id.clone()) + .or_default() + .push(router); + } + } + Ok(by_building + .into_iter() + .map(|(building_id, mut routers)| { + routers.sort_by(|a, b| a.sensor_id.cmp(&b.sensor_id)); + CollectorBuilding { + building_id, + routers, + } + }) + .collect()) + } } /// Whether `sensor_id` is this building's and not already changed by the batch; records why not. @@ -292,6 +316,43 @@ mod tests { harness(FakeSensors::default()) } + fn device(building: &str, id: &str, kind: &str, room: Option<&str>) -> Sensor { + Sensor { + building_id: building.to_owned(), + room_id: room.map(str::to_owned), + sensor_id: id.to_owned(), + name: id.to_owned(), + sensor_type: kind.to_owned(), + driver: None, + endpoint: None, + } + } + + #[tokio::test] + async fn a_collector_sees_every_indoor_router_grouped_by_building() { + let h = harness(FakeSensors { + registered: std::sync::Mutex::new(vec![ + device("b2", "r9", "router", Some("hall")), + device("b1", "r2", "router", Some("lab")), + device("b1", "r1", "router", Some("lobby")), + device("b1", "roof", "router", None), + device("b1", "t1", "temperature", Some("lab")), + ]), + ..Default::default() + }); + + let found = h.sensors.for_collector().await.unwrap(); + + let ids: Vec<(&str, Vec<&str>)> = found + .iter() + .map(|b| { + let routers = b.routers.iter().map(|r| r.sensor_id.as_str()).collect(); + (b.building_id.as_str(), routers) + }) + .collect(); + assert_eq!(ids, vec![("b1", vec!["r1", "r2"]), ("b2", vec!["r9"])]); + } + fn stored(h: &Harness) -> Vec { h.store.registered.lock().unwrap().clone() } diff --git a/backend/telemetry/src/lib.rs b/backend/telemetry/src/lib.rs index 41ed07cc..4c408402 100644 --- a/backend/telemetry/src/lib.rs +++ b/backend/telemetry/src/lib.rs @@ -24,11 +24,18 @@ pub fn router(state: Arc) -> Router { let ingest = Router::new().route("/ingest", post(c::ingest)).layer( axum::middleware::from_fn_with_state( - state.ingest_key.clone(), + state.device_keys.clone(), ingest_auth::verify_signature, ), ); + let collector = Router::new().route("/collector", get(c::collector)).layer( + axum::middleware::from_fn_with_state( + state.device_keys.clone(), + ingest_auth::verify_collector_request, + ), + ); + let protected = Router::new() .route( "/thresholds/buildings/{buildingId}", @@ -58,6 +65,14 @@ pub fn router(state: Arc) -> Router { "/sensors/buildings/{buildingId}/rooms/{roomId}", get(c::room_sensors), ) + .route( + "/connected-devices/buildings/{buildingId}", + get(c::connected_devices), + ) + .route( + "/device-keys/buildings/{buildingId}", + post(c::issue_device_key), + ) .route( "/simulation/buildings/{buildingId}", get(c::simulation_status) @@ -71,6 +86,7 @@ pub fn router(state: Arc) -> Router { public .merge(ingest) + .merge(collector) .merge(protected) .with_state(state) .layer(axum::middleware::from_fn(metrics::track_metrics)) diff --git a/backend/telemetry/src/main.rs b/backend/telemetry/src/main.rs index ad8acd0c..06d02a60 100644 --- a/backend/telemetry/src/main.rs +++ b/backend/telemetry/src/main.rs @@ -10,13 +10,13 @@ use telemetry::adapters::driven::threshold_cache::CachedThresholds; use telemetry::adapters::driven::twin_directory::TwinDirectory; use telemetry::adapters::driving::kafka_consumer; use telemetry::adapters::health::probe_exit_code; -use telemetry::adapters::ingest_auth::IngestKey; +use telemetry::adapters::ingest_auth::DeviceKeys; use telemetry::kernel::actions::Actions; use telemetry::kernel::devices::DeviceCatalog; use telemetry::kernel::ingest::Ingest; use telemetry::kernel::ports::{ - Alerts, BuildingDirectory, BuildingStore, Clock, Fanout, ReadingStore, RegistrationEvents, - SensorStore, SimulatorControl, ThresholdStore, + Alerts, BuildingDirectory, BuildingStore, Clock, DeviceKeyStore, Fanout, ReadingStore, + RegistrationEvents, SensorStore, SimulatorControl, ThresholdStore, }; use telemetry::kernel::readings::Readings; use telemetry::kernel::registration::Registration; @@ -45,7 +45,11 @@ async fn main() -> anyhow::Result<()> { } let database_url = std::env::var("DATABASE_URL")?; - let ingest_key = IngestKey::new(&std::env::var("TELEMETRY_INGEST_SECRET")?)?; + let master_key = std::env::var("TELEMETRY_DEVICE_MASTER_KEY")?; + // Dev and simulators only: signs for any building. Production leaves it unset. + let shared_key = std::env::var("TELEMETRY_INGEST_SECRET") + .ok() + .filter(|secret| !secret.is_empty()); let redis_url = env_or("REDIS_URL", "redis://redis:6379"); let brokers = env_or("KAFKA_BROKERS", "kafka:9092"); let twin_url = env_or("DIGITAL_TWIN_URL", "http://digital-twin:3000"); @@ -72,6 +76,12 @@ async fn main() -> anyhow::Result<()> { )))); let sensors_store = Arc::new(PgSensors::new(pool.clone())); // Shared for the same reason: a re-registration must reach the names cache ingest reads. + // Uncached on purpose: a rotated key must stop working on every replica at once. + let device_keys = DeviceKeys::new( + &master_key, + shared_key.as_deref(), + Arc::new(PgBuildings::new(pool.clone())) as Arc, + )?; let buildings_store = Arc::new(CachedBuildings::new(Arc::new(PgBuildings::new( pool.clone(), )))); @@ -111,7 +121,7 @@ async fn main() -> anyhow::Result<()> { directory: directory.clone() as Arc, dispatch: dispatch.clone(), pool: pool.clone(), - ingest_key, + device_keys, ingest: Ingest { registry: registry.clone(), readings: readings_store.clone() as Arc, diff --git a/backend/telemetry/src/plugins/device_count.rs b/backend/telemetry/src/plugins/device_count.rs index ad1c8b5b..6d163eeb 100644 --- a/backend/telemetry/src/plugins/device_count.rs +++ b/backend/telemetry/src/plugins/device_count.rs @@ -5,9 +5,12 @@ use crate::types::plugin::{ use crate::types::reading::Reading; use serde_json::Value; +/// Devices a router hears; its rooms add up to the building's total. +pub const TOTAL_DEVICE_COUNT: &str = "totalDeviceCount"; + static TOTAL_DESCRIPTOR: MetricDescriptor = MetricDescriptor { - value_field: "totalDeviceCount", - key: "totalDeviceCount", + value_field: TOTAL_DEVICE_COUNT, + key: TOTAL_DEVICE_COUNT, label: "Total Device Count", interface_name: "ITotalDeviceCount", unit: Some("devices"), diff --git a/backend/telemetry/src/state.rs b/backend/telemetry/src/state.rs index 06d17a05..3030ab24 100644 --- a/backend/telemetry/src/state.rs +++ b/backend/telemetry/src/state.rs @@ -1,5 +1,5 @@ use crate::adapters::driven::dispatch::HttpDispatch; -use crate::adapters::ingest_auth::IngestKey; +use crate::adapters::ingest_auth::DeviceKeys; use crate::kernel::actions::Actions; use crate::kernel::devices::DeviceCatalog; use crate::kernel::ingest::Ingest; @@ -28,7 +28,7 @@ pub struct AppState { pub pool: sqlx::PgPool, pub directory: Arc, pub dispatch: Arc, - pub ingest_key: IngestKey, + pub device_keys: DeviceKeys, pub ingest: Ingest, pub readings: Readings, pub thresholds: Thresholds, diff --git a/backend/telemetry/src/types/reading.rs b/backend/telemetry/src/types/reading.rs index 59d2e74c..0d1170e3 100644 --- a/backend/telemetry/src/types/reading.rs +++ b/backend/telemetry/src/types/reading.rs @@ -9,3 +9,10 @@ pub struct Reading { pub value: f64, pub payload: Map, } + +/// One metric summed over a building's rooms at its newest report. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct BuildingTotal { + pub ts_ms: i64, + pub value: f64, +} diff --git a/backend/telemetry/src/types/sensor.rs b/backend/telemetry/src/types/sensor.rs index acf5f55b..9e3680cc 100644 --- a/backend/telemetry/src/types/sensor.rs +++ b/backend/telemetry/src/types/sensor.rs @@ -36,3 +36,10 @@ pub struct Command { pub action: String, pub arguments: Map, } + +/// A building's routers in rooms: what a collector may poll there. +#[derive(Debug, Clone, PartialEq)] +pub struct CollectorBuilding { + pub building_id: String, + pub routers: Vec, +} diff --git a/backend/telemetry/tests/api.rs b/backend/telemetry/tests/api.rs index ccb7801a..fd19873b 100644 --- a/backend/telemetry/tests/api.rs +++ b/backend/telemetry/tests/api.rs @@ -481,6 +481,207 @@ async fn sensor_app(label: &str) -> support::test_app::TestApp { test_app(pool, vec!["eng"]).await } +fn now_s() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() as i64 +} + +#[tokio::test] +async fn a_collector_reads_every_indoor_router_with_its_address_and_no_login() { + let app = sensor_app("collector").await; + let (_, created) = app + .send_json( + "POST", + SENSORS_B1, + Some(&staff()), + json!({ "create": [ + { "ref": "hall", "name": "Hall AP", "sensorType": "router", "roomId": "r1", + "driver": "openwrt-hostapd", "endpoint": "http://10.0.4.12/ubus" }, + { "ref": "lab", "name": "Lab AP", "sensorType": "router", "roomId": "r2" }, + { "ref": "yard", "name": "Yard AP", "sensorType": "router", "roomId": null }, + { "ref": "t", "name": "Thermostat", "sensorType": "temperature", "roomId": "r1" }, + ]}), + ) + .await; + let id = |reference: &str| { + created["created"] + .as_array() + .unwrap() + .iter() + .find(|c| c["ref"] == reference) + .unwrap()["sensorId"] + .as_str() + .unwrap() + .to_owned() + }; + let mut routers = vec![ + json!({ "sensorId": id("hall"), "roomId": "r1", + "driver": "openwrt-hostapd", "endpoint": "http://10.0.4.12/ubus" }), + json!({ "sensorId": id("lab"), "roomId": "r2" }), + ]; + routers.sort_by_key(|router| router["sensorId"].as_str().unwrap().to_owned()); + + let (status, body) = app.collector_at(now_s()).await; + + assert_eq!(status, StatusCode::OK); + assert_eq!( + body, + json!({ "buildings": [{ "buildingId": "b1", "routers": routers }] }) + ); +} + +fn devices_tick(ts_ms: i64, rooms: &[(&str, u32)]) -> serde_json::Value { + json!({ + "buildingId": "b1", + "readings": rooms.iter().map(|(room, count)| json!({ + "type": "totalDeviceCount", "roomId": room, + "timestamp": ts_ms, "totalDeviceCount": count + })).collect::>(), + }) +} + +const CONNECTED_B1: &str = "/connected-devices/buildings/b1"; +const CONNECTED_WIRE: &str = include_str!("../../../schemas/fixtures/connected-devices.json"); + +#[tokio::test] +async fn a_building_answers_its_connected_devices_at_the_newest_report() { + let app = sensor_app("connected").await; + app.ingest(devices_tick(BASE_MS, &[("r1", 5), ("r2", 2)])) + .await; + app.ingest(devices_tick(BASE_MS + 5000, &[("r1", 3)])).await; + + let (status, body) = app.get(CONNECTED_B1, Some(&customer())).await; + + assert_eq!(status, StatusCode::OK); + assert_eq!( + body, + json!({ "buildingId": "b1", "totalDeviceCount": 3, "timestamp": BASE_MS + 5000 }) + ); + let wire: serde_json::Value = serde_json::from_str(CONNECTED_WIRE).unwrap(); + let pinned = wire["cases"][0]["body"].as_object().unwrap(); + let served = body.as_object().unwrap(); + assert!( + served.keys().eq(pinned.keys()), + "served {served:?}, pinned {pinned:?}" + ); +} + +#[tokio::test] +async fn connected_devices_are_not_found_before_any_report_and_forbidden_to_outsiders() { + let app = sensor_app("connectedauth").await; + let (never, _) = app.get(CONNECTED_B1, Some(&customer())).await; + let (outsider, _) = app.get(CONNECTED_B1, Some(&outsider())).await; + assert_eq!(never, StatusCode::NOT_FOUND); + assert_eq!(outsider, StatusCode::FORBIDDEN); +} + +async fn keyed_app(label: &str) -> support::test_app::TestApp { + let pool = fresh_db(label).await; + seed_building(&pool, "b1", &["r1", "r2"]).await; + seed_building(&pool, "b2", &["r9"]).await; + test_app(pool, vec!["eng"]).await +} + +async fn issue_key(app: &support::test_app::TestApp, building: &str) -> String { + let uri = format!("/device-keys/buildings/{building}"); + let (status, body) = app + .send_json("POST", &uri, Some(&staff()), json!(null)) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["buildingId"], building); + body["key"].as_str().unwrap().to_owned() +} + +fn temperature_in(building: &str) -> serde_json::Value { + let mut body = temperature("r1", BASE_MS, 21.0); + body["buildingId"] = json!(building); + body +} + +#[tokio::test] +async fn only_an_editor_gets_a_building_key_and_it_signs_for_that_building_alone() { + let app = keyed_app("devicekey").await; + let (reader, _) = app + .send_json( + "POST", + "/device-keys/buildings/b1", + Some(&customer()), + json!(null), + ) + .await; + let (ghost, _) = app + .send_json( + "POST", + "/device-keys/buildings/ghost", + Some(&staff()), + json!(null), + ) + .await; + let key = issue_key(&app, "b1").await; + + let (own, _) = app.ingest_with(&key, temperature_in("b1")).await; + let (other, _) = app.ingest_with(&key, temperature_in("b2")).await; + + assert_eq!(reader, StatusCode::FORBIDDEN); + assert_eq!(ghost, StatusCode::NOT_FOUND); + assert_eq!(own, StatusCode::ACCEPTED); + assert_eq!(other, StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn issuing_a_building_key_again_revokes_the_previous_one() { + let app = keyed_app("devicekeyrotate").await; + let old = issue_key(&app, "b1").await; + let new = issue_key(&app, "b1").await; + + let (with_old, _) = app.ingest_with(&old, temperature_in("b1")).await; + let (with_new, _) = app.ingest_with(&new, temperature_in("b1")).await; + + assert_eq!(with_old, StatusCode::UNAUTHORIZED); + assert_eq!(with_new, StatusCode::ACCEPTED); +} + +#[tokio::test] +async fn a_building_key_reads_only_its_own_routers() { + let app = keyed_app("devicekeyread").await; + for (building, room) in [("b1", "r1"), ("b2", "r9")] { + app.send_json( + "POST", + &format!("/sensors/buildings/{building}"), + Some(&staff()), + json!({ "create": [{ "ref": "ap", "name": "AP", "sensorType": "router", "roomId": room }] }), + ) + .await; + } + let key = issue_key(&app, "b1").await; + + let (status, body) = app.collector_as(&key, Some("b1"), now_s()).await; + let (other, _) = app.collector_as(&key, Some("b2"), now_s()).await; + let (every, _) = app.collector_as(&key, None, now_s()).await; + + assert_eq!(status, StatusCode::OK); + let buildings: Vec<&str> = body["buildings"] + .as_array() + .unwrap() + .iter() + .map(|b| b["buildingId"].as_str().unwrap()) + .collect(); + assert_eq!(buildings, vec!["b1"]); + assert_eq!(other, StatusCode::UNAUTHORIZED); + assert_eq!(every, StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn a_collector_request_unsigned_or_stale_is_refused() { + let app = sensor_app("collectorauth").await; + let (unsigned, _) = app.get("/collector", None).await; + let (stale, _) = app.collector_at(now_s() - 3600).await; + assert_eq!(unsigned, StatusCode::UNAUTHORIZED); + assert_eq!(stale, StatusCode::UNAUTHORIZED); +} + #[tokio::test] async fn a_sensor_batch_returns_server_ids_and_the_list_shows_them() { let app = sensor_app("sensorsave").await; diff --git a/backend/telemetry/tests/persistence.rs b/backend/telemetry/tests/persistence.rs index b4618547..88dee511 100644 --- a/backend/telemetry/tests/persistence.rs +++ b/backend/telemetry/tests/persistence.rs @@ -7,9 +7,10 @@ use telemetry::adapters::driven::postgres::{PgBuildings, PgReadings, PgSensors, use telemetry::kernel::ports::{BuildingStore, ReadingStore, SensorStore, ThresholdStore}; use telemetry::kernel::registry::PluginRegistry; use telemetry::plugins::air_quality::AirQualityPlugin; +use telemetry::plugins::device_count::TotalDeviceCountPlugin; use telemetry::plugins::temperature::TemperaturePlugin; use telemetry::types::building::{RegisteredBuilding, Room}; -use telemetry::types::reading::Reading; +use telemetry::types::reading::{BuildingTotal, Reading}; use telemetry::types::sensor::{Sensor, SensorChanges, SensorUpdate}; const HOUR_MS: i64 = 3_600_000; @@ -20,6 +21,7 @@ fn registry() -> Arc { PluginRegistry::new(vec![ Box::new(TemperaturePlugin), Box::new(AirQualityPlugin), + Box::new(TotalDeviceCountPlugin), ]) .unwrap(), ) @@ -163,6 +165,55 @@ async fn latest_per_room_returns_the_newest_row_for_each_room_and_nothing_else() assert_eq!(rows[1].room_id, "r2"); } +fn devices(building: &str, room: &str, ts_ms: i64, count: u32) -> Reading { + Reading { + building_id: building.to_owned(), + ..reading( + "totalDeviceCount", + room, + ts_ms, + f64::from(count), + json!({ "buildingId": building, "roomId": room, "timestamp": ts_ms, "totalDeviceCount": count }), + ) + } +} + +#[tokio::test] +async fn a_building_total_sums_each_room_once_at_the_newest_report() { + let pool = fresh_db("buildingtotal").await; + let readings = PgReadings::new(pool, registry()); + let (before, newest) = (BASE_MS, BASE_MS + 5000); + readings + .insert(&[ + devices("b1", "r1", before, 5), + devices("b1", "gone", before, 9), + devices("b1", "r1", newest, 3), + devices("b1", "r2", newest, 4), + devices("b1", "r2", newest, 4), + devices("b2", "r9", newest + 1000, 7), + ]) + .await + .unwrap(); + + let total = readings + .building_total("b1", "totalDeviceCount") + .await + .unwrap(); + let never = readings + .building_total("b3", "totalDeviceCount") + .await + .unwrap(); + + assert_eq!( + total, + Some(BuildingTotal { + ts_ms: newest, + value: 7.0 + }) + ); + assert_eq!(never, None); +} + #[tokio::test] async fn series_buckets_hourly_for_a_one_day_range() { let pool = fresh_db("series").await; diff --git a/backend/telemetry/tests/support/test_app.rs b/backend/telemetry/tests/support/test_app.rs index 79adea9e..f0fd1473 100644 --- a/backend/telemetry/tests/support/test_app.rs +++ b/backend/telemetry/tests/support/test_app.rs @@ -10,13 +10,15 @@ use std::sync::Arc; use telemetry::adapters::driven::dispatch::HttpDispatch; use telemetry::adapters::driven::postgres::{PgBuildings, PgReadings, PgSensors, PgThresholds}; use telemetry::adapters::driven::simulators::HttpSimulators; -use telemetry::adapters::ingest_auth::{IngestKey, SIGNATURE_HEADER}; +use telemetry::adapters::ingest_auth::{ + DeviceKeys, IngestKey, SIGNATURE_HEADER, TIMESTAMP_HEADER, collector_message, +}; use telemetry::kernel::actions::Actions; use telemetry::kernel::devices::DeviceCatalog; use telemetry::kernel::ingest::Ingest; use telemetry::kernel::ports::{ - Alerts, BuildingDirectory, BuildingStore, Clock, Fanout, ReadingStore, SensorStore, - SimulatorControl, ThresholdStore, + Alerts, BuildingDirectory, BuildingStore, Clock, DeviceKeyStore, Fanout, ReadingStore, + SensorStore, SimulatorControl, ThresholdStore, }; use telemetry::kernel::readings::Readings; use telemetry::kernel::registration::Registration; @@ -40,6 +42,7 @@ pub struct TestApp { } pub const INGEST_SECRET: &str = "test-ingest-secret-0123456789abcdef"; +pub const MASTER_KEY: &str = "test-device-master-key-0123456789abcdef"; pub const BINDINGS: &str = r#"{ "tp-simulator": { @@ -76,6 +79,12 @@ pub async fn test_app_with_simulators( let buildings_store = Arc::new(PgBuildings::new(pool.clone())); let dispatch = Arc::new(HttpDispatch::from_json(pool.clone(), BINDINGS).unwrap()); let ingest_key = IngestKey::new(INGEST_SECRET).unwrap(); + let device_keys = DeviceKeys::new( + MASTER_KEY, + Some(INGEST_SECRET), + buildings_store.clone() as Arc, + ) + .unwrap(); let alerts = Arc::new(StubAlerts::default()); let fanout = Arc::new(StubFanout::default()); let (simulators, simulator_specs) = HttpSimulators::from_json(simulators).unwrap(); @@ -97,7 +106,7 @@ pub async fn test_app_with_simulators( pool: pool.clone(), directory: directory.clone() as Arc, dispatch: dispatch.clone(), - ingest_key: ingest_key.clone(), + device_keys, ingest: Ingest { registry: registry.clone(), readings: readings_store.clone() as Arc, @@ -203,6 +212,44 @@ impl TestApp { read_json(self.send(request).await).await } + /// `GET /collector` for every building, signed with the shared key at `timestamp` (unix s). + pub async fn collector_at(&self, timestamp: i64) -> (StatusCode, Value) { + self.collector_as(INGEST_SECRET, None, timestamp).await + } + + /// `GET /collector`, signed with `key` for `building` at `timestamp` (unix seconds). + pub async fn collector_as( + &self, + key: &str, + building: Option<&str>, + timestamp: i64, + ) -> (StatusCode, Value) { + let stamp = timestamp.to_string(); + let message = collector_message(building, &stamp); + let uri = match building { + Some(building) => format!("/collector?buildingId={building}"), + None => "/collector".to_owned(), + }; + let request = Request::builder() + .method("GET") + .uri(uri) + .header( + SIGNATURE_HEADER, + IngestKey::new(key).unwrap().sign(message.as_bytes()), + ) + .header(TIMESTAMP_HEADER, stamp) + .body(Body::empty()) + .unwrap(); + read_json(self.send(request).await).await + } + + /// Posts `body` to ingest, signed with `key` instead of the shared one. + pub async fn ingest_with(&self, key: &str, body: Value) -> (StatusCode, Value) { + let raw = body.to_string(); + let signature = IngestKey::new(key).unwrap().sign(raw.as_bytes()); + self.ingest_signed(&raw, &signature).await + } + pub async fn ingest_unsigned(&self, body: Value) -> (StatusCode, Value) { self.send_json("POST", "/ingest", None, body).await } diff --git a/devices/ap-collector/app/__main__.py b/devices/ap-collector/app/__main__.py index 424f29f4..ced9066a 100644 --- a/devices/ap-collector/app/__main__.py +++ b/devices/ap-collector/app/__main__.py @@ -6,15 +6,17 @@ import time from typing import TYPE_CHECKING -from app.collector import build_sessions, build_trackers, readings_for_building, run +from app.collector import build_sessions, build_trackers, readings_for_building, run, sync_state from app.config import Config from app.ingest import IngestError, post_batch -from app.zones import DEFAULT_FROZEN_POLLS +from app.routers import SyncError, buildings_from, fetch_routers +from app.zones import DEFAULT_FROZEN_POLLS, ZoneTracker if TYPE_CHECKING: - from collections import Counter from collections.abc import Callable + from app.collector import TickResult + def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(prog="python -m app") @@ -44,17 +46,15 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: def _make_print_tick( config: Config, -) -> Callable[[dict[str, tuple[dict[str, str], Counter[tuple[str, str]]]]], None]: +) -> Callable[[dict[str, TickResult]], None]: """Dry-run `on_tick`: shares `readings_for_building` with the real post path so the preview always matches what would actually be sent, `devicesPerPerson` included.""" - buildings_by_name = {building.name: building for building in config.buildings} - def on_tick(results: dict[str, tuple[dict[str, str], Counter[tuple[str, str]]]]) -> None: + def on_tick(results: dict[str, TickResult]) -> None: now_ms = int(time.time() * 1000) - for building_name, (assignment, moves) in results.items(): - building = buildings_by_name[building_name] + for building_name, (assignment, moves, answered) in results.items(): readings = readings_for_building( - building, assignment, now_ms, config.devices_per_person + assignment, answered, now_ms, config.devices_per_person ) counts: dict[str, dict[str, object]] = {} for reading in readings: @@ -72,24 +72,28 @@ def on_tick(results: dict[str, tuple[dict[str, str], Counter[tuple[str, str]]]]) def _make_post_tick( config: Config, -) -> Callable[[dict[str, tuple[dict[str, str], Counter[tuple[str, str]]]]], None]: +) -> Callable[[dict[str, TickResult]], None]: """Real-mode `on_tick`: turn each building's confirmed assignment into occupancy readings and POST them. Built once (needs config's secret/URL/buildings), not per tick. A failed POST is dropped, not raised: an escaping IngestError would end the process, and a tick is a snapshot the next supersedes. Caught per building, so one rejected batch does not skip the buildings after it.""" - buildings_by_name = {building.name: building for building in config.buildings} ingest_url = config.telemetry_service.rstrip("/") + "/ingest" - secret = config.telemetry_secret.encode("utf-8") - def on_tick(results: dict[str, tuple[dict[str, str], Counter[tuple[str, str]]]]) -> None: + def on_tick(results: dict[str, TickResult]) -> None: now_ms = int(time.time() * 1000) - for building_name, (assignment, _moves) in results.items(): - building = buildings_by_name[building_name] + for building_name, (assignment, _moves, answered) in results.items(): readings = readings_for_building( - building, assignment, now_ms, config.devices_per_person + assignment, answered, now_ms, config.devices_per_person ) + if not readings: + # No router answered and no device is held: nothing was measured, so stay silent. + continue + secret = config.key_for(building_name) + if secret is None: + print(f"{building_name}: no device key, dropping this tick", file=sys.stderr) + continue try: post_batch( ingest_url, secret, building_name, readings, timeout=config.default_timeout @@ -100,6 +104,24 @@ def on_tick(results: dict[str, tuple[dict[str, str], Counter[tuple[str, str]]]]) return on_tick +def _refreshing( + on_tick: Callable[[dict[str, TickResult]], None], + refresh: Callable[[], None], + every: int, +) -> Callable[[dict[str, TickResult]], None]: + """`on_tick`, then a router-list refresh after every `every` ticks.""" + ticks = 0 + + def wrapped(results: dict[str, TickResult]) -> None: + nonlocal ticks + on_tick(results) + ticks += 1 + if ticks % every == 0: + refresh() + + return wrapped + + def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) @@ -115,12 +137,14 @@ def main(argv: list[str] | None = None) -> int: config = Config([]) config.load_from_config_file(args.config) + # Even a dry run reads its routers from telemetry: there is no other list. + config.load_env() + on_tick = _make_print_tick(config) if args.dry_run else _make_post_tick(config) - if args.dry_run: - on_tick = _make_print_tick(config) - else: - config.load_env() - on_tick = _make_post_tick(config) + def new_tracker() -> ZoneTracker: + return ZoneTracker( + args.hysteresis_polls, args.hysteresis_margin_db, args.absent_polls, args.frozen_polls + ) sessions_by_building = build_sessions(config, timeout=config.default_timeout) trackers_by_building = build_trackers( @@ -131,12 +155,37 @@ def main(argv: list[str] | None = None) -> int: frozen_polls=args.frozen_polls, ) + def refresh() -> None: + # Each building key reads its own building; the shared dev key reads every one. + shared = config.shared_key + reads = [(b, k) for b, k in config.keys.items()] or ([(None, shared)] if shared else []) + found: list[dict] = [] + for building_id, key in reads: + try: + answer = fetch_routers( + config.telemetry_service, key, config.default_timeout, building_id + ) + except SyncError as error: + print(f"keeping the previous router list: {error}", file=sys.stderr) + return + found += answer["buildings"] + config.buildings = buildings_from({"buildings": found}, config.site) + sync_state( + config, + sessions_by_building, + trackers_by_building, + config.default_timeout, + new_tracker, + ) + + refresh() + every = max(1, round(config.sync_interval / config.poll_interval)) run( config, sessions_by_building, trackers_by_building, interval_s=config.poll_interval, - on_tick=on_tick, + on_tick=_refreshing(on_tick, refresh, every), max_ticks=1 if args.once else None, ) return 0 diff --git a/devices/ap-collector/app/collector.py b/devices/ap-collector/app/collector.py index 5b6a6154..8326e0f3 100644 --- a/devices/ap-collector/app/collector.py +++ b/devices/ap-collector/app/collector.py @@ -10,20 +10,29 @@ import math import time from concurrent.futures import ThreadPoolExecutor -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple from app.ubus import ApSession, UbusError from app.zones import ZoneTracker, best_by_zone if TYPE_CHECKING: from collections import Counter - from collections.abc import Callable, Mapping, Sequence + from collections.abc import Callable, Collection, Mapping, Sequence - from app.config import Building, Config + from app.config import AccessPoint, Building, Config from app.ubus import StationsSource from app.zones import Reading +class TickResult(NamedTuple): + """One building's tick: where each device is, who moved, and which rooms were measured.""" + + assignment: dict[str, str] + moves: Counter[tuple[str, str]] + answered: set[str] + """Rooms with a router that answered; any other room was not measured this tick.""" + + def poll_one(ap_name: str, session: StationsSource) -> tuple[str, list[tuple[str, int]] | None]: """One AP's stations this tick, or None if it failed to answer. @@ -71,7 +80,7 @@ def batch_poll_by_building( def tick_building( building: Building, sessions: Mapping[str, StationsSource], tracker: ZoneTracker -) -> tuple[dict[str, str], Counter[tuple[str, str]]]: +) -> TickResult: """One building, one tick, fully assembled: poll, resolve per-zone signal strength, and hand it to the building's own long-lived `ZoneTracker` for the confirmed result. @@ -81,14 +90,15 @@ def tick_building( """ readings, available = batch_poll_by_building(building, sessions) observed = best_by_zone(readings, _ap_zones(building)) - return tracker.update(observed, available) + assignment, moves = tracker.update(observed, available) + return TickResult(assignment, moves, available) def tick( config: Config, sessions_by_building: Mapping[str, Mapping[str, StationsSource]], trackers_by_building: Mapping[str, ZoneTracker], -) -> dict[str, tuple[dict[str, str], Counter[tuple[str, str]]]]: +) -> dict[str, TickResult]: """One tick across every building in `config`, keyed by building name. `sessions_by_building` and `trackers_by_building` are external, long-lived state -- built @@ -119,6 +129,40 @@ def build_sessions(config: Config, timeout: int) -> dict[str, dict[str, ApSessio } +def _same_ap(a: AccessPoint, b: AccessPoint) -> bool: + fields = ("url", "username", "password", "ifaces", "reader") + return all(getattr(a, f) == getattr(b, f) for f in fields) + + +def sync_state( + config: Config, + sessions_by_building: dict[str, dict[str, ApSession]], + trackers_by_building: dict[str, ZoneTracker], + timeout: int, + new_tracker: Callable[[], ZoneTracker], +) -> None: + """Reconcile the long-lived state with `config.buildings` after a router-list refresh. + + Kept in place, not rebuilt: a router that stayed keeps its login token, and a building + that stayed keeps its tracker -- rebuilding would reset every device's hysteresis. + """ + wanted = {building.name: building for building in config.buildings} + for name in set(sessions_by_building) - set(wanted): + del sessions_by_building[name] + for name in set(trackers_by_building) - set(wanted): + del trackers_by_building[name] + for name, building in wanted.items(): + current = sessions_by_building.setdefault(name, {}) + aps = {ap.name: ap for ap in building.ap} + for ap_name in set(current) - set(aps): + del current[ap_name] + for ap_name, ap in aps.items(): + held = current.get(ap_name) + if held is None or not _same_ap(held.ap, ap): + current[ap_name] = ApSession(ap, timeout) + trackers_by_building.setdefault(name, new_tracker()) + + def build_trackers( config: Config, polls: int, margin_db: float, absent_polls: int, frozen_polls: int ) -> dict[str, ZoneTracker]: @@ -136,8 +180,7 @@ def run( trackers_by_building: Mapping[str, ZoneTracker], interval_s: float, *, - on_tick: Callable[[dict[str, tuple[dict[str, str], Counter[tuple[str, str]]]]], None] - | None = None, + on_tick: Callable[[dict[str, TickResult]], None] | None = None, max_ticks: int | None = None, now: Callable[[], float] = time.monotonic, sleep: Callable[[float], None] = time.sleep, @@ -169,18 +212,22 @@ def run( def readings_for_building( - building: Building, assignment: Mapping[str, str], + answered: Collection[str], now_ms: int, devices_per_person: float | None = None, ) -> list[dict[str, str | int]]: - """Confirmed zone assignment -> the tick's readings, one pair per declared zone. + """Confirmed assignment -> the tick's readings, one pair per room that has something to say. + + A room reports when a router in it answered (0 is then a real empty room) or the tracker + still holds devices in it. A room with neither was not measured: a zero for it would keep + a dead router's room reading a fresh 0 forever. `totalDeviceCount` is measured; `ratioDeviceCount` divides it by the site's devices-per-person factor, and is skipped entirely when none is configured. Rounds up: flooring 0.4 of a person would report an occupied room as empty. """ - counts = dict.fromkeys(set(_ap_zones(building).values()), 0) + counts = dict.fromkeys(answered, 0) for zone in assignment.values(): counts[zone] = counts.get(zone, 0) + 1 diff --git a/devices/ap-collector/app/config.py b/devices/ap-collector/app/config.py index fc87a4df..2c3f634a 100644 --- a/devices/ap-collector/app/config.py +++ b/devices/ap-collector/app/config.py @@ -1,7 +1,9 @@ from __future__ import annotations +import dataclasses import json import os +from dataclasses import dataclass, replace from pathlib import Path READERS = ("hostapd", "iwinfo") @@ -10,6 +12,50 @@ """Must stay <= DEFAULT_POLL_INTERVAL -- see Config._validate.""" DEFAULT_DEVICES_PER_PERSON = 2.5 """Used only when useDevicesPerPerson is true and devicesPerPerson is omitted.""" +DEFAULT_SYNC_INTERVAL = 300 +"""How often the router list is re-read from telemetry, in seconds.""" +DEFAULT_IFACES = ("wlan0",) + + +@dataclass(frozen=True) +class RouterLogin: + """How the collector logs in to a router. Stays on the site: telemetry never holds it.""" + + username: str + password: str + ifaces: tuple[str, ...] + + +@dataclass(frozen=True) +class Site: + """Everything about this site's routers that is not the platform's to know.""" + + login: RouterLogin + overrides: dict[str, dict] = dataclasses.field(default_factory=dict) + endpoint_template: str | None = None + + def login_for(self, sensor_id: str) -> RouterLogin: + override = self.overrides.get(sensor_id, {}) + return replace( + self.login, + username=override.get("username", self.login.username), + password=override.get("password", self.login.password), + ifaces=tuple(override.get("ifaces", self.login.ifaces)), + ) + + @classmethod + def from_json(cls, data: dict) -> Site: + ubus = data.get("ubus") + if not isinstance(ubus, dict) or not ubus.get("username"): + raise ValueError("config: ubus.username and ubus.password must be set") + ifaces = tuple(data.get("ifaces", DEFAULT_IFACES)) + if not ifaces or not all(isinstance(i, str) and i for i in ifaces): + raise ValueError("config: ifaces must be a non-empty list of interface names") + return cls( + login=RouterLogin(ubus["username"], ubus.get("password", ""), ifaces), + overrides=dict(data.get("overrides", {})), + endpoint_template=data.get("endpointTemplate"), + ) class AccessPoint: @@ -88,11 +134,23 @@ def __init__( self.poll_interval: int = poll_interval self.default_timeout: int = default_timeout self.devices_per_person: float | None = devices_per_person + self.sync_interval: float = DEFAULT_SYNC_INTERVAL + self.site: Site = Site(RouterLogin("", "", DEFAULT_IFACES)) + self.keys: dict[str, bytes] = {} + self.telemetry_secret: str | None = None def load_from_config_file(self, config_file_path: str) -> None: with Path.open(Path(config_file_path)) as f: data = json.load(f) - buildings = [Building.from_json(building) for building in data.get("buildings", [])] + if "buildings" in data: + raise ValueError("config: buildings are read from telemetry; remove them from the file") + site = Site.from_json(data) + keys = data.get("keys", {}) + if not isinstance(keys, dict) or not all( + isinstance(key, str) and len(key) >= 32 for key in keys.values() + ): + raise ValueError("config: keys must map building ids to device keys") + sync_interval = data.get("syncIntervalS", DEFAULT_SYNC_INTERVAL) poll_interval = data.get("pollIntervalS", DEFAULT_POLL_INTERVAL) default_timeout = data.get("requestTimeoutS", DEFAULT_REQUEST_TIMEOUT) devices_per_person = ( @@ -100,8 +158,10 @@ def load_from_config_file(self, config_file_path: str) -> None: if data.get("useDevicesPerPerson", False) else None ) - self._validate(buildings, poll_interval, default_timeout, devices_per_person) - self.buildings = buildings + self._validate(poll_interval, default_timeout, devices_per_person) + self.site = site + self.keys = {building: key.encode("utf-8") for building, key in keys.items()} + self.sync_interval = sync_interval self.poll_interval = poll_interval self.default_timeout = default_timeout self.devices_per_person = devices_per_person @@ -112,24 +172,27 @@ def load_env(self) -> None: if not telemetry_service: raise ValueError("config: TELEMETRY_SERVICE_URL must be set") - if not telemetry_secret: - raise ValueError("config: TELEMETRY_SERVICE_SECRET must be set") + if not telemetry_secret and not self.keys: + raise ValueError("config: set per-building keys, or TELEMETRY_SERVICE_SECRET in dev") self.telemetry_service = telemetry_service - self.telemetry_secret = telemetry_secret + self.telemetry_secret = telemetry_secret or None + + @property + def shared_key(self) -> bytes | None: + """The dev key that signs for any building; production sets none.""" + return self.telemetry_secret.encode("utf-8") if self.telemetry_secret else None + + def key_for(self, building_id: str) -> bytes | None: + """The building's own device key, else the shared dev key, else None.""" + return self.keys.get(building_id, self.shared_key) def _validate( self, - buildings: list[Building], poll_interval: float, default_timeout: float, devices_per_person: float | None, ) -> None: - if not buildings: - raise ValueError("config: buildings must not be empty") - names = [b.name for b in buildings] - if len(set(names)) != len(names): - raise ValueError("config: building names must be unique") if default_timeout > poll_interval: # Otherwise a single unreachable AP holds a tick open past when the next one # should start, and the real poll rate drifts away from what phase 3's hysteresis diff --git a/devices/ap-collector/app/routers.py b/devices/ap-collector/app/routers.py new file mode 100644 index 00000000..1a66300a --- /dev/null +++ b/devices/ap-collector/app/routers.py @@ -0,0 +1,127 @@ +"""Which routers to poll: read from telemetry, joined with this site's own logins. + +Telemetry knows where each router is (building, room, ubus address). The site knows how to +log in to it. Neither half ever travels to the other side. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import TYPE_CHECKING, Any + +from app.config import AccessPoint, Building + +if TYPE_CHECKING: + from collections.abc import Callable + + from app.config import Site + +COLLECTOR_PATH = "/collector" +IWINFO_DRIVER = "openwrt-iwinfo" +_ROUTER_KEYS = {"sensorId", "roomId", "driver", "endpoint"} + + +class SyncError(RuntimeError): + """The router list could not be read -- telemetry unreachable, refusing, or malformed.""" + + +def sign_request(secret: bytes, building_id: str | None, timestamp: str) -> str: + """A GET has no body, so it signs this canonical string; the timestamp bounds replay.""" + canonical = f"GET {COLLECTOR_PATH}\n{building_id or ''}\n{timestamp}".encode() + return hmac.new(secret, canonical, hashlib.sha256).hexdigest() + + +def fetch_routers( + telemetry_url: str, + secret: bytes, + timeout: float, + building_id: str | None = None, + now: Callable[[], float] = time.time, +) -> dict[str, Any]: + """The routers `secret` may read: one building's with its key, every one with the shared key.""" + stamp = str(int(now())) + query = f"?{urllib.parse.urlencode({'buildingId': building_id})}" if building_id else "" + signature = sign_request(secret, building_id, stamp) + request = urllib.request.Request( # noqa: S310 + telemetry_url.rstrip("/") + COLLECTOR_PATH + query, + headers={"x-signature": signature, "x-timestamp": stamp}, + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 + answer = json.loads(response.read()) + except urllib.error.HTTPError as error: + raise SyncError(f"{request.full_url} returned {error.code}: {error.reason}") from error + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + raise SyncError(f"{request.full_url} unreadable: {error}") from error + try: + return parse_answer(answer) + except ValueError as error: + raise SyncError(f"{request.full_url} answered a malformed router list: {error}") from error + + +def parse_answer(answer: Any) -> dict[str, Any]: + """The answer, checked against the contract; raises ValueError on any other shape.""" + if not isinstance(answer, dict) or set(answer) != {"buildings"}: + raise ValueError("the answer must carry exactly buildings") + if not isinstance(answer["buildings"], list): + raise ValueError("buildings must be a list") + for building in answer["buildings"]: + if not isinstance(building, dict) or set(building) != {"buildingId", "routers"}: + raise ValueError("a building must carry exactly buildingId and routers") + if not _text(building["buildingId"]): + raise ValueError("buildingId must be a non-empty string") + routers = building["routers"] + if not isinstance(routers, list) or not routers: + raise ValueError(f"{building['buildingId']}: routers must be a non-empty list") + for router in routers: + if not isinstance(router, dict) or not set(router) <= _ROUTER_KEYS: + raise ValueError(f"a router may carry only {sorted(_ROUTER_KEYS)}") + if not all(_text(router.get(key)) for key in ("sensorId", "roomId")): + raise ValueError("a router's sensorId and roomId must be non-empty strings") + if not all(_text(router[key]) for key in ("driver", "endpoint") if key in router): + raise ValueError("a router's driver and endpoint, when sent, must be non-empty") + return answer + + +def buildings_from(answer: dict[str, Any], site: Site) -> list[Building]: + """One Building per answered building that still has a router the site can reach.""" + buildings: list[Building] = [] + for entry in answer["buildings"]: + aps = [ap for ap in (_access_point(r, site) for r in entry["routers"]) if ap] + if aps: + buildings.append(Building(name=entry["buildingId"], ap=aps)) + return buildings + + +def _access_point(router: dict[str, str], site: Site) -> AccessPoint | None: + sensor_id = router["sensorId"] + url = router.get("endpoint") + if url is None and site.endpoint_template: + url = site.endpoint_template.format(sensorId=sensor_id) + if url is None: + print(f"router {sensor_id}: no endpoint and no endpointTemplate, skipped", file=sys.stderr) + return None + login = site.login_for(sensor_id) + return AccessPoint.from_json( + { + "name": sensor_id, + "zone": router["roomId"], + "url": url, + "username": login.username, + "password": login.password, + "ifaces": login.ifaces, + "reader": "iwinfo" if router.get("driver") == IWINFO_DRIVER else "hostapd", + } + ) + + +def _text(value: object) -> bool: + return isinstance(value, str) and bool(value) diff --git a/devices/ap-collector/collector.dev.json b/devices/ap-collector/collector.dev.json index 2de1111c..8f219251 100644 --- a/devices/ap-collector/collector.dev.json +++ b/devices/ap-collector/collector.dev.json @@ -1,29 +1,10 @@ { "pollIntervalS": 5.0, "requestTimeoutS": 3.0, - "buildings": [ - { - "name": "b1", - "ap": [ - { - "name": "ap-a", - "zone": "zone-a", - "url": "http://ap-simulator:3000/ap-a/ubus", - "username": "collector", - "password": "collector", - "ifaces": ["wlan0"], - "reader": "hostapd" - }, - { - "name": "ap-b", - "zone": "zone-b", - "url": "http://ap-simulator:3000/ap-b/ubus", - "username": "collector", - "password": "collector", - "ifaces": ["wlan0"], - "reader": "iwinfo" - } - ] - } - ] + "syncIntervalS": 15, + "useDevicesPerPerson": true, + "devicesPerPerson": 1.4, + "ubus": { "username": "collector", "password": "collector" }, + "ifaces": ["wlan0"], + "endpointTemplate": "http://ap-simulator:3000/{sensorId}/ubus" } diff --git a/devices/ap-collector/collector.example.json b/devices/ap-collector/collector.example.json index a828b951..d108a000 100644 --- a/devices/ap-collector/collector.example.json +++ b/devices/ap-collector/collector.example.json @@ -1,31 +1,12 @@ { - "pollIntervalS": 5.0, - "requestTimeoutS": 3.0, - "useDevicesPerPerson": false, + "pollIntervalS": 60, + "requestTimeoutS": 10, + "syncIntervalS": 300, + "useDevicesPerPerson": true, "devicesPerPerson": 1.4, - "buildings": [ - { - "name": "b1", - "ap": [ - { - "name": "ap-a", - "zone": "zone-a", - "url": "http://localhost:3003/ap-a/ubus", - "username": "collector", - "password": "collector", - "ifaces": ["wlan0"], - "reader": "hostapd" - }, - { - "name": "ap-b", - "zone": "zone-b", - "url": "http://localhost:3003/ap-b/ubus", - "username": "collector", - "password": "collector", - "ifaces": ["wlan0"], - "reader": "iwinfo" - } - ] - } - ] + "ubus": { "username": "collector", "password": "change-me" }, + "ifaces": ["phy0-ap0", "phy1-ap0"], + "overrides": { + "e6f70819-2a3b-4c4d-8e5f-60718293a4b5": { "password": "this-router-differs", "ifaces": ["wlan0"] } + } } diff --git a/devices/ap-collector/docker-compose.dev.yml b/devices/ap-collector/docker-compose.dev.yml index fb836d22..14134b89 100644 --- a/devices/ap-collector/docker-compose.dev.yml +++ b/devices/ap-collector/docker-compose.dev.yml @@ -4,3 +4,5 @@ services: watch: - action: rebuild path: ./app + - action: rebuild + path: ./collector.dev.json diff --git a/devices/ap-collector/docker-compose.yml b/devices/ap-collector/docker-compose.yml index a160638f..b5001e36 100644 --- a/devices/ap-collector/docker-compose.yml +++ b/devices/ap-collector/docker-compose.yml @@ -6,3 +6,6 @@ services: environment: - TELEMETRY_SERVICE_URL=${TELEMETRY_SERVICE_URL:-http://gateway/telemetry} - TELEMETRY_SERVICE_SECRET=${TELEMETRY_INGEST_SECRET} + # The image defaults to --dry-run; the stack posts, so simulated routers reach the dashboard. + # 3 frozen polls (15 s), not the default 30: a stopped simulation must go quiet quickly. + command: ["--config", "collector.json", "--frozen-polls", "3"] diff --git a/devices/ap-collector/moon.yml b/devices/ap-collector/moon.yml index 75d4d913..a9fe5ca7 100644 --- a/devices/ap-collector/moon.yml +++ b/devices/ap-collector/moon.yml @@ -11,6 +11,8 @@ tasks: - 'app/**/*' - 'tests/**/*' - 'pyproject.toml' + # Contract tests read the shared fixtures; without these a fixture edit replays a cached pass. + - '/schemas/fixtures/**/*' lint: script: 'uv run ruff check . && uv run ruff format --check . && uv run pyright' diff --git a/devices/ap-collector/tests/test_collector.py b/devices/ap-collector/tests/test_collector.py index bc02165c..a6087adc 100644 --- a/devices/ap-collector/tests/test_collector.py +++ b/devices/ap-collector/tests/test_collector.py @@ -14,6 +14,7 @@ poll_one, readings_for_building, run, + sync_state, tick, tick_building, ) @@ -217,10 +218,11 @@ def test_tick_building_first_sighting_is_not_a_transition(): sessions = {"ap-a": _FakeSession(result=[("aa:bb:cc:00:00:01", -60)])} tracker = ZoneTracker(polls=1, margin_db=0, absent_polls=1) - assignment, moves = tick_building(building, sessions, tracker) + assignment, moves, answered = tick_building(building, sessions, tracker) assert assignment == {"aa:bb:cc:00:00:01": "lobby"} assert moves == Counter() + assert answered == {"lobby"} def test_tick_building_confirms_move_across_two_ticks(): @@ -239,7 +241,7 @@ def test_tick_building_confirms_move_across_two_ticks(): tracker, ) - assignment, moves = tick_building( + assignment, moves, _ = tick_building( building, { "ap-lobby": _FakeSession(result=[]), @@ -257,12 +259,13 @@ def test_tick_building_freezes_when_an_ap_fails_this_tick(): tracker = ZoneTracker(polls=1, margin_db=0, absent_polls=1) tick_building(building, {"ap-a": _FakeSession(result=[("aa:bb:cc:00:00:01", -60)])}, tracker) - assignment, moves = tick_building( + assignment, moves, answered = tick_building( building, {"ap-a": _FakeSession(error=UbusError("down"))}, tracker ) assert assignment == {"aa:bb:cc:00:00:01": "lobby"} assert moves == Counter() + assert answered == set() def test_tick_processes_every_building_independently(): @@ -280,8 +283,8 @@ def test_tick_processes_every_building_independently(): results = tick(config, sessions_by_building, trackers_by_building) - assert results["b1"] == ({"aa:bb:cc:00:00:01": "lobby"}, Counter()) - assert results["b2"] == ({"aa:bb:cc:00:00:02": "lobby"}, Counter()) + assert results["b1"] == ({"aa:bb:cc:00:00:01": "lobby"}, Counter(), {"lobby"}) + assert results["b2"] == ({"aa:bb:cc:00:00:02": "lobby"}, Counter(), {"lobby"}) def test_build_sessions_creates_one_session_per_ap_grouped_by_building(): @@ -396,13 +399,9 @@ def sleep(seconds): def test_readings_for_building_counts_devices_per_zone(): - building = Building( - name="b1", - ap=[_ap(name="ap-a", zone="lobby"), _ap(name="ap-b", zone="hall")], - ) assignment = {"aa:bb:cc:00:00:01": "lobby", "aa:bb:cc:00:00:02": "lobby"} - readings = readings_for_building(building, assignment, now_ms=1_000) + readings = readings_for_building(assignment, {"lobby", "hall"}, now_ms=1_000) assert sorted(readings, key=lambda r: r["roomId"]) == [ {"type": "totalDeviceCount", "roomId": "hall", "timestamp": 1_000, "totalDeviceCount": 0}, @@ -410,25 +409,38 @@ def test_readings_for_building_counts_devices_per_zone(): ] -def test_readings_for_building_reports_zero_not_absence_for_an_empty_zone(): - """A declared zone with nobody in it still gets a reading -- 0 is real data (the - room is empty), not the same fact as the zone being missing entirely.""" - building = Building(name="b1", ap=[_ap(name="ap-a", zone="lobby")]) - - readings = readings_for_building(building, assignment={}, now_ms=1_000) +def test_readings_for_building_reports_zero_for_a_room_that_answered_empty(): + """A room whose router answered with nobody in it still gets a reading -- 0 is real + data (the room is empty), not the same fact as the room being missing entirely.""" + readings = readings_for_building({}, {"lobby"}, now_ms=1_000) assert readings == [ {"type": "totalDeviceCount", "roomId": "lobby", "timestamp": 1_000, "totalDeviceCount": 0} ] +def test_a_room_nobody_measured_and_nobody_is_held_in_gets_no_reading(): + """Its router did not answer, so zero would claim an empty room nobody looked at -- + and would keep a dead router's room reading a fresh 0 forever.""" + assert readings_for_building({}, set(), now_ms=1_000, devices_per_person=2.0) == [] + + +def test_a_silent_room_still_reports_the_devices_held_in_it(): + """The tracker freezes a silent router's devices to ride out a reboot; while it holds + them the room keeps its count, and goes quiet only once they are dropped.""" + readings = readings_for_building({"aa:bb:cc:00:00:01": "lobby"}, set(), now_ms=1_000) + + assert readings == [ + {"type": "totalDeviceCount", "roomId": "lobby", "timestamp": 1_000, "totalDeviceCount": 1} + ] + + def test_readings_for_building_emits_no_estimate_without_a_factor(): """No configured factor, no ratioDeviceCount: an estimate silently equal to the device count is a claim about people that nobody made.""" - building = Building(name="b1", ap=[_ap(name="ap-a", zone="lobby")]) assignment = {"aa:bb:cc:00:00:01": "lobby", "aa:bb:cc:00:00:02": "lobby"} - readings = readings_for_building(building, assignment, now_ms=1_000, devices_per_person=None) + readings = readings_for_building(assignment, {"lobby"}, now_ms=1_000, devices_per_person=None) assert readings == [ {"type": "totalDeviceCount", "roomId": "lobby", "timestamp": 1_000, "totalDeviceCount": 2} @@ -439,14 +451,13 @@ def test_readings_for_building_emits_measurement_and_estimate_side_by_side(): """Both numbers, every tick. The raw count is what was measured; the ratio is what it was divided by a factor that gets re-measured -- keeping both is what lets a corrected factor re-derive the history instead of leaving every past bucket wrong.""" - building = Building(name="b1", ap=[_ap(name="ap-a", zone="lobby")]) assignment = { "aa:bb:cc:00:00:01": "lobby", "aa:bb:cc:00:00:02": "lobby", "aa:bb:cc:00:00:03": "lobby", } - readings = readings_for_building(building, assignment, now_ms=1_000, devices_per_person=1.4) + readings = readings_for_building(assignment, {"lobby"}, now_ms=1_000, devices_per_person=1.4) # 3 / 1.4 = 2.14... -> 3 assert readings == [ @@ -458,10 +469,8 @@ def test_readings_for_building_emits_measurement_and_estimate_side_by_side(): def test_readings_for_building_never_converts_a_present_device_into_an_empty_room(): """One device under a factor of 2.5 is 0.4 of a person. Rounded, that is 0 -- an occupied room reported as empty. A zone with anyone in it must never read zero.""" - building = Building(name="b1", ap=[_ap(name="ap-a", zone="lobby")]) - readings = readings_for_building( - building, {"aa:bb:cc:00:00:01": "lobby"}, now_ms=1_000, devices_per_person=2.5 + {"aa:bb:cc:00:00:01": "lobby"}, {"lobby"}, now_ms=1_000, devices_per_person=2.5 ) assert readings == [ @@ -472,9 +481,7 @@ def test_readings_for_building_never_converts_a_present_device_into_an_empty_roo def test_readings_for_building_keeps_an_empty_zone_at_zero_under_conversion(): """The floor is the only thing that moves: an empty room stays empty in both metrics.""" - building = Building(name="b1", ap=[_ap(name="ap-a", zone="lobby")]) - - readings = readings_for_building(building, {}, now_ms=1_000, devices_per_person=2.5) + readings = readings_for_building({}, {"lobby"}, now_ms=1_000, devices_per_person=2.5) assert readings == [ {"type": "totalDeviceCount", "roomId": "lobby", "timestamp": 1_000, "totalDeviceCount": 0}, @@ -496,9 +503,56 @@ def test_the_readings_match_the_shape_telemetry_pins(): timestamp = expected[0]["timestamp"] zone = expected[0]["roomId"] - building = Building(name="b1", ap=[_ap(name="ap-a", zone=zone)]) assignment = {f"aa:bb:cc:00:00:{n:02x}": zone for n in range(47)} - readings = readings_for_building(building, assignment, now_ms=timestamp, devices_per_person=2.0) + readings = readings_for_building(assignment, {zone}, now_ms=timestamp, devices_per_person=2.0) assert readings == expected + + +def _router(name: str, zone: str, url: str | None = None) -> AccessPoint: + return AccessPoint.from_json( + { + "name": name, + "zone": zone, + "url": url or f"http://{name}.example/ubus", + "username": "collector", + "password": "collector", + "ifaces": ["wlan0"], + } + ) + + +def _new_tracker() -> ZoneTracker: + return ZoneTracker(polls=3, margin_db=6.0, absent_polls=3, frozen_polls=30) + + +def test_a_new_router_list_keeps_what_stayed_and_adds_and_drops_the_rest(): + config = Config( + [ + Building("b1", [_router("r1", "lobby"), _router("r2", "lab")]), + Building("b2", [_router("r9", "hall")]), + ] + ) + sessions = build_sessions(config, timeout=3) + trackers = {"b1": _new_tracker(), "b2": _new_tracker()} + kept_session, kept_tracker = sessions["b1"]["r1"], trackers["b1"] + + config.buildings = [ + Building( + "b1", + [ + _router("r1", "lobby"), + _router("r2", "lab", url="http://moved.example/ubus"), + _router("r3", "hall"), + ], + ), + Building("b3", [_router("r7", "atrium")]), + ] + sync_state(config, sessions, trackers, timeout=3, new_tracker=_new_tracker) + + assert sessions["b1"]["r1"] is kept_session + assert sessions["b1"]["r2"].ap.url == "http://moved.example/ubus" + assert set(sessions["b1"]) == {"r1", "r2", "r3"} + assert trackers["b1"] is kept_tracker + assert set(sessions) == set(trackers) == {"b1", "b3"} diff --git a/devices/ap-collector/tests/test_config.py b/devices/ap-collector/tests/test_config.py index a07fad8e..5c9dcfb0 100644 --- a/devices/ap-collector/tests/test_config.py +++ b/devices/ap-collector/tests/test_config.py @@ -6,12 +6,16 @@ DEFAULT_DEVICES_PER_PERSON, DEFAULT_POLL_INTERVAL, DEFAULT_REQUEST_TIMEOUT, + DEFAULT_SYNC_INTERVAL, READERS, AccessPoint, Building, Config, + RouterLogin, ) +SITE = {"ubus": {"username": "collector", "password": "collector"}} + GOOD_AP = { "name": "ap-a", "zone": "zone-a", @@ -105,66 +109,49 @@ def test_get_ap_finds_by_name_and_returns_none_for_unknown(): assert building.get_ap("nope") is None -def test_load_from_config_file_parses_multiple_buildings(tmp_path): - data = { - "buildings": [ - {"name": "b1", "ap": [GOOD_AP]}, - { - "name": "b2", - "ap": [ - dict(GOOD_AP, name="ap-c", zone="zone-c", url="http://localhost:3003/ap-c/ubus") - ], - }, - ] - } - path = _write(tmp_path, data) - +def test_the_site_login_ifaces_overrides_and_template_are_read(tmp_path): + data = dict( + SITE, + ifaces=["wlan0", "wlan1"], + overrides={"r1": {"password": "other", "ifaces": ["phy0-ap0"]}}, + endpointTemplate="http://ap-simulator:3000/{sensorId}/ubus", + syncIntervalS=120, + ) config = Config([]) - config.load_from_config_file(path) - - assert [b.name for b in config.buildings] == ["b1", "b2"] - assert config.buildings[0].ap[0].name == "ap-a" - assert config.buildings[1].ap[0].name == "ap-c" - + config.load_from_config_file(_write(tmp_path, data)) -def test_load_from_config_file_empty_buildings_raises(tmp_path): - path = _write(tmp_path, {"buildings": []}) - - with pytest.raises(ValueError, match=r"(?i)buildings"): - Config([]).load_from_config_file(path) - - -def test_load_from_config_file_missing_buildings_key_raises(tmp_path): - path = _write(tmp_path, {}) - - with pytest.raises(ValueError, match=r"(?i)buildings"): - Config([]).load_from_config_file(path) + assert config.site.login == RouterLogin("collector", "collector", ("wlan0", "wlan1")) + assert config.site.login_for("r1") == RouterLogin("collector", "other", ("phy0-ap0",)) + assert config.site.login_for("r2") == config.site.login + assert config.site.endpoint_template == "http://ap-simulator:3000/{sensorId}/ubus" + assert config.sync_interval == 120 -def test_load_from_config_file_duplicate_building_names_raise(tmp_path): - data = {"buildings": [{"name": "b1", "ap": [GOOD_AP]}, {"name": "b1", "ap": [GOOD_AP]}]} - path = _write(tmp_path, data) +def test_the_site_defaults_to_wlan0_no_template_and_a_five_minute_sync(tmp_path): + config = Config([]) + config.load_from_config_file(_write(tmp_path, SITE)) - with pytest.raises(ValueError, match=r"(?i)building names"): - Config([]).load_from_config_file(path) + assert config.site.login.ifaces == ("wlan0",) + assert config.site.endpoint_template is None + assert config.sync_interval == DEFAULT_SYNC_INTERVAL -def test_a_failed_reload_does_not_clobber_the_previous_buildings(tmp_path): - original = [Building.from_json({"name": "b1", "ap": [GOOD_AP]})] - config = Config(original) - bad_path = _write(tmp_path, {"buildings": []}) +def test_a_missing_router_login_raises(tmp_path): + with pytest.raises(ValueError, match=r"(?i)ubus"): + Config([]).load_from_config_file(_write(tmp_path, {})) - with pytest.raises(ValueError): - config.load_from_config_file(bad_path) - assert config.buildings is original +def test_a_file_that_still_lists_buildings_is_refused(tmp_path): + """Routers come from telemetry now; a stale hand-written list must not be silently ignored.""" + with pytest.raises(ValueError, match=r"(?i)buildings"): + Config([]).load_from_config_file(_write(tmp_path, dict(SITE, buildings=[]))) def test_poll_interval_and_timeout_are_read_from_json(tmp_path): data = { "pollIntervalS": 10, "requestTimeoutS": 4, - "buildings": [{"name": "b1", "ap": [GOOD_AP]}], + **SITE, } path = _write(tmp_path, data) @@ -178,7 +165,7 @@ def test_poll_interval_and_timeout_are_read_from_json(tmp_path): def test_poll_interval_and_timeout_default_when_omitted_from_json(tmp_path): """Missing from the file means the module DEFAULT_*, not whatever the instance already happened to hold -- a reload must not leak a previous load's values in.""" - data = {"buildings": [{"name": "b1", "ap": [GOOD_AP]}]} + data = {**SITE} path = _write(tmp_path, data) config = Config([], poll_interval=999, default_timeout=999) @@ -192,7 +179,7 @@ def test_request_timeout_greater_than_poll_interval_raises(tmp_path): data = { "pollIntervalS": 2, "requestTimeoutS": 5, - "buildings": [{"name": "b1", "ap": [GOOD_AP]}], + **SITE, } path = _write(tmp_path, data) @@ -205,7 +192,7 @@ def test_a_failed_timeout_validation_does_not_clobber_previous_settings(tmp_path bad_data = { "pollIntervalS": 2, "requestTimeoutS": 5, - "buildings": [{"name": "b2", "ap": [GOOD_AP]}], + **SITE, } bad_path = _write(tmp_path, bad_data) @@ -236,17 +223,29 @@ def test_load_env_raises_when_telemetry_service_url_is_unset(monkeypatch): config.load_env() -def test_load_env_raises_when_telemetry_service_secret_is_unset(monkeypatch): +def test_load_env_raises_when_there_is_neither_a_shared_key_nor_a_building_key(monkeypatch): monkeypatch.setenv("TELEMETRY_SERVICE_URL", "http://telemetry:8080") monkeypatch.delenv("TELEMETRY_SERVICE_SECRET", raising=False) config = Config([]) - with pytest.raises(ValueError, match=r"(?i)TELEMETRY_SERVICE_SECRET must be set"): + with pytest.raises(ValueError, match=r"(?i)keys"): config.load_env() +def test_building_keys_replace_the_shared_key_for_their_building(tmp_path, monkeypatch): + monkeypatch.setenv("TELEMETRY_SERVICE_URL", "http://telemetry:8080") + monkeypatch.delenv("TELEMETRY_SERVICE_SECRET", raising=False) + config = Config([]) + config.load_from_config_file(_write(tmp_path, dict(SITE, keys={"b1": "k" * 64}))) + + config.load_env() + + assert config.key_for("b1") == b"k" * 64 + assert config.key_for("b2") is None + + def test_devices_per_person_is_none_when_flag_is_false_or_omitted(tmp_path): - data = {"devicesPerPerson": 1.4, "buildings": [{"name": "b1", "ap": [GOOD_AP]}]} + data = {"devicesPerPerson": 1.4, **SITE} path = _write(tmp_path, data) config = Config([]) @@ -259,7 +258,7 @@ def test_devices_per_person_is_used_when_flag_is_true(tmp_path): data = { "useDevicesPerPerson": True, "devicesPerPerson": 1.4, - "buildings": [{"name": "b1", "ap": [GOOD_AP]}], + **SITE, } path = _write(tmp_path, data) @@ -270,7 +269,7 @@ def test_devices_per_person_is_used_when_flag_is_true(tmp_path): def test_devices_per_person_defaults_when_flag_true_but_ratio_omitted(tmp_path): - data = {"useDevicesPerPerson": True, "buildings": [{"name": "b1", "ap": [GOOD_AP]}]} + data = {"useDevicesPerPerson": True, **SITE} path = _write(tmp_path, data) config = Config([]) @@ -283,7 +282,7 @@ def test_devices_per_person_zero_or_negative_raises_when_enabled(tmp_path): data = { "useDevicesPerPerson": True, "devicesPerPerson": 0, - "buildings": [{"name": "b1", "ap": [GOOD_AP]}], + **SITE, } path = _write(tmp_path, data) diff --git a/devices/ap-collector/tests/test_main.py b/devices/ap-collector/tests/test_main.py index 70a9aed6..03f9dc9d 100644 --- a/devices/ap-collector/tests/test_main.py +++ b/devices/ap-collector/tests/test_main.py @@ -1,8 +1,11 @@ import email.message +import hashlib +import hmac import json import urllib.error +from pathlib import Path -from app.__main__ import main +from app.__main__ import _refreshing, main class _FakeResponse: @@ -33,11 +36,31 @@ def _fake_ubus_urlopen(request, timeout=None): return _FakeResponse(json.dumps(payload).encode()) -def _make_fake_urlopen(ingest_calls): - """Routes to the ubus fake for ubus JSON-RPC calls, records everything else as an - ingest POST -- lets one test drive the whole pipeline (poll -> tick -> post) for real.""" +def _collector_answer(building_names): + return { + "buildings": [ + { + "buildingId": name, + "routers": [ + { + "sensorId": "ap-a", + "roomId": "lobby", + "endpoint": f"http://{name}-ap-a.example/ubus", + } + ], + } + for name in building_names + ] + } + + +def _make_fake_urlopen(ingest_calls, building_names=("b1",)): + """Answers the router list, the ubus fake, and records every ingest POST -- lets one test + drive the whole pipeline (sync -> poll -> tick -> post) for real.""" def fake(request, timeout=None): + if request.get_method() == "GET": + return _FakeResponse(json.dumps(_collector_answer(building_names)).encode()) body = json.loads(request.data) if isinstance(body, dict) and body.get("method") == "call": return _fake_ubus_urlopen(request, timeout) @@ -47,37 +70,27 @@ def fake(request, timeout=None): return fake -def _write_config(tmp_path, building_names=("b1",), devices_per_person=None): +def _write_config(tmp_path, devices_per_person=None): data = { "pollIntervalS": 5, "requestTimeoutS": 3, "useDevicesPerPerson": devices_per_person is not None, "devicesPerPerson": devices_per_person, - "buildings": [ - { - "name": name, - "ap": [ - { - "name": "ap-a", - "zone": "lobby", - "url": f"http://{name}-ap-a.example/ubus", - "username": "collector", - "password": "collector", - "ifaces": ["wlan0"], - "reader": "hostapd", - } - ], - } - for name in building_names - ], + "ubus": {"username": "collector", "password": "collector"}, } path = tmp_path / "collector.json" path.write_text(json.dumps(data), encoding="utf-8") return str(path) +def _telemetry(monkeypatch): + monkeypatch.setenv("TELEMETRY_SERVICE_URL", "http://telemetry.example/telemetry") + monkeypatch.setenv("TELEMETRY_SERVICE_SECRET", "x" * 32) + + def test_main_dry_run_once_prints_one_tick_batch(tmp_path, monkeypatch, capsys): - monkeypatch.setattr("urllib.request.urlopen", _fake_ubus_urlopen) + monkeypatch.setattr("urllib.request.urlopen", _make_fake_urlopen([])) + _telemetry(monkeypatch) config_path = _write_config(tmp_path) exit_code = main(["--config", config_path, "--once", "--dry-run"]) @@ -92,8 +105,7 @@ def test_main_dry_run_once_prints_one_tick_batch(tmp_path, monkeypatch, capsys): def test_main_without_dry_run_posts_a_signed_occupancy_batch(tmp_path, monkeypatch): ingest_calls = [] monkeypatch.setattr("urllib.request.urlopen", _make_fake_urlopen(ingest_calls)) - monkeypatch.setenv("TELEMETRY_SERVICE_URL", "http://telemetry.example/telemetry") - monkeypatch.setenv("TELEMETRY_SERVICE_SECRET", "x" * 32) + _telemetry(monkeypatch) config_path = _write_config(tmp_path) exit_code = main(["--config", config_path, "--once"]) @@ -117,8 +129,7 @@ def test_main_without_dry_run_posts_a_signed_occupancy_batch(tmp_path, monkeypat def test_main_posts_both_metrics_when_a_conversion_factor_is_configured(tmp_path, monkeypatch): ingest_calls = [] monkeypatch.setattr("urllib.request.urlopen", _make_fake_urlopen(ingest_calls)) - monkeypatch.setenv("TELEMETRY_SERVICE_URL", "http://telemetry.example/telemetry") - monkeypatch.setenv("TELEMETRY_SERVICE_SECRET", "x" * 32) + _telemetry(monkeypatch) config_path = _write_config(tmp_path, devices_per_person=2.5) exit_code = main(["--config", config_path, "--once"]) @@ -139,6 +150,8 @@ def test_main_survives_a_rejected_batch_and_still_posts_every_other_building( posted = [] def fake(request, timeout=None): + if request.get_method() == "GET": + return _FakeResponse(json.dumps(_collector_answer(("b1", "b2"))).encode()) body = json.loads(request.data) if isinstance(body, dict) and body.get("method") == "call": return _fake_ubus_urlopen(request, timeout) @@ -150,9 +163,8 @@ def fake(request, timeout=None): return _FakeResponse(b"") monkeypatch.setattr("urllib.request.urlopen", fake) - monkeypatch.setenv("TELEMETRY_SERVICE_URL", "http://telemetry.example/telemetry") - monkeypatch.setenv("TELEMETRY_SERVICE_SECRET", "x" * 32) - config_path = _write_config(tmp_path, ("b1", "b2")) + _telemetry(monkeypatch) + config_path = _write_config(tmp_path) exit_code = main(["--config", config_path, "--once"]) @@ -161,6 +173,77 @@ def fake(request, timeout=None): assert "b1" in capsys.readouterr().err +def test_main_with_a_building_key_reads_and_posts_for_that_building_only(tmp_path, monkeypatch): + requests = [] + monkeypatch.setattr("urllib.request.urlopen", _make_fake_urlopen(requests, ("b1",))) + monkeypatch.setenv("TELEMETRY_SERVICE_URL", "http://telemetry.example/telemetry") + monkeypatch.delenv("TELEMETRY_SERVICE_SECRET", raising=False) + config = json.loads(Path(_write_config(tmp_path)).read_text()) + config["keys"] = {"b1": "k" * 64} + path = tmp_path / "keyed.json" + path.write_text(json.dumps(config)) + + exit_code = main(["--config", str(path), "--once"]) + + assert exit_code == 0 + (ingest,) = requests + assert ( + ingest.get_header("X-signature") + == hmac.new(b"k" * 64, ingest.data, hashlib.sha256).hexdigest() + ) + + +def test_main_posts_nothing_for_a_building_whose_routers_do_not_answer(tmp_path, monkeypatch): + """A stopped simulator, or a site whose routers are down: a batch of fresh zeros would + read as a measured empty building, so the collector stays silent instead.""" + ingest_calls = [] + + def fake(request, timeout=None): + if request.get_method() == "GET": + return _FakeResponse(json.dumps(_collector_answer(("b1",))).encode()) + if json.loads(request.data).get("method") == "call": + raise urllib.error.HTTPError( + request.full_url, 404, "Not Found", email.message.Message(), None + ) + ingest_calls.append(request) + return _FakeResponse(b"") + + monkeypatch.setattr("urllib.request.urlopen", fake) + _telemetry(monkeypatch) + + exit_code = main(["--config", _write_config(tmp_path), "--once"]) + + assert exit_code == 0 + assert ingest_calls == [] + + +def test_main_starts_with_no_building_when_telemetry_is_unreachable(tmp_path, monkeypatch, capsys): + """A collector booting before the platform must keep running and pick routers up later.""" + + def down(request, timeout=None): + raise urllib.error.URLError("refused") + + monkeypatch.setattr("urllib.request.urlopen", down) + _telemetry(monkeypatch) + + exit_code = main(["--config", _write_config(tmp_path), "--once"]) + + assert exit_code == 0 + assert "refused" in capsys.readouterr().err + + +def test_the_router_list_is_refreshed_every_sync_interval(): + calls = [] + on_tick = _refreshing( + lambda results: calls.append("tick"), lambda: calls.append("sync"), every=2 + ) + + for _ in range(4): + on_tick({}) + + assert calls == ["tick", "tick", "sync", "tick", "tick", "sync"] + + def test_main_replay_is_not_implemented_yet(tmp_path, capsys): exit_code = main(["--config", "unused.json", "--replay", "survey.csv"]) diff --git a/devices/ap-collector/tests/test_routers.py b/devices/ap-collector/tests/test_routers.py new file mode 100644 index 00000000..2f3fbdbe --- /dev/null +++ b/devices/ap-collector/tests/test_routers.py @@ -0,0 +1,118 @@ +import json +import urllib.error +from pathlib import Path + +import pytest + +from app.config import RouterLogin, Site +from app.routers import SyncError, buildings_from, fetch_routers, parse_answer, sign_request + +_FIXTURES = Path(__file__).resolve().parents[3] / "schemas" / "fixtures" +REQUEST = json.loads((_FIXTURES / "collector-request.json").read_text()) +ROUTERS = json.loads((_FIXTURES / "collector-routers.json").read_text()) +SITE = Site(RouterLogin("collector", "secret", ("wlan0",))) + + +def _answer(name: str) -> dict: + return next(c for c in ROUTERS["cases"] if c["name"] == name)["body"] + + +def test_requests_sign_as_the_fixture_pins(): + secret = REQUEST["secret"].encode() + for case in REQUEST["cases"]: + building = case["buildingId"] or None + signature = sign_request(secret, building, case["timestamp"]) + assert signature == case["signature"], case["name"] + + +@pytest.mark.parametrize("case", ROUTERS["cases"], ids=lambda c: c["name"]) +def test_every_answer_the_fixture_accepts_parses(case): + parse_answer(case["body"]) + + +@pytest.mark.parametrize("case", ROUTERS["rejected"], ids=lambda c: c["name"]) +def test_every_answer_the_fixture_rejects_is_refused(case): + with pytest.raises(ValueError): + parse_answer(case["body"]) + + +def test_each_router_becomes_an_ap_in_its_room_read_as_its_driver_says(): + (aula, iwinfo) = buildings_from( + _answer("two buildings, one router without driver or endpoint"), SITE + ) + ap = aula.ap[0] + assert (aula.name, ap.name, ap.zone, ap.url, ap.reader) == ( + "bldg-3f2b4c5d", + "e6f70819-2a3b-4c4d-8e5f-60718293a4b5", + "room-aula-magna", + "http://10.0.4.12/ubus", + "hostapd", + ) + assert (ap.username, ap.password, ap.ifaces) == ("collector", "secret", ("wlan0",)) + assert iwinfo.ap[0].reader == "iwinfo" + + +def test_a_router_without_an_endpoint_is_skipped_unless_the_site_has_a_template(): + answer = _answer("two buildings, one router without driver or endpoint") + assert [len(b.ap) for b in buildings_from(answer, SITE)] == [1, 1] + + templated = Site(SITE.login, endpoint_template="http://ap-simulator:3000/{sensorId}/ubus") + aula = buildings_from(answer, templated)[0] + assert aula.ap[1].url == "http://ap-simulator:3000/f7081920-3b4c-4d5e-9f60-718293a4b5c6/ubus" + + +def test_a_per_router_override_replaces_the_site_login(): + site = Site(SITE.login, overrides={"e6f70819-2a3b-4c4d-8e5f-60718293a4b5": {"password": "x"}}) + aula = buildings_from(_answer("two buildings, one router without driver or endpoint"), site)[0] + assert aula.ap[0].password == "x" + + +def test_a_building_left_with_no_reachable_router_is_dropped(): + answer = {"buildings": [{"buildingId": "b1", "routers": [{"sensorId": "r", "roomId": "a"}]}]} + assert buildings_from(answer, SITE) == [] + + +class _Response: + def __init__(self, body: bytes): + self._body = body + + def read(self): + return self._body + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def test_fetch_signs_a_timestamped_get_and_parses_the_answer(monkeypatch): + seen = [] + + def fake(request, timeout=None): + seen.append((request, timeout)) + return _Response(json.dumps(_answer("no building has a router")).encode()) + + monkeypatch.setattr("urllib.request.urlopen", fake) + answer = fetch_routers( + "http://t.example/telemetry", b"k" * 32, timeout=3, building_id="b 1", now=lambda: 1000.9 + ) + + (request, timeout) = seen[0] + assert answer == {"buildings": []} + assert (request.get_method(), request.full_url, timeout) == ( + "GET", + "http://t.example/telemetry/collector?buildingId=b+1", + 3, + ) + assert request.get_header("X-timestamp") == "1000" + assert request.get_header("X-signature") == sign_request(b"k" * 32, "b 1", "1000") + + +def test_fetch_reports_an_unreachable_or_refusing_telemetry_as_a_sync_error(monkeypatch): + def refuse(request, timeout=None): + raise urllib.error.URLError("refused") + + monkeypatch.setattr("urllib.request.urlopen", refuse) + with pytest.raises(SyncError): + fetch_routers("http://t.example/telemetry", b"k" * 32, timeout=3) diff --git a/documentation/architecture/auth-architecture.qd b/documentation/architecture/auth-architecture.qd index 7b5af97f..bba1750b 100644 --- a/documentation/architecture/auth-architecture.qd +++ b/documentation/architecture/auth-architecture.qd @@ -199,7 +199,7 @@ Downstream of either path, the receiving service does the same thing: decode `x- `/agent/*` is not gated by `require_gateway_auth` in the Caddyfile, and has no equivalent `AuthorizationPolicy` requirement at the Istio edge. `evals/run_evals.py`'s local-dev bypass token is HS256-signed with a secret claims-gateway never sees — it was never going to pass gateway verification, by design. `agent` checks for the mesh claims header first and falls back to its own eval-token verification (`app/auth.py`). Since the edge can't *inject* a verified `x-gateway-claims` on an ungated route, both edges **strip** any client-supplied one on `/agent` (agent decodes that header without a signature check, so an ungated caller could otherwise forge it); real user traffic reaches agent through `chat`, which forwards mesh-verified claims internally. .box {telemetry ingest is the other ungated route — and it is an exact path} type:{warning} - Devices and simulators carry no user JWT, so `/telemetry/ingest` is ungated at both edges: `handle` (not `handle_path …/*`) in the `Caddyfile`, and a no-principal `paths:` entry in `k8s/istio-request-authentication.yml`\. Both are **exact paths, kept in sync by hand** — an ingest sub-path such as `/telemetry/ingest/batch` falls through to the gated rule and returns `401` to every gateway. The request is HMAC-verified inside telemetry instead (`adapters/ingest_auth.rs`). `backend/acceptance/edge/` asserts the Caddy half; nothing exercises Istio locally, so the Kubernetes half stays manual. + Devices and simulators carry no user JWT, so `/telemetry/ingest` — and `/telemetry/collector`, where a collector reads its routers — is ungated at both edges: `handle` (not `handle_path …/*`) in the `Caddyfile`, and a no-principal `paths:` entry in `k8s/istio-request-authentication.yml`\. Both are **exact paths, kept in sync by hand** — an ingest sub-path such as `/telemetry/ingest/batch` falls through to the gated rule and returns `401` to every gateway. The request is HMAC-verified inside telemetry instead (`adapters/ingest_auth.rs`). `backend/acceptance/edge/` asserts the Caddy half; nothing exercises Istio locally, so the Kubernetes half stays manual. The internal HMAC-signed contract used for `claims-gateway` → `tenancy` and `provisioner` → `registry`/`tenancy` calls is the same convention across the whole Go control plane — see each service's `internal/api/internalauth.go` for the `X-Signature` implementation. diff --git a/documentation/architecture/data-flow.qd b/documentation/architecture/data-flow.qd index 547bac53..c32d143b 100644 --- a/documentation/architecture/data-flow.qd +++ b/documentation/architecture/data-flow.qd @@ -113,7 +113,7 @@ These calls are intentionally few. The design keeps the request path short and a ### Device Ingestion -`POST /ingest` on `telemetry` — no user token (a sensor gateway has none to present), so it is exempt from the gateway's auth gate and carries its own credential instead: `X-Signature`, an HMAC-SHA256 over the raw body keyed by `TELEMETRY_INGEST_SECRET` — the same `X-Signature` scheme as the control-plane hops above, but a separate key, since this one leaves the mesh and lives on customer premises. Unsigned or mis-signed requests are `401` before any work. The endpoint validates, persists and evaluates thresholds before answering `202 Accepted`, so a `202` means the reading is durable and queryable. +`POST /ingest` on `telemetry` — no user token (a sensor gateway has none to present), so it is exempt from the gateway's auth gate and carries its own credential instead: `X-Signature`, an HMAC-SHA256 over the raw body keyed by the building's device key (derived from `TELEMETRY_DEVICE_MASTER_KEY`; dev's shared `TELEMETRY_INGEST_SECRET` also signs) — the same `X-Signature` scheme as the control-plane hops above, but a separate key, since this one leaves the mesh and lives on customer premises. Unsigned or mis-signed requests are `401` before any work. The endpoint validates, persists and evaluates thresholds before answering `202 Accepted`, so a `202` means the reading is durable and queryable. --- diff --git a/documentation/architecture/deployment.qd b/documentation/architecture/deployment.qd index ba8d80bc..c2fff58c 100644 --- a/documentation/architecture/deployment.qd +++ b/documentation/architecture/deployment.qd @@ -103,7 +103,7 @@ Kubernetes ingress is Istio's Gateway API. `k8s/istio-gateway.yml` defines one ` **Per-service routes.** Each API route gets a `URLRewrite`/`ReplacePrefixMatch` filter that strips the service prefix before forwarding — so `/twin/buildings` reaches `digital-twin` as `/buildings`\. `socket`'s route and the client's `/` catch-all forward the path unrewritten; `/grafana` is forwarded intact to Grafana, which owns that subpath. Gateway API matches by path specificity, so the client catch-all is evaluated last regardless of file order. The routes cover every service including `claims-gateway` (`/gateway`) and `tenancy` (`/tenancy`). -**Edge security.** The Gateway runs with Istio ambient enrollment underneath it — `ztunnel` L4 mTLS, `PeerAuthentication: STRICT`, and edge `RequestAuthentication` verifying the gateway JWT once and injecting the validated claims as `x-gateway-claims`\. An `AuthorizationPolicy` requires a verified JWT principal on the protected service paths (`/twin`, `/telemetry`, `/notification`, `/chat`, `/socket.io`, `/tenancy`, `/contracts`); `/gateway`, `/agent`, `/telemetry/ingest`, `/grafana`, and the client SPA are deliberately open. `docker-compose` + Caddy's `forward_auth` mirrors this exactly for the dev loop. See [Service Mesh Architecture](service-mesh.qd) for the full migration and [Kubernetes Configuration](../config/kubernetes.qd#istio-ambient-mesh) for the setup commands and the real gotchas hit standing it up (CNI bin dir, flannel IP exhaustion, Git Bash path mangling). +**Edge security.** The Gateway runs with Istio ambient enrollment underneath it — `ztunnel` L4 mTLS, `PeerAuthentication: STRICT`, and edge `RequestAuthentication` verifying the gateway JWT once and injecting the validated claims as `x-gateway-claims`\. An `AuthorizationPolicy` requires a verified JWT principal on the protected service paths (`/twin`, `/telemetry`, `/notification`, `/chat`, `/socket.io`, `/tenancy`, `/contracts`); `/gateway`, `/agent`, `/telemetry/ingest`, `/telemetry/collector`, `/grafana`, and the client SPA are deliberately open. `docker-compose` + Caddy's `forward_auth` mirrors this exactly for the dev loop. See [Service Mesh Architecture](service-mesh.qd) for the full migration and [Kubernetes Configuration](../config/kubernetes.qd#istio-ambient-mesh) for the setup commands and the real gotchas hit standing it up (CNI bin dir, flannel IP exhaustion, Git Bash path mangling). --- diff --git a/documentation/architecture/frontend-architecture.qd b/documentation/architecture/frontend-architecture.qd index 19f7c1d1..d287008d 100644 --- a/documentation/architecture/frontend-architecture.qd +++ b/documentation/architecture/frontend-architecture.qd @@ -131,6 +131,7 @@ Colours are **Material 3 roles**, declared once in `src/assets/main.css`: ## Dashboard - **Simulator toggle** (`SimulatorToggle`, `composables/dashboard/simulator.ts`): `PUT` / `DELETE` / `GET /telemetry/simulation/buildings/{id}`, 10 s timeout. It sends no rooms and no sensors — telemetry reads the building's own and picks the simulators — and the browser never calls a simulator. +- **Connected devices card** (`useConnectedDevices`, `utils/dashboard/connectedDevices.ts`): `GET /telemetry/connected-devices/buildings/{id}`. Telemetry sums the rooms; the browser never adds them — a device-count report on the socket only triggers a re-read. `404` shows "No reading yet", not 0; a report older than 5 min is flagged stale. - A refused start (`404` no simulator configured, `502` a simulator down) throws, and the toggle re-reads the status instead of trusting its optimistic flip. - `e2e/simulator.spec.ts` checks the start/stop calls, that nothing reaches `/simulator/`, and that a refused start leaves the toggle on Start. diff --git a/documentation/config/sensor-simulator.qd b/documentation/config/sensor-simulator.qd index 71e99619..bdfdff5e 100644 --- a/documentation/config/sensor-simulator.qd +++ b/documentation/config/sensor-simulator.qd @@ -1,14 +1,13 @@ .docname {External Simulators} .include {docs} -CrowdVision ships two simulators that exist **outside the main application cluster**. They are not microservices — they are test stand-ins for systems that in production would be real external hardware or third-party services. +CrowdVision ships three simulators that exist **outside the main application cluster**. They are not microservices — they are test stand-ins for systems that in production would be real external hardware or third-party services. | Simulator | Location | Simulates | |---|---|---| | **Sensor Simulator** | `simulators/sensor-simulator/` | Thermostats (`temperature`) and people counters (`peopleCount`) | | **Air Quality Simulator** | `simulators/aq-simulator/` | Air-quality stations (`airQuality`) | - -Routers have no simulator: `ap-simulator` fakes access points for `devices/ap-collector`, which polls them itself and is not started from the dashboard. +| **AP Simulator** | `simulators/ap-simulator/` | Routers (`router`), as OpenWrt access points `devices/ap-collector` polls | --- @@ -33,7 +32,7 @@ Routers have no simulator: `ap-simulator` fakes access points for `devices/ap-co | `temperature` | sensor-simulator | one `temperature`, in the sensor's room | | `peopleCount` | sensor-simulator | one `peopleCount`, in the sensor's room | | `airQuality` | aq-simulator | one `airQuality`, in the sensor's room, with its own drifting state | -| `router` | — | nothing | +| `router` | ap-simulator | none: serves `(mac, rssi)` station lists over ubus; ap-collector turns them into `totalDeviceCount` / `ratioDeviceCount` | - **Outdoor sensors** (no room) are never simulated: readings are room-keyed. - **Two sensors of one kind in one room** give two readings for that room each tick. @@ -54,6 +53,8 @@ Routers have no simulator: `ap-simulator` fakes access points for `devices/ap-co participant T as telemetry participant S as sensor-simulator participant A as aq-simulator + participant P as ap-simulator + participant C as ap-collector UI->>T: PUT /telemetry/simulation/buildings/b1 T->>T: read b1's sensors, split by kind @@ -61,12 +62,17 @@ Routers have no simulator: `ap-simulator` fakes access points for `devices/ap-co T->>S: POST /control/start {b1, [thermostats, people counters]} and T->>A: POST /control/start {b1, [air-quality stations]} + and + T->>P: POST /control/start {b1, [routers]} end T-->>UI: 204 loop Every 10 seconds S->>T: POST TELEMETRY_INGEST_URL (signed batch) A->>T: POST TELEMETRY_INGEST_URL (signed batch) + C->>T: GET /telemetry/collector (routers, every syncIntervalS) + C->>P: POST //ubus (station list per AP) + C->>T: POST TELEMETRY_INGEST_URL (signed batch) end **Stop** (`DELETE`) tells every simulator to drop the building. **Status** (`GET`) is `running` when any simulator runs it. The browser never calls a simulator. @@ -75,22 +81,51 @@ Routers have no simulator: `ap-simulator` fakes access points for `devices/ap-co ## Control API -Called by telemetry only. The start body is pinned by `schemas/fixtures/simulation-start.json`, which both simulators' tests assert. +Called by telemetry only. The start body is pinned by `schemas/fixtures/simulation-start.json`, which every simulator's tests assert. | Method | Endpoint | Body | Effect | |---|---|---|---| -| `POST` | `/control/start` | `{ buildingId, sensors: [{ sensorId, sensorType, roomId }] }` | Replaces what the building simulates. Empty `sensors` stops it. Any other key (e.g. the old `targetUrl`) is refused. | +| `POST` | `/control/start` | `{ buildingId, sensors: [{ sensorId, sensorType, roomId }] }` | Replaces what the building simulates. Empty `sensors` stops it. Any other key (e.g. the old `targetUrl`, or a coordinate) is refused. | | `POST` | `/control/stop` | `{ buildingId }` | Stops the building. Stopping one it never simulated succeeds. | | `GET` | `/control/status?buildingId=X` | — | `{ isRunning }` | --- +## AP Simulator + +Fakes the routers, not their readings: the collector stays the only thing that turns stations into counts, so its zone logic runs on simulated data too. + +- **One AP per router**, served at `//ubus` (credentials `collector`/`collector`). A restart drops the building's sessions; the collector logs in again. +- **No coordinates**: a router reports `(mac, rssi)` and knows nothing of where it is. The simulator works from router ids and rooms only; position guessing, if any, happens downstream from the strengths. +- **Neighbours are invented**: the router rooms form a ring, shuffled and seeded by `buildingId`. +- **People**: `AP_SIM_PHONES_PER_ROOM` (4) phones per router room. Each visits 3–5 stops, 70 % of the time a neighbouring room, holds 2–15 min, and is in the building about 08:30–17:30 local time. Seeded by `buildingId`: a restart walks the same people. +- **Signal** for a phone in room R, standing somewhere between R's middle and the door to one neighbour: + +.tablebyrows {.headers} + - - Router in + - RSSI + - - R + - −45 → −57 dBm, weaker toward the door + - - the neighbour it stands near + - −80 → −65 dBm, stronger toward the door + - - another neighbour + - −82 dBm, at the edge of hearing + - - any other room + - not heard + +- Each router adds a fixed ±3 dB of its own, so two routers in one room still hear a phone differently. 2 dB noise per poll, −85 dBm cutoff. +- **ap-collector finds them through the platform**, like at a real site: it reads the router sensors from `GET /telemetry/collector`. Simulated routers have no `endpoint`, so the dev config (`collector.dev.json`) fills it with `endpointTemplate: http://ap-simulator:3000/{sensorId}/ubus`. +- **Stop**: ap-simulator drops the building, so its routers stop answering. The collector holds the last counts for 3 polls (15 s in the stack, `--frozen-polls 3`), then posts nothing for those rooms; the dashboard's connected-devices card turns stale after 5 min rather than reading 0. +- The preset topology (`AP_SIM_SCENARIO`) is no longer polled by any collector; `/debug/*` still inspects and drives it for hand tests. + +--- + ## Configuration | Variable | Default | Purpose | |---|---|---| | `TELEMETRY_INGEST_URL` | `http://gateway/telemetry/ingest` | Where readings go. Set by the simulator's own environment, never by a caller. | -| `TELEMETRY_INGEST_SECRET` | *(empty)* | HMAC key for `X-Signature`; must match telemetry's. | +| `TELEMETRY_INGEST_SECRET` | *(empty)* | Telemetry's shared key: signs for any building, which a simulator started for any building needs. Dev only. | --- diff --git a/documentation/config/setting-up.qd b/documentation/config/setting-up.qd index 43ac385f..269288ef 100644 --- a/documentation/config/setting-up.qd +++ b/documentation/config/setting-up.qd @@ -102,7 +102,8 @@ This chains ten small, idempotent POSIX-sh scripts (`scripts/env/*.sh`), each re - - `gateway-key.sh` - A stable RSA signing key for `claims-gateway`, written to `secrets/gateway-dev-key.pem` — **not** \.`env` (see below). - - `ingest-key.sh` - - `TELEMETRY_INGEST_SECRET` — the HMAC key a gateway signs `/telemetry/ingest` bodies with. Deliberately distinct from `INTERNAL_SIGNING_SECRET`: it leaves the mesh and lives on customer premises. + - `TELEMETRY_DEVICE_MASTER_KEY` — every building's device key is derived from it; a collector signs `/telemetry/ingest` and `/telemetry/collector` with its building's key. Losing it revokes them all. + - `TELEMETRY_INGEST_SECRET` — the shared key that signs for **any** building: simulators and dev only, leave it unset in production. Deliberately distinct from `INTERNAL_SIGNING_SECRET`: it leaves the mesh and lives on customer premises. - - `telemetry-db.sh` - `TELEMETRY_DB_PASSWORD` — backs `telemetry-db`. Read only by the Kubernetes path (`just k8s secrets`); Compose hardcodes `telemetry` in `backend/telemetry/docker-compose.yml`. diff --git a/documentation/config/toolchain.qd b/documentation/config/toolchain.qd index 2b50dddd..77ba7c80 100644 --- a/documentation/config/toolchain.qd +++ b/documentation/config/toolchain.qd @@ -187,7 +187,7 @@ Because the pipeline is inherited, a standard service's entire moon configuratio - - `sensor-simulator`, `aq-simulator` - Each asserts `schemas/fixtures/ingest-batch.json`: they hand-build the batch telemetry hand-parses, so the fixture is the only thing holding the three languages to one shape. The Python one overrides `test`/`lint` to call `uv` directly, because the inherited tasks shell out to `npm` and the root `.gitignore` ignores every `package.json`. - - `ap-simulator` - - Serves a fake ubus endpoint rather than producing telemetry, so it has no fixture to assert and excludes the inherited `test` task; its CI leg runs `smoke.py` instead. + - Serves a fake ubus endpoint rather than producing telemetry, so it asserts only `schemas/fixtures/simulation-start.json`. Overrides `test`/`lint` to call `uv` directly, like aq-simulator; its CI leg also runs `smoke.py`. - - `tooling`, `eslint-config` - Tagged `language: 'javascript'` so they inherit only `deps`. (`acceptance` is `language: 'python'` and excludes the inherited `test`/`lint` tasks: its pytest suite runs via `just test integration` against a composed stack, never as a moon `:test`.) - - `claims-gateway`, `tenancy`, `registry`, `provisioner` diff --git a/documentation/contributing/cicd.qd b/documentation/contributing/cicd.qd index 6a06a052..f63cdda3 100644 --- a/documentation/contributing/cicd.qd +++ b/documentation/contributing/cicd.qd @@ -179,8 +179,11 @@ The two device simulators ship as GHCR images but are not application services, - - `aq-simulator` - Python 3.12 - `ruff check` (default rules: undefined names, unused imports), `pyright`, `uv audit`, then `pytest` — which asserts the ingest-batch and simulation-start fixtures. + - - `ap-simulator` + - Python 3.12 + - `ruff check`, `pyright`, `uv audit`, `pytest` — which asserts the simulation-start fixture — then `smoke.py`. -Both were previously ungated despite shipping images until this workflow gave them a real bucket. +sensor-simulator and aq-simulator were previously ungated despite shipping images until this workflow gave them a real bucket. --- diff --git a/documentation/design/wireless-flow.qd b/documentation/design/wireless-flow.qd index 19b8542b..aae4a780 100644 --- a/documentation/design/wireless-flow.qd +++ b/documentation/design/wireless-flow.qd @@ -144,6 +144,7 @@ calibration — and zone granularity is exactly what the twin renders. AP1 -->|RSSI per station| C AP2 --> C AP3 --> C + T2[GET /telemetry/collector] -->|routers: sensor id, room, ubus address| C C -->|aggregate counts, HMAC-signed| T[POST /telemetry/ingest] T --> RD{{Redis}} T --> PG[(Timescale)] @@ -153,6 +154,12 @@ calibration — and zone granularity is exactly what the twin renders. A small daemon on the building network. Polls each AP every few seconds, holds one in-memory table, emits one signed batch per tick. +- **Which APs**: the router sensors placed in the twin, read from `GET /telemetry/collector` at start and every `syncIntervalS` (300 s). A router that stays keeps its session and every device's hysteresis; telemetry unreachable keeps the previous list. +- **How to reach them**: the sensor's `endpoint` is the ubus URL (`endpointTemplate` fills a missing one); `driver` `openwrt-iwinfo` selects the iwinfo reader, anything else hostapd. +- **What it reports**: a room gets a reading when a router in it answered (0 is then a real empty room) or devices are still held in it. A room whose routers are all silent keeps its held count for `--frozen-polls` polls (30; a reboot is a couple), then gets **no reading** — never a fresh 0 nobody measured. With every room silent, nothing is posted. +- **How to log in**: the collector's own file — one site-wide `ubus` login and `ifaces`, per-router `overrides`. Never sent to the platform. +- **How it proves which building it is**: `keys: { "": "" }` in the same file, issued by an editor (`POST /telemetry/device-keys/buildings/{id}`). Each key reads that building's routers and signs its batches; it cannot sign for another building. + On OpenWrt the readings come from hostapd over `ubus` — `hostapd. get_clients` returns every associated station with its `signal` in dBm. `iw dev station dump` is the equivalent without ubus. @@ -208,6 +215,11 @@ Both follow the shape of the existing `peopleCount` plugin (`backend/telemetry/src/plugins/people_count.rs`): per building and room, non-negative integer, dashboard catalog, twin overlay, agent tool. +**A room's count is its router's, not the room's.** A router hears devices wherever they +stand, across walls and floors, so the reading keyed to its room says "near this router". +The figure that means what it says is the building's: telemetry sums the rooms at the +building's newest report and serves it at `GET /telemetry/connected-devices/buildings/{id}`. + **One device, two metrics.** The access point is what gets registered as a sensor — device kind `router` in telemetry's `plugins::devices()`, which reports both. Registering it once per metric would count one device twice. @@ -374,7 +386,8 @@ recommendations. | **One SSID across all APs** | Per-SSID randomization means a second SSID is a second identity for the same phone. | | **One SSID across 2.4 and 5 GHz** | Split-band SSIDs are two networks; use band steering. Standard practice regardless. | | **WPA2-AES or WPA3** | On open networks Apple rotates the private address every 2 weeks. Encryption pins it. | -| **Known AP → zone mapping** | The collector needs it. A flat config file; no coordinate survey. | +| **Every router placed in the twin** | A router sensor in its room *is* the AP → zone mapping; the collector reads it from `GET /telemetry/collector`. No coordinate survey. | +| **One read-only ubus login on the routers** | The collector's own file holds it (per-router `overrides` where one differs). Logins never reach the platform. | --- diff --git a/documentation/packages/telemetry-schema.qd b/documentation/packages/telemetry-schema.qd index 2064bafb..929f1df5 100644 --- a/documentation/packages/telemetry-schema.qd +++ b/documentation/packages/telemetry-schema.qd @@ -90,12 +90,15 @@ Two telemetry shapes have no type here: one side walks raw JSON or is the only R | Fixture | Direction | Built by | Parsed by | |---|---|---|---| | `ingest-batch.json` | device → telemetry `/ingest` | ap-collector, aq-simulator, sensor-simulator | telemetry | -| `simulation-start.json` | telemetry → simulator `/control/start` | telemetry | sensor-simulator, aq-simulator | +| `simulation-start.json` | telemetry → simulator `/control/start` | telemetry | sensor-simulator, aq-simulator, ap-simulator | +| `collector-routers.json` | telemetry → collector, `GET /collector` | telemetry | ap-collector | +| `connected-devices.json` | telemetry → browser, `GET /connected-devices/buildings/{id}` | telemetry | frontend | `simulation-start` body: `{ buildingId, sensors: [{ sensorId, sensorType, roomId }] }`. - `sensors` replaces what the simulator held for the building; empty = stop simulating it. - `roomId` is always a room: readings are room-keyed, so outdoor sensors are never sent. +- No coordinates: positions are digital-twin's. ap-simulator models signal overlap from rooms alone. - No `targetUrl`: a simulator posts to its own `TELEMETRY_INGEST_URL`, so no caller can redirect its signed readings. The schema refuses extra keys, so the old body fails. --- diff --git a/documentation/services/telemetry.qd b/documentation/services/telemetry.qd index 354aad99..11e0a218 100644 --- a/documentation/services/telemetry.qd +++ b/documentation/services/telemetry.qd @@ -112,7 +112,7 @@ A bound names the payload `field` it compares — not the reading's value, since **Ingestion** — `POST /ingest`, device-facing so no user JWT and still exempt from the edge gate, but not open: the caller signs the request itself. The route takes a *batch*, always — a lone device sends one reading in the array, so there is no second route and no second code path. -0. `X-Signature` must equal `HMAC-SHA256(TELEMETRY_INGEST_SECRET, raw_body)` in lowercase hex — missing, malformed or mismatched is `401` before any parsing, persistence or fanout. Bodies over 1 MiB are `413`. +0. `X-Signature` must equal `HMAC-SHA256(key, raw_body)` in lowercase hex, `key` being the body's `buildingId` device key (or the dev shared key — see **Device keys**) — missing, malformed or mismatched is `401` before any parsing, persistence or fanout. Bodies over 1 MiB are `413`. 1. Envelope `{buildingId, readings[]}`, capped at 500 readings. Each reading carries its own `type`, which selects the plugin and is stripped; the envelope supplies `buildingId`. A reading naming a *different* building is rejected — a batch maps to exactly one fan-out channel. 2. Every reading is validated before anything is written. All-or-nothing: one failure persists nothing, publishes nothing, and one `422` names every offending field, each reading's prefixed with its index (`readings[3]: ...`). An unknown metric is one of those errors, not a `404` — within a batch it is a bad element of a payload, not an unroutable request. 3. One bulk insert, with one threshold lookup for the whole tick running alongside it. @@ -129,13 +129,34 @@ The signature covers the exact bytes on the wire, so a re-serialised body fails: There is deliberately no `/ingest/batch` alongside a single-reading `/ingest`. Two routes meant two kernel entry points, two store methods and two fan-out methods to keep in step — and a sub-path the edge does not ungate, since `Caddyfile` and the Istio policy exempt the exact path `/telemetry/ingest`. One route with N ≥ 1 removes both problems. -.box {One shared secret, not one per building} type:{note} - Every gateway currently signs with the same key, so a legitimate gateway could still post for a building that is not its own. That is bounded by there being no third-party gateways yet. Per-building keys are a store lookup behind the same header, with no wire change. Replay is likewise undefended — a captured body stays valid — which matches the mesh's documented "hard perimeter, guarded interior" posture, where an in-mesh attacker could forge `x-gateway-claims` regardless. +**Collector read** — `GET /collector`, the router list an on-site collector polls. Device-facing like ingest: exempt from the edge gate, signed by the caller. + +- `?buildingId=X`, `X-Timestamp` (unix seconds) and `X-Signature` = `HMAC-SHA256(X's device key, "GET /collector\n" + X + "\n" + timestamp)`. A GET has no body, so the timestamp is what is signed; more than 300 s off telemetry's clock is `401`, which bounds replay. Without `buildingId` only the shared key signs, and it reads every building. Pinned by `schemas/fixtures/collector-request.json`. +- Answers `{ buildings: [{ buildingId, routers: [{ sensorId, roomId, driver?, endpoint? }] }] }`: X's router sensors in rooms (or every building's, with the shared key). Outdoor routers and buildings without a router are left out. Pinned by `schemas/fixtures/collector-routers.json`. +- **Addresses only, never logins.** `endpoint` is the router's ubus URL on the site network; its credentials live in the collector's own file. + +**Device keys** — one per building, so a key taken from one site's collector signs for that building alone. + +- `key = hex(HMAC-SHA256(TELEMETRY_DEVICE_MASTER_KEY, "{buildingId}:{epoch}"))`. Derived, never stored: the database holds only `buildings.device_key_epoch`. +- `POST /device-keys/buildings/{buildingId}` (editing role) bumps the epoch and returns `{ buildingId, key }` — the only time the key is shown. The previous key stops working at once; other buildings are untouched. +- The epoch is read uncached on every signed request, so a rotation holds on every replica immediately. +- `TELEMETRY_INGEST_SECRET`, when set, is a **shared key** that signs for any building — dev and simulators only. Production leaves it unset. + +.box {The master key is the one secret} type:{warning} + Anyone holding `TELEMETRY_DEVICE_MASTER_KEY` can derive every building's key. Changing it revokes every device key at once, so every collector needs a new one. Replay of an ingest body is still undefended — a captured body stays valid — which matches the mesh's documented "hard perimeter, guarded interior" posture. A threshold-evaluation failure is logged and does **not** fail the ingest; a persistence failure does, and is counted by `telemetry_ingest_persist_failures_total`. **Reads** — `GET /{sensorType}/latest`, `/entireBuilding`, `/dashboard`. Each resolves the building's domains through digital-twin and evaluates Cedar before touching data. `latest` on an empty room is `404`. +**Connected devices** — `GET /connected-devices/buildings/{buildingId}` → `{ buildingId, totalDeviceCount, timestamp }`, pinned by `schemas/fixtures/connected-devices.json`. + +- A router counts the devices it hears wherever they stand, so a room's `totalDeviceCount` is a router's figure; the building's sum is the one that means what it says. +- Summed over the rooms at the building's **newest** report: a room whose last report is older — its router removed or moved — is left out. +- Each room counts once at that timestamp, so a report delivered twice does not double it. +- `timestamp` is that newest report; judging it stale is the reader's call. Never reported → `404`, not `0`. Requires `Read` on the building. +- Assumes one collector per building: two would stamp different timestamps, and the newest-report rule would drop one. + A read is served in the **same flat shape as the `telemetry` socket event**: envelope plus every metric field at the top level under its own name. The trimmed `payload` column is a storage detail — it is re-inflated on the way out and never appears in a response. A client that reads a REST snapshot and then live events must not have to handle two shapes; when it did, a dashboard column stayed blank until the first socket tick overwrote the row. **Sensors** — `POST /sensors/buildings/{id}` takes one building's `create` / `update` / `delete` lists as a batch, the same all-or-nothing rule as ingest and room thresholds. The editor saves many changes at once, and half a save would leave the twin disagreeing with what was placed. @@ -175,10 +196,12 @@ Positions are **not** stored here: they are physical layout and belong to digita - The simulators stay dumb: they simulate exactly what they are told. - - Every configured simulator is told, even with an empty list; empty = stop the building - A sensor removed in the twin must stop producing readings. - - - Routers (no simulator claims them) and outdoor sensors are never sent - - Nothing simulates a router; readings are room-keyed, so an outdoor reading has nowhere to go. + - - Kinds no simulator claims, and outdoor sensors, are never sent + - Readings are room-keyed, so an outdoor reading has nowhere to go. Compose gives routers to ap-simulator. - - Body pinned by `schemas/fixtures/simulation-start.json`; no `targetUrl` - - Two simulators in two languages parse it; a simulator posts to its own `TELEMETRY_INGEST_URL`. + - Three simulators in two languages parse it; a simulator posts to its own `TELEMETRY_INGEST_URL`. + - - No coordinates in the body + - Positions are digital-twin's bounded context; telemetry never holds or forwards them. - - Calls run concurrently, 5 s timeout each; all are tried, then any failure is `502` - One dead simulator must not leave the others stale. A retry is safe: start replaces. - - Status is `running` when any simulator runs the building; an unreachable one counts as not running diff --git a/frontend/e2e/connected-devices.spec.ts b/frontend/e2e/connected-devices.spec.ts new file mode 100644 index 00000000..f3c89e14 --- /dev/null +++ b/frontend/e2e/connected-devices.spec.ts @@ -0,0 +1,95 @@ +import { readFileSync } from 'node:fs' +import { expect, test } from '@playwright/test' +import type { Page, Route } from '@playwright/test' + +/** + * The dashboard's connected-devices card, with the backend stubbed: telemetry sums the + * building's devices, so the browser only shows what it answers and never adds rooms itself. + */ + +const DOMAIN = 'eng' +const BUILDING = 'bldg-e2e' + +const building = { + id: BUILDING, + name: 'Engineering Block', + domains: [DOMAIN], + rooms: [ + { + id: 'Room-101', + name: 'Room 101', + capacity: 20, + position: { x: 0, y: 2, z: 0 }, + dimensions: { width: 8, height: 4, depth: 6 }, + }, + ], +} + +const wire = JSON.parse( + readFileSync(new URL('../../schemas/fixtures/connected-devices.json', import.meta.url), 'utf8'), +) as { cases: { body: { totalDeviceCount: number; timestamp: number } }[] } + +/** Stubs the dashboard's backend; `connected` answers the building's devices, or 404 when null. */ +async function stubBackend(page: Page, connected: { status: number; body?: unknown }) { + const json = (route: Route, body: unknown) => route.fulfill({ json: body }) + + await page.route('**/gateway/me', (route) => json(route, { sub: 'u1', accountName: 'Tester' })) + await page.route('**/tenancy/me/memberships', (route) => + json(route, [{ domain: DOMAIN, role: 'business_admin' }]), + ) + await page.route(`**/twin/buildings/${DOMAIN}`, (route) => json(route, [building])) + await page.route('**/telemetry/contracts', (route) => json(route, { metrics: [] })) + await page.route('**/telemetry/*/dashboard*', (route) => json(route, { data: [] })) + await page.route('**/telemetry/*/entireBuilding*', (route) => json(route, { data: [] })) + await page.route(`**/telemetry/sensors/buildings/${BUILDING}`, (route) => + json(route, { data: [] }), + ) + await page.route(`**/telemetry/simulation/buildings/${BUILDING}`, (route) => + json(route, { running: false }), + ) + await page.route(`**/telemetry/connected-devices/buildings/${BUILDING}`, (route) => + route.fulfill({ status: connected.status, json: connected.body ?? { message: 'none' } }), + ) +} + +const card = (page: Page) => page.getByRole('article').filter({ hasText: 'Connected devices' }) + +async function openDashboard(page: Page) { + await page.goto('/dashboards') + const later = page.getByRole('dialog', { name: /alerts/i }).getByRole('button', { name: 'Later' }) + if (await later.isVisible().catch(() => false)) await later.click() +} + +test("shows the building's connected devices as telemetry sums them", async ({ page }) => { + const [answer] = wire.cases + await stubBackend(page, { + status: 200, + body: { ...answer!.body, buildingId: BUILDING, timestamp: Date.now() }, + }) + + await openDashboard(page) + + await expect(card(page)).toContainText(String(answer!.body.totalDeviceCount)) + await expect(card(page)).toContainText('Across every router') +}) + +test('says so when the building never reported, instead of showing zero', async ({ page }) => { + await stubBackend(page, { status: 404 }) + + await openDashboard(page) + + await expect(card(page)).toContainText('—') + await expect(card(page)).toContainText('No reading yet') +}) + +test('flags a figure the collector stopped refreshing', async ({ page }) => { + const [answer] = wire.cases + await stubBackend(page, { + status: 200, + body: { ...answer!.body, buildingId: BUILDING, timestamp: Date.now() - 10 * 60_000 }, + }) + + await openDashboard(page) + + await expect(card(page)).toContainText('No report for 5+ min') +}) diff --git a/frontend/src/components/dashboard/kpi/KpiStrip.vue b/frontend/src/components/dashboard/kpi/KpiStrip.vue index 97eb93a4..931e908d 100644 --- a/frontend/src/components/dashboard/kpi/KpiStrip.vue +++ b/frontend/src/components/dashboard/kpi/KpiStrip.vue @@ -2,12 +2,19 @@ import { computed } from 'vue' import { useI18n } from 'vue-i18n' import { aqiBand, type BuildingSummary } from '@/utils/dashboard/dashboard.ts' +import type { ConnectedDevices } from '@/utils/dashboard/connectedDevices.ts' import KpiCard from '@/components/dashboard/kpi/KpiCard.vue' import ProgressBar from '@/components/commons/base/ProgressBar.vue' import BaseBadge from '@/components/commons/base/BaseBadge.vue' import type { Tone } from '@/utils/commons/tone.ts' -const props = defineProps<{ summary: BuildingSummary; loading: boolean }>() +const props = defineProps<{ + summary: BuildingSummary + loading: boolean + devices: ConnectedDevices | null + devicesStale: boolean + devicesLoading: boolean +}>() const { t, n } = useI18n() @@ -18,10 +25,14 @@ const occupancy = computed(() => props.summary.occupancy === null ? EMPTY : n(props.summary.occupancy, 'percent'), ) const band = computed(() => aqiBand(props.summary.averageAqi)) +const devicesCaption = computed(() => { + if (props.devices === null) return t('dashboard.kpi.noReading') + return props.devicesStale ? t('dashboard.kpi.devicesStale') : t('dashboard.kpi.devicesCaption') +})