From d60f5f7f313e16cae587472746913d44b393bed2 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 13 Sep 2026 09:24:42 -0400 Subject: [PATCH 01/12] feat(admin): add protected account lifecycle endpoints --- README.md | 13 +- cmd/beacon/main.go | 2 + docs/docs.go | 358 +++++++++++++++++++++++++ docs/swagger.json | 358 +++++++++++++++++++++++++ docs/swagger.yaml | 236 ++++++++++++++++ internal/api/account_requests.go | 8 + internal/api/handlers/accounts.go | 173 ++++++++++++ internal/api/handlers/accounts_test.go | 147 ++++++++++ internal/api/router/accounts_test.go | 53 ++++ 9 files changed, 1346 insertions(+), 2 deletions(-) create mode 100644 internal/api/account_requests.go create mode 100644 internal/api/handlers/accounts.go create mode 100644 internal/api/handlers/accounts_test.go create mode 100644 internal/api/router/accounts_test.go diff --git a/README.md b/README.md index 2609e748..6767640b 100644 --- a/README.md +++ b/README.md @@ -133,8 +133,7 @@ broker workers, not connection status or a tunable processing-worker pool). The CORS lists are the options supplied to the middleware; its normal matching normalization still applies. The response excludes credential fields, broker addresses, channel material, database settings and -other configuration. Account operations are not implemented; unknown admin paths -return 404 and unsupported +other configuration. Unknown admin paths return 404 and unsupported methods on the config endpoint return 405 after authentication. Global CORS preflights remain public. Use a long, randomly generated key, keep it out of source control and logs, and send it only in the Authorization header, @@ -158,6 +157,16 @@ auth/credential fields and broker count cannot be changed here; there is no configurable `ingest.worker_count`. Cross-origin admin clients need PUT allowed in the saved CORS methods. CORS controls browser access, not authentication. +Operator accounts are available at `GET/POST /api/v1/admin/accounts` and +`GET/DELETE /api/v1/admin/accounts/{id}`. POST accepts a JSON `name` field in a +body up to 4 KiB; names are trimmed, case-sensitive and limited to 128 Unicode +characters without control characters. Active names are unique. DELETE soft +deactivates the record (204); missing IDs return 404 and an already inactive +record returns 409. A deactivated name may be reused by a new account. +Lists include active and inactive records, newest first, without pagination. +These are operator-defined records; no login, session or API token is created. +Cross-origin clients must have their methods allowed in the existing CORS config. + ### Environment variables (`.env`) | Variable | Default | Description | diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index b518ab1f..39f1c610 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -18,6 +18,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/db" _ "github.com/MeshCore-Beacon/beacon-server/docs" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/api/router" "github.com/MeshCore-Beacon/beacon-server/internal/background" "github.com/MeshCore-Beacon/beacon-server/internal/cache" @@ -307,6 +308,7 @@ func main() { MaxConnsPerIP: resolved.MaxConnsPerIP, MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, + AdminRoutes: map[string]http.Handler{"/accounts": handlers.AccountsRouter(store)}, }) srv := &http.Server{ diff --git a/docs/docs.go b/docs/docs.go index 7b096572..5973f96a 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -22,6 +22,322 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { + "/admin/accounts": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Includes active and inactive accounts, newest first. These records are not login users.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "List operator accounts", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "post": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Trims the name; names are case-sensitive and unique among active accounts. No credential or login is created.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Create an operator account", + "parameters": [ + { + "description": "Account name (1-128 Unicode characters, no control characters); body at most 4 KiB", + "name": "account", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/accounts/{id}": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns active or inactive account records.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Get an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "delete": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Soft deactivation preserves the record. Its name may be reused by a new account.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Deactivate an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "Deactivated" + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/config": { "get": { "security": [ @@ -2640,6 +2956,37 @@ const docTemplate = `{ } }, "definitions": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Account": { + "type": "object", + "properties": { + "active": { + "type": "boolean" + }, + "created_at": { + "type": "string" + }, + "deactivated_at": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig": { "type": "object", "properties": { @@ -2893,6 +3240,17 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 9803c083..52f700e1 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -20,6 +20,322 @@ "host": "localhost:8080", "basePath": "/api/v1", "paths": { + "/admin/accounts": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Includes active and inactive accounts, newest first. These records are not login users.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "List operator accounts", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "post": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Trims the name; names are case-sensitive and unique among active accounts. No credential or login is created.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Create an operator account", + "parameters": [ + { + "description": "Account name (1-128 Unicode characters, no control characters); body at most 4 KiB", + "name": "account", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/accounts/{id}": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns active or inactive account records.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Get an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "delete": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Soft deactivation preserves the record. Its name may be reused by a new account.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Deactivate an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "Deactivated" + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/config": { "get": { "security": [ @@ -2638,6 +2954,37 @@ } }, "definitions": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Account": { + "type": "object", + "properties": { + "active": { + "type": "boolean" + }, + "created_at": { + "type": "string" + }, + "deactivated_at": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig": { "type": "object", "properties": { @@ -2891,6 +3238,17 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f1d4293b..6ef3079a 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,5 +1,25 @@ basePath: /api/v1 definitions: + github_com_MeshCore-Beacon_beacon-server_internal_api.Account: + properties: + active: + type: boolean + created_at: + type: string + deactivated_at: + type: string + id: + type: string + name: + type: string + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList: + properties: + items: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account' + type: array + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig: properties: configured: @@ -177,6 +197,13 @@ definitions: nodeTypeName: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest: + properties: + name: + type: string + required: + - name + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop: properties: fromIata: @@ -1289,6 +1316,215 @@ info: title: MeshCore Beacon API version: 1.6.0 paths: + /admin/accounts: + get: + description: Includes active and inactive accounts, newest first. These records + are not login users. + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList' + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: List operator accounts + tags: + - Admin + post: + consumes: + - application/json + description: Trims the name; names are case-sensitive and unique among active + accounts. No credential or login is created. + parameters: + - description: Account name (1-128 Unicode characters, no control characters); + body at most 4 KiB + in: body + name: account + required: true + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest' + produces: + - application/json + responses: + "201": + description: Created + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account' + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "409": + description: Conflict + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "413": + description: Request Entity Too Large + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "415": + description: Unsupported Media Type + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Create an operator account + tags: + - Admin + /admin/accounts/{id}: + delete: + description: Soft deactivation preserves the record. Its name may be reused + by a new account. + parameters: + - description: Account UUID + format: uuid + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "204": + description: Deactivated + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "404": + description: Not Found + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "409": + description: Conflict + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Deactivate an operator account + tags: + - Admin + get: + description: Returns active or inactive account records. + parameters: + - description: Account UUID + format: uuid + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account' + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "404": + description: Not Found + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Get an operator account + tags: + - Admin /admin/config: get: description: Returns current CORS options, auth configuration status and configured diff --git a/internal/api/account_requests.go b/internal/api/account_requests.go new file mode 100644 index 00000000..f040f980 --- /dev/null +++ b/internal/api/account_requests.go @@ -0,0 +1,8 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +type CreateAccountRequest struct { + Name string `json:"name" validate:"required"` +} diff --git a/internal/api/handlers/accounts.go b/internal/api/handlers/accounts.go new file mode 100644 index 00000000..92c1fc20 --- /dev/null +++ b/internal/api/handlers/accounts.go @@ -0,0 +1,173 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "encoding/json" + "errors" + "io" + "mime" + "net/http" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" + "github.com/google/uuid" +) + +// AccountsRouter must be mounted inside the authenticated admin subtree. +func AccountsRouter(store api.AccountStore) http.Handler { + if store == nil { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { respondError(w, 503, "account storage is unavailable") }) + } + r := chi.NewRouter() + r.Get("/", listAccounts(store)) + r.Post("/", createAccount(store)) + r.Get("/{id}", getAccount(store)) + r.Delete("/{id}", deactivateAccount(store)) + return r +} + +// listAccounts godoc +// @Summary List operator accounts +// @Description Includes active and inactive accounts, newest first. These records are not login users. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Success 200 {object} api.AccountList +// @Failure 401,503,500 {object} map[string]APIError +// @Router /admin/accounts [get] +func listAccounts(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + items, err := store.ListAccounts(r.Context()) + if err != nil { + accountError(w, err) + return + } + if items == nil { + items = []api.Account{} + } + respond(w, 200, api.AccountList{Items: items}) + } +} + +// createAccount godoc +// @Summary Create an operator account +// @Description Trims the name; names are case-sensitive and unique among active accounts. No credential or login is created. +// @Tags Admin +// @Accept json +// @Produce json +// @Security AdminKey +// @Param account body api.CreateAccountRequest true "Account name (1-128 Unicode characters, no control characters); body at most 4 KiB" +// @Success 201 {object} api.Account +// @Failure 400,401,409,413,415,503,500 {object} map[string]APIError +// @Router /admin/accounts [post] +func createAccount(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/json" { + respondError(w, 415, "Content-Type must be application/json") + return + } + r.Body = http.MaxBytesReader(w, r.Body, 4096) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + var input api.CreateAccountRequest + err = decoder.Decode(&input) + if err == nil { + var extra any + err = decoder.Decode(&extra) + if errors.Is(err, io.EOF) { + err = nil + } else if err == nil { + err = errors.New("multiple JSON values") + } + } + if err != nil { + var sizeError *http.MaxBytesError + if errors.As(err, &sizeError) { + respondError(w, 413, "request body exceeds 4 KiB") + } else { + respondError(w, 400, "invalid account JSON") + } + return + } + name, err := api.NormalizeAccountName(input.Name) + if err != nil { + accountError(w, err) + return + } + account, err := store.CreateAccount(r.Context(), name) + if err != nil { + accountError(w, err) + return + } + respond(w, 201, account) + } +} + +// getAccount godoc +// @Summary Get an operator account +// @Description Returns active or inactive account records. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Param id path string true "Account UUID" format(uuid) +// @Success 200 {object} api.Account +// @Failure 400,401,404,503,500 {object} map[string]APIError +// @Router /admin/accounts/{id} [get] +func getAccount(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id, err := uuid.Parse(chi.URLParam(r, "id")) + if err != nil { + respondError(w, 400, "invalid account ID") + return + } + account, err := store.GetAccount(r.Context(), id) + if err != nil { + accountError(w, err) + return + } + respond(w, 200, account) + } +} + +// deactivateAccount godoc +// @Summary Deactivate an operator account +// @Description Soft deactivation preserves the record. Its name may be reused by a new account. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Param id path string true "Account UUID" format(uuid) +// @Success 204 "Deactivated" +// @Failure 400,401,404,409,503,500 {object} map[string]APIError +// @Router /admin/accounts/{id} [delete] +func deactivateAccount(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id, err := uuid.Parse(chi.URLParam(r, "id")) + if err != nil { + respondError(w, 400, "invalid account ID") + return + } + if err := store.DeactivateAccount(r.Context(), id); err != nil { + accountError(w, err) + return + } + w.WriteHeader(204) + } +} + +func accountError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, api.ErrAccountNameInvalid): + respondError(w, 400, api.ErrAccountNameInvalid.Error()) + case errors.Is(err, api.ErrAccountNameConflict): + respondError(w, 409, api.ErrAccountNameConflict.Error()) + case errors.Is(err, api.ErrAccountNotFound): + respondError(w, 404, api.ErrAccountNotFound.Error()) + case errors.Is(err, api.ErrAccountInactive): + respondError(w, 409, api.ErrAccountInactive.Error()) + default: + respondError(w, 500, "internal server error") + } +} diff --git a/internal/api/handlers/accounts_test.go b/internal/api/handlers/accounts_test.go new file mode 100644 index 00000000..d6b96833 --- /dev/null +++ b/internal/api/handlers/accounts_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + mw "github.com/MeshCore-Beacon/beacon-server/internal/api/middleware" + "github.com/google/uuid" +) + +type accountStub struct { + account api.Account + items []api.Account + err error + calls int + name string + id uuid.UUID +} + +func (s *accountStub) CreateAccount(_ context.Context, name string) (api.Account, error) { + s.calls++ + s.name = name + return s.account, s.err +} +func (s *accountStub) ListAccounts(context.Context) ([]api.Account, error) { + s.calls++ + return s.items, s.err +} +func (s *accountStub) GetAccount(_ context.Context, id uuid.UUID) (api.Account, error) { + s.calls++ + s.id = id + return s.account, s.err +} +func (s *accountStub) DeactivateAccount(_ context.Context, id uuid.UUID) error { + s.calls++ + s.id = id + return s.err +} + +func accountRequest(handler http.Handler, method, path, body, contentType, token string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + return w +} + +func TestAccountInputAndErrors(t *testing.T) { + id := uuid.New() + for _, tc := range []struct { + name, method, path, body, media string + err error + status, calls int + }{ + {"create", "POST", "/", `{"name":" Alice "}`, "application/json", nil, 201, 1}, + {"media parameters", "POST", "/", `{"name":"Alice"}`, "application/json; charset=utf-8", nil, 201, 1}, + {"empty", "POST", "/", `{}`, "application/json", nil, 400, 0}, + {"null", "POST", "/", `null`, "application/json", nil, 400, 0}, + {"wrong field", "POST", "/", `{"name":"Alice","admin":true}`, "application/json", nil, 400, 0}, + {"trailing JSON", "POST", "/", `{"name":"Alice"} {}`, "application/json", nil, 400, 0}, + {"malformed", "POST", "/", `{"name":`, "application/json", nil, 400, 0}, + {"wrong name type", "POST", "/", `{"name":3}`, "application/json", nil, 400, 0}, + {"control", "POST", "/", `{"name":"x\u0000y"}`, "application/json", nil, 400, 0}, + {"oversized", "POST", "/", `{"name":"` + strings.Repeat("a", 5000) + `"}`, "application/json", nil, 413, 0}, + {"oversized trailing space", "POST", "/", `{"name":"Alice"}` + strings.Repeat(" ", 5000), "application/json", nil, 413, 0}, + {"missing media", "POST", "/", `{"name":"Alice"}`, "", nil, 415, 0}, + {"wrong media", "POST", "/", `{"name":"Alice"}`, "text/plain", nil, 415, 0}, + {"conflict", "POST", "/", `{"name":"Alice"}`, "application/json", fmt.Errorf("private-error: %w", api.ErrAccountNameConflict), 409, 1}, + {"get", "GET", "/" + id.String(), "", "", nil, 200, 1}, + {"bad ID", "GET", "/invalid", "", "", nil, 400, 0}, + {"bad delete ID", "DELETE", "/invalid", "", "", nil, 400, 0}, + {"not found", "GET", "/" + id.String(), "", "", api.ErrAccountNotFound, 404, 1}, + {"deactivate", "DELETE", "/" + id.String(), "", "", nil, 204, 1}, + {"inactive", "DELETE", "/" + id.String(), "", "", api.ErrAccountInactive, 409, 1}, + {"delete absent", "DELETE", "/" + id.String(), "", "", api.ErrAccountNotFound, 404, 1}, + {"list database failure", "GET", "/", "", "", errors.New("private-error"), 500, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + store := &accountStub{account: api.Account{ID: id, Name: "Alice", CreatedAt: time.Unix(1, 0).UTC(), Active: true}, err: tc.err} + handler := mw.BearerAuth("test-key", AccountsRouter(store)) + w := accountRequest(handler, tc.method, tc.path, tc.body, tc.media, "test-key") + if w.Code != tc.status || store.calls != tc.calls { + t.Fatalf("status=%d calls=%d", w.Code, store.calls) + } + if w.Header().Get("Cache-Control") != "no-store" || strings.Contains(w.Body.String(), "private-error") { + t.Fatal("cache policy or error disclosure") + } + if tc.status == 204 { + if w.Body.Len() != 0 { + t.Fatal("204 body") + } + return + } + if w.Header().Get("Content-Type") != "application/json" || !json.Valid(w.Body.Bytes()) { + t.Fatal("invalid JSON response") + } + if tc.status == 201 && store.name != "Alice" { + t.Fatal("untrimmed name reached storage") + } + if (tc.method == "DELETE" || tc.method == "GET") && tc.calls > 0 && tc.path != "/" && store.id != id { + t.Fatal("wrong account ID") + } + }) + } +} + +func TestAccountListAndAuthorization(t *testing.T) { + for _, items := range [][]api.Account{nil, {{ID: uuid.New(), Name: "inactive", Active: false}}} { + store := &accountStub{items: items} + handler := mw.BearerAuth("test-key", AccountsRouter(store)) + w := accountRequest(handler, "GET", "/", "", "", "test-key") + var body api.AccountList + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &body) != nil || body.Items == nil || len(body.Items) != len(items) { + t.Fatal("account list shape") + } + } + for _, key := range []string{"", "test-key"} { + store := &accountStub{} + handler := mw.BearerAuth(key, AccountsRouter(store)) + for _, method := range []string{"GET", "POST", "DELETE"} { + w := accountRequest(handler, method, "/", "{}", "application/json", "") + want := 401 + if key == "" { + want = 503 + } + if w.Code != want || store.calls != 0 { + t.Fatal("unauthorized store access") + } + } + } +} diff --git a/internal/api/router/accounts_test.go b/internal/api/router/accounts_test.go new file mode 100644 index 00000000..7cd7fd10 --- /dev/null +++ b/internal/api/router/accounts_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/google/uuid" +) + +type routedAccountStore struct { + api.AccountStore + calls int +} + +func (s *routedAccountStore) CreateAccount(_ context.Context, name string) (api.Account, error) { + s.calls++ + return api.Account{ID: uuid.New(), Name: name, Active: true}, nil +} +func TestAccountRouterUsesProtectedStore(t *testing.T) { + store := &routedAccountStore{} + r := New(nil, nil, nil, Options{Auth: config.AuthConfig{APIKey: "test-key"}, AdminRoutes: map[string]http.Handler{"/accounts": handlers.AccountsRouter(store)}}) + for _, token := range []string{"", "wrong", "test-key"} { + req := httptest.NewRequest("POST", "/api/v1/admin/accounts", strings.NewReader(`{"name":"test"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + want := 401 + if token == "test-key" { + want = 201 + } + if w.Code != want { + t.Fatalf("route status=%d want=%d", w.Code, want) + } + } + if store.calls != 1 { + t.Fatal("auth boundary was bypassed") + } + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest("POST", "/api/v1/accounts", nil)) + if w.Code != 404 || store.calls != 1 { + t.Fatal("accounts mounted publicly") + } +} From 614747e7b2b61c619846ff157a0e3c264f69d9d2 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 20 Sep 2026 12:00:39 -0400 Subject: [PATCH 02/12] docs(admin): explain cross-origin account methods --- README.md | 7 ++++++- config.yaml.example | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6767640b..24af883c 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,12 @@ deactivates the record (204); missing IDs return 404 and an already inactive record returns 409. A deactivated name may be reused by a new account. Lists include active and inactive records, newest first, without pagination. These are operator-defined records; no login, session or API token is created. -Cross-origin clients must have their methods allowed in the existing CORS config. +Cross-origin account clients need both `POST` and `DELETE` in the saved +`cors.allowed_methods`; the default `GET, HEAD, OPTIONS` is read-only. For an +admin UI that also updates configuration, use `[GET, HEAD, OPTIONS, POST, PUT, +DELETE]`, restrict `cors.allowed_origins` to that UI, and allow `Authorization` +and `Content-Type` headers. Otherwise browser preflight blocks these requests +even when the same bearer-authenticated request works with curl. ### Environment variables (`.env`) diff --git a/config.yaml.example b/config.yaml.example index c32525b7..8f955b36 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -166,7 +166,7 @@ websocket: #cors: # allowed_origins: # - "https://app.example.com" -# allowed_methods: [GET, HEAD, OPTIONS, POST] +# allowed_methods: [GET, HEAD, OPTIONS, POST, PUT, DELETE] # admin UI writes # allowed_headers: [Accept, Authorization, Content-Type] # allow_credentials: false # max_age: 300 From 7e53a8596d7ed4b37da12324df6fb0fc7a864f5f Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 13:25:11 -0400 Subject: [PATCH 03/12] feat(backup): add protected database and config download --- .build/Dockerfile | 3 +- .github/workflows/ci.yml | 6 + README.md | 4 +- cmd/beacon/main.go | 21 ++- docs/backup-export.md | 49 +++++ docs/docs.go | 85 +++++++++ docs/swagger.json | 85 +++++++++ docs/swagger.yaml | 57 ++++++ internal/api/handlers/backup.go | 101 ++++++++++ internal/api/handlers/backup_test.go | 103 +++++++++++ .../api/router/backup_integration_test.go | 172 ++++++++++++++++++ internal/api/router/backup_test.go | 38 ++++ internal/backup/connection.go | 97 ++++++++++ internal/backup/connection_test.go | 48 +++++ internal/backup/export.go | 21 ++- internal/config/config.go | 6 + 16 files changed, 890 insertions(+), 6 deletions(-) create mode 100644 internal/api/handlers/backup.go create mode 100644 internal/api/handlers/backup_test.go create mode 100644 internal/api/router/backup_integration_test.go create mode 100644 internal/api/router/backup_test.go create mode 100644 internal/backup/connection.go create mode 100644 internal/backup/connection_test.go diff --git a/.build/Dockerfile b/.build/Dockerfile index 19e41ba7..7703603a 100644 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -9,7 +9,8 @@ COPY . . RUN go build -o beacon ./cmd/beacon FROM alpine:3.19 -RUN apk add --no-cache ca-certificates tzdata +ARG POSTGRES_CLIENT_MAJOR=16 +RUN apk add --no-cache ca-certificates tzdata postgresql${POSTGRES_CLIENT_MAJOR}-client WORKDIR /app COPY --from=builder /app/beacon . CMD ["./beacon"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0972374f..32cce6c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,7 @@ jobs: run: | CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup.test" ./internal/backup + CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup-http.test" ./internal/api/router docker run --rm --network host \ -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable \ @@ -58,6 +59,11 @@ jobs: -v "$RUNNER_TEMP/beacon-backup:/beacon-backup:ro" \ -v "$RUNNER_TEMP/backup.test:/backup.test:ro" \ postgres:16-alpine /backup.test -test.run '^TestExportPostgres$' -test.v + docker run --rm --network host \ + -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ + -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable -e BEACON_BACKUP_TEST_POSTGRES=1 \ + -v "$RUNNER_TEMP/backup-http.test:/backup-http.test:ro" \ + postgres:16-alpine /backup-http.test -test.run '^TestBackupDownloadPostgres$' -test.v - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/README.md b/README.md index 24af883c..becb8bc4 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ in PostgreSQL, and streams live events to WebSocket clients. For deployment instructions including the frontend app, see the deployment docs. For a bounded private database and saved-config bundle, see -[backup export](docs/backup-export.md). This is a standalone export tool; the -backup web interface and import workflow are separate follow-ups. +[backup export](docs/backup-export.md). A standalone export tool is available; the +protected download API is opt-in; browser login and import remain separate follow-ups. --- diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 39f1c610..6c735701 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -10,6 +10,7 @@ import ( "log/slog" "net/http" "os" + "os/exec" "os/signal" "strconv" "syscall" @@ -21,6 +22,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/api/router" "github.com/MeshCore-Beacon/beacon-server/internal/background" + "github.com/MeshCore-Beacon/beacon-server/internal/backup" "github.com/MeshCore-Beacon/beacon-server/internal/cache" "github.com/MeshCore-Beacon/beacon-server/internal/config" "github.com/MeshCore-Beacon/beacon-server/internal/hub" @@ -125,7 +127,19 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - pool, err := pgxpool.New(ctx, getEnv("POSTGRES_DSN")) + dsn := getEnv("POSTGRES_DSN") + var backupOpts backup.Options + if cfg.Backup.Enabled { + service, connectionErr := backup.ConnectionService(dsn) + _, clientErr := exec.LookPath("pg_dump") + if cfg.Auth.APIKey == "" || connectionErr != nil || clientErr != nil { + slog.Error("backup download requires an admin key, a supported PostgreSQL URL and pg_dump in this runtime", "component", "startup") + os.Exit(1) + } + backupOpts = backup.Options{ConfigPath: configPath, MaxBytes: backup.DefaultMaxBytes, + Timeout: backup.DefaultTimeout, Version: version, ConnectionService: service} + } + pool, err := pgxpool.New(ctx, dsn) if err != nil { // Parse errors can embed the complete DSN, including its password. slog.Error("invalid PostgreSQL connection configuration; check POSTGRES_DSN", "component", "startup") @@ -308,7 +322,10 @@ func main() { MaxConnsPerIP: resolved.MaxConnsPerIP, MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, - AdminRoutes: map[string]http.Handler{"/accounts": handlers.AccountsRouter(store)}, + AdminRoutes: map[string]http.Handler{ + "/accounts": handlers.AccountsRouter(store), + "/backup": handlers.BackupRouter(backupOpts), + }, }) srv := &http.Server{ diff --git a/docs/backup-export.md b/docs/backup-export.md index 09afb501..5061061c 100644 --- a/docs/backup-export.md +++ b/docs/backup-export.md @@ -31,6 +31,55 @@ without publishing an output. Normal failure, timeout and handled interruption remove temporary files; a power loss or SIGKILL can leave a private `.beacon-backup-*` staging directory for the operator to inspect. +## Protected download API + +Set `backup.enabled: true` in saved YAML and configure the existing admin bearer +key to enable `GET /api/v1/admin/backup`. Supply the key only in the `Authorization: +Bearer ...` header over HTTPS. No query parameters or request body are accepted. +This is an operator API, not account login or a browser admin panel. Keep it off +on public previews that do not need private exports. + +The server requires `pg_dump` on its own PATH when this option is enabled. The +Docker image includes PostgreSQL 16's client by default; build argument +`POSTGRES_CLIENT_MAJOR` selects another available Alpine package. Match the source +database major version. A client on the host or in a different container is not +sufficient. No Docker socket or host-command bridge is used. + +The download uses the startup `POSTGRES_DSN`, not the standalone export's ambient +`PGDATABASE`. This first adapter supports a single-host `postgres://` or +`postgresql://` URL with an explicit user and database. Passwords, IPv6 and URL +escaping are supported. It copies TLS settings, `passfile`, `connect_timeout`, +`application_name`, `target_session_attrs` and `options`; known `pool_*` controls +are omitted from libpq. Supported PG environment defaults are captured at startup +and URL values override them. Prefer explicit `sslmode=verify-full` and a trusted +`sslrootcert` for remote databases; password-file/TLS paths must be readable in +the same runtime. Their contents are not included in the archive. + +Keyword DSNs, service indirection, multiple hosts, duplicate/unknown query options, +newer protocol negotiation settings and service-file values with line breaks or +edge whitespace are rejected rather than silently changing targets or TLS policy. +Ambient `PGSERVICE`, `PGSERVICEFILE`, `PGSSLNEGOTIATION`, `PGMINPROTOCOLVERSION`, +`PGMAXPROTOCOLVERSION` and `PGTZ` are also unsupported for this adapter. The +standalone command retains native PG* support for those deployments. Errors do +not echo the DSN. Native connection settings live only in a 0600 service file in +private staging; process arguments contain no connection string or credential, +and no request changes Beacon's environment. + +Only one export or transfer is allowed at a time; another request receives 409. +The existing ten-minute/1 GiB SQL/1 MiB YAML limits apply, and disconnecting +cancels an in-progress export. The completed transfer has a ten-minute write +deadline. Temporary files are removed after completion/failure/cancellation; +abrupt process termination can leave private staging. Responses use `no-store` +and a fixed attachment filename. A failed export returns an error without any +partial archive. Authentication missing/invalid is 401, unconfigured access is +503, invalid input is 400, export failure is 500 and timeout is 504. + +Provision a private writable temporary directory (`TMPDIR` on Unix) with space +for roughly twice the SQL limit plus overhead. A small read-only-container tmpfs +is insufficient for a large backup. Exports include saved YAML verbatim and may +include keys/message data; the exclusions below still apply. This API does not +implement archive upload, import/overwrite, schedules or remote storage. + ## Format 1 The tar contains exactly three regular files with fixed names: diff --git a/docs/docs.go b/docs/docs.go index 5973f96a..931f6049 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -338,6 +338,91 @@ const docTemplate = `{ } } }, + "/admin/backup": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint.", + "produces": [ + "application/gzip" + ], + "tags": [ + "Admin" + ], + "summary": "Download a private database and saved-config backup", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + }, + "headers": { + "Content-Disposition": { + "type": "string", + "description": "attachment; filename=beacon-backup.tar.gz" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "504": { + "description": "Gateway Timeout", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/config": { "get": { "security": [ diff --git a/docs/swagger.json b/docs/swagger.json index 52f700e1..785f690e 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -336,6 +336,91 @@ } } }, + "/admin/backup": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint.", + "produces": [ + "application/gzip" + ], + "tags": [ + "Admin" + ], + "summary": "Download a private database and saved-config backup", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + }, + "headers": { + "Content-Disposition": { + "type": "string", + "description": "attachment; filename=beacon-backup.tar.gz" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "504": { + "description": "Gateway Timeout", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/config": { "get": { "security": [ diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6ef3079a..c3f1f423 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1525,6 +1525,63 @@ paths: summary: Get an operator account tags: - Admin + /admin/backup: + get: + description: Opt-in operator export. Returns only a completed bundle; one export/transfer + at a time. Contains secrets. Login sessions, external deployment files and + import are outside this endpoint. + produces: + - application/gzip + responses: + "200": + description: OK + headers: + Content-Disposition: + description: attachment; filename=beacon-backup.tar.gz + type: string + schema: + type: file + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "409": + description: Conflict + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "504": + description: Gateway Timeout + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Download a private database and saved-config backup + tags: + - Admin /admin/config: get: description: Returns current CORS options, auth configuration status and configured diff --git a/internal/api/handlers/backup.go b/internal/api/handlers/backup.go new file mode 100644 index 00000000..db3fe779 --- /dev/null +++ b/internal/api/handlers/backup.go @@ -0,0 +1,101 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "sync/atomic" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/go-chi/chi/v5" +) + +// BackupRouter returns an operator-only subrouter with one bounded export or +// transfer at a time. Its caller must apply the admin authentication middleware. +// The options are startup-owned; the request selects no targets or paths. +func BackupRouter(opts backup.Options) http.Handler { + r := chi.NewRouter() + r.Get("/", backupDownload(opts, backup.Export)) + return r +} + +// backupDownload godoc +// +// @Summary Download a private database and saved-config backup +// @Description Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint. +// @Tags Admin +// @Produce application/gzip +// @Security AdminKey +// @Success 200 {file} binary +// @Header 200 {string} Content-Disposition "attachment; filename=beacon-backup.tar.gz" +// @Failure 400 {object} map[string]APIError +// @Failure 401 {object} map[string]APIError +// @Failure 409 {object} map[string]APIError +// @Failure 500 {object} map[string]APIError +// @Failure 503 {object} map[string]APIError +// @Failure 504 {object} map[string]APIError +// @Router /admin/backup [get] +func backupDownload(opts backup.Options, export func(context.Context, backup.Options) error) http.HandlerFunc { + var busy atomic.Bool + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if opts.ConnectionService == "" { + respondError(w, 503, "backup download is not configured") + return + } + if r.URL.RawQuery != "" || r.ContentLength != 0 || len(r.TransferEncoding) != 0 { + respondError(w, 400, "backup download accepts no query parameters or body") + return + } + if !busy.CompareAndSwap(false, true) { + respondError(w, 409, "a backup download is already in progress") + return + } + defer busy.Store(false) + dir, err := os.MkdirTemp("", "beacon-download-") + if err != nil { + respondError(w, 500, "cannot stage backup") + return + } + defer os.RemoveAll(dir) + requestOpts := opts + requestOpts.OutputPath = filepath.Join(dir, "backup.tar.gz") + if err := export(r.Context(), requestOpts); err != nil { + if errors.Is(err, context.DeadlineExceeded) { + respondError(w, 504, "backup export timed out") + } else if r.Context().Err() == nil { + respondError(w, 500, "backup export failed; check private client configuration and capacity") + } + return + } + file, err := os.Open(requestOpts.OutputPath) + if err != nil { + respondError(w, 500, "cannot read completed backup") + return + } + defer file.Close() + info, err := file.Stat() + if err != nil { + respondError(w, 500, "cannot inspect completed backup") + return + } + // Keep slow downloads from retaining the only slot and private files + // indefinitely. The native server supports ResponseController deadlines. + controller := http.NewResponseController(w) + _ = controller.SetWriteDeadline(time.Now().Add(backup.DefaultTimeout)) + defer controller.SetWriteDeadline(time.Time{}) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Disposition", `attachment; filename="beacon-backup.tar.gz"`) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Content-Length", strconv.FormatInt(info.Size(), 10)) + _, _ = io.Copy(w, file) + } +} diff --git a/internal/api/handlers/backup_test.go b/internal/api/handlers/backup_test.go new file mode 100644 index 00000000..5243a3d0 --- /dev/null +++ b/internal/api/handlers/backup_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" +) + +func TestBackupDownload(t *testing.T) { + opts := backup.Options{ConnectionService: "synthetic-private-service", MaxBytes: 1024, Timeout: time.Second, ConfigPath: "saved.yaml"} + var output string + export := func(ctx context.Context, got backup.Options) error { + if got.ConfigPath != opts.ConfigPath || got.ConnectionService != opts.ConnectionService || got.MaxBytes != opts.MaxBytes || got.Timeout != opts.Timeout { + t.Fatal("startup options changed") + } + output = got.OutputPath + return os.WriteFile(output, []byte("completed fixture"), 0600) + } + handler := backupDownload(opts, export) + for _, query := range []string{"?database=other", "?path=/private", "?command=anything"} { + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup"+query, nil)) + if w.Code != 400 || output != "" { + t.Fatal("caller-selected arguments accepted") + } + } + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", strings.NewReader("body"))) + if w.Code != 400 || output != "" { + t.Fatal("request body accepted") + } + w = httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", nil)) + if w.Code != 200 || w.Body.String() != "completed fixture" || w.Header().Get("Content-Type") != "application/gzip" || + w.Header().Get("Cache-Control") != "no-store" || w.Header().Get("Content-Length") != "17" || w.Header().Get("Content-Disposition") != `attachment; filename="beacon-backup.tar.gz"` { + t.Fatalf("unexpected download: %d %v", w.Code, w.Header()) + } + if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { + t.Fatal("download staging retained") + } + for _, failure := range []error{errors.New("PRIVATE_CANARY"), context.DeadlineExceeded} { + handler = backupDownload(opts, func(_ context.Context, got backup.Options) error { + output = got.OutputPath + return failure + }) + w = httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", nil)) + want := 500 + if errors.Is(failure, context.DeadlineExceeded) { + want = 504 + } + if w.Code != want || strings.Contains(w.Body.String(), "PRIVATE_CANARY") || w.Header().Get("Content-Disposition") != "" { + t.Fatal("failure leaked diagnostics or published a download") + } + if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { + t.Fatal("failed staging retained") + } + } + w = httptest.NewRecorder() + BackupRouter(backup.Options{}).ServeHTTP(w, httptest.NewRequest("GET", "/", nil)) + if w.Code != 503 { + t.Fatal("unconfigured download enabled") + } +} + +func TestBackupDownloadConcurrencyAndCancellation(t *testing.T) { + started, finished := make(chan struct{}), make(chan struct{}) + var output string + handler := backupDownload(backup.Options{ConnectionService: "fixture"}, func(ctx context.Context, opts backup.Options) error { + output = opts.OutputPath + close(started) + <-ctx.Done() + return ctx.Err() + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + defer close(finished) + handler(httptest.NewRecorder(), httptest.NewRequest("GET", "/backup", nil).WithContext(ctx)) + }() + <-started + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", nil)) + if w.Code != http.StatusConflict { + t.Fatal("concurrent export started") + } + cancel() + <-finished + if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { + t.Fatal("cancelled staging retained") + } +} diff --git a/internal/api/router/backup_integration_test.go b/internal/api/router/backup_integration_test.go new file mode 100644 index 00000000..33965a4b --- /dev/null +++ b/internal/api/router/backup_integration_test.go @@ -0,0 +1,172 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// This opt-in test only creates/restores/drops its own randomly named databases. +func TestBackupDownloadPostgres(t *testing.T) { + if os.Getenv("BEACON_BACKUP_TEST_POSTGRES") != "1" { + t.Skip("requires pg_dump/psql and an explicitly selected private PostgreSQL test server") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, "") + if err != nil { + t.Fatal("cannot connect to private PostgreSQL") + } + defer admin.Close(context.Background()) + sourceName, targetName := "beacon_download_"+strings.ToLower(rand.Text()), "beacon_download_"+strings.ToLower(rand.Text()) + for _, name := range []string{sourceName, targetName} { + ident := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil { + t.Fatal("cannot create private fixture") + } + defer func() { + if _, err := admin.Exec(ctx, "DROP DATABASE "+ident+" WITH (FORCE)"); err != nil { + t.Error("cannot remove private fixture") + } + }() + } + cfg, err := pgxpool.ParseConfig("") + if err != nil { + t.Fatal("invalid fixture configuration") + } + cfg.ConnConfig.Database = sourceName + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal("cannot open source fixture") + } + defer pool.Close() + if err := db.RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, "CREATE TABLE download_fixture (id bigint GENERATED ALWAYS AS IDENTITY, message text); INSERT INTO download_fixture(message) VALUES ('café 雪')"); err != nil { + t.Fatal(err) + } + dsn := (&url.URL{Scheme: "postgres", Host: net.JoinHostPort(cfg.ConnConfig.Host, strconv.Itoa(int(cfg.ConnConfig.Port))), + User: url.UserPassword(cfg.ConnConfig.User, cfg.ConnConfig.Password), Path: "/" + sourceName, + RawQuery: "sslmode=disable&pool_max_conns=2"}).String() + service, err := backup.ConnectionService(dsn) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + configPath := filepath.Join(dir, "config.yaml") + saved := []byte("# verbatim saved file\nbackup:\n enabled: true\n") + if err := os.WriteFile(configPath, saved, 0600); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", dir) + server := httptest.NewServer(New(nil, nil, nil, Options{ + Auth: config.AuthConfig{APIKey: "fixture-key"}, + AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ + ConnectionService: service, ConfigPath: configPath, MaxBytes: 16 << 20, Timeout: 30 * time.Second, Version: "http-fixture", + })}, + })) + defer server.Close() + request, _ := http.NewRequestWithContext(ctx, "GET", server.URL+"/api/v1/admin/backup", nil) + request.Header.Set("Authorization", "Bearer fixture-key") + response, err := server.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != 200 || response.Header.Get("Cache-Control") != "no-store" { + t.Fatalf("download status=%d", response.StatusCode) + } + gz, err := gzip.NewReader(response.Body) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tarReader, members := tar.NewReader(gz), map[string][]byte{} + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil || header.Typeflag != tar.TypeReg || header.Mode != 0600 { + t.Fatal("invalid member") + } + members[header.Name], err = io.ReadAll(tarReader) + if err != nil { + t.Fatal(err) + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + t.Fatal(err) + } + var manifest backup.Manifest + if json.Unmarshal(members["manifest.json"], &manifest) != nil || len(members) != 3 || manifest.FormatVersion != 1 || !bytes.Equal(members["config.yaml"], saved) { + t.Fatal("invalid manifest/config") + } + for _, file := range manifest.Files { + sum := sha256.Sum256(members[file.Name]) + if file.Size != int64(len(members[file.Name])) || file.SHA256 != hex.EncodeToString(sum[:]) { + t.Fatal("checksum mismatch") + } + } + if bytes.Contains(members["database.sql"], []byte("beacon_backup]")) { + t.Fatal("connection settings leaked into bundle") + } + restore := exec.CommandContext(ctx, "psql", "-X", "--set=ON_ERROR_STOP=on", "--single-transaction") + restore.Env = append(os.Environ(), "PGDATABASE="+targetName) + restore.Stdin = bytes.NewReader(members["database.sql"]) + if err := restore.Run(); err != nil { + t.Fatal("cannot restore downloaded SQL into empty fixture") + } + targetCfg := admin.Config().Copy() + targetCfg.Database = targetName + target, err := pgx.ConnectConfig(ctx, targetCfg) + if err != nil { + t.Fatal("cannot connect to restored fixture") + } + defer target.Close(context.Background()) + const fingerprint = `SELECT md5(string_agg(table_name||column_name||data_type, ',' ORDER BY table_name,ordinal_position)) FROM information_schema.columns WHERE table_schema='public'` + var before, after, message string + if pool.QueryRow(ctx, fingerprint).Scan(&before) != nil || target.QueryRow(ctx, fingerprint).Scan(&after) != nil || before != after { + t.Fatal("schema differs after restore") + } + if target.QueryRow(ctx, "SELECT message FROM download_fixture").Scan(&message) != nil || message != "café 雪" { + t.Fatal("wrong database or changed data") + } + var nextID int + if target.QueryRow(ctx, "INSERT INTO download_fixture(message) VALUES ('next') RETURNING id").Scan(&nextID) != nil || nextID != 2 { + t.Fatal("identity sequence changed") + } + paths, _ := filepath.Glob(filepath.Join(dir, "beacon-download-*")) + if len(paths) != 0 { + t.Fatal("private HTTP staging retained") + } + t.Log("HTTP bundle restored current Beacon schema and Unicode data; native client ignored ambient PGDATABASE") +} diff --git a/internal/api/router/backup_test.go b/internal/api/router/backup_test.go new file mode 100644 index 00000000..76bdf6e7 --- /dev/null +++ b/internal/api/router/backup_test.go @@ -0,0 +1,38 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestBackupAuthenticationBeforeExport(t *testing.T) { + for _, key := range []string{"", "fixture-key"} { + handler := New(nil, nil, nil, Options{Auth: config.AuthConfig{APIKey: key}, AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ConnectionService: "fixture"})}}) + for _, token := range []string{"", "wrong", "fixture-key"} { + r := httptest.NewRequest("GET", "/api/v1/admin/backup?database=other", nil) + if token != "" { + r.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + want := 503 + if key != "" { + want = 401 + if token == key { + want = 400 + } + } + if w.Code != want || w.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("status=%d want=%d", w.Code, want) + } + } + } +} diff --git a/internal/backup/connection.go b/internal/backup/connection.go new file mode 100644 index 00000000..7bb8969f --- /dev/null +++ b/internal/backup/connection.go @@ -0,0 +1,97 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "errors" + "net/url" + "os" + "sort" + "strconv" + "strings" + "unicode" +) + +// ConnectionService captures a single-host PostgreSQL URL for pg_dump. The +// result contains secrets and must only be written to private staging, never +// logged or passed as a process argument. Unsupported settings fail closed. +func ConnectionService(dsn string) (string, error) { + return connectionService(dsn, os.Environ()) +} + +func connectionService(dsn string, environ []string) (string, error) { + invalid := errors.New("backup requires a single-host PostgreSQL URL with explicit user and database and supported libpq settings") + u, err := url.Parse(dsn) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.User == nil || u.User.Username() == "" || + u.Hostname() == "" || strings.Contains(u.Hostname(), ",") || strings.TrimLeft(u.Path, "/") == "" || u.Fragment != "" { + return "", invalid + } + settings := map[string]string{"host": u.Hostname(), "port": "5432", "user": u.User.Username(), + "dbname": strings.TrimLeft(u.Path, "/"), "sslmode": "prefer", "gssencmode": "disable"} + // Match the pgx startup environment for the supported native client settings. + // Target fields come exclusively from the URL. Service indirection and newer + // protocol options need separate compatibility work, not a guessed translation. + envKeys := map[string]string{"PGPASSWORD": "password", "PGPASSFILE": "passfile", "PGAPPNAME": "application_name", + "PGCONNECT_TIMEOUT": "connect_timeout", "PGSSLMODE": "sslmode", "PGSSLKEY": "sslkey", "PGSSLCERT": "sslcert", + "PGSSLROOTCERT": "sslrootcert", "PGSSLPASSWORD": "sslpassword", "PGSSLSNI": "sslsni", + "PGTARGETSESSIONATTRS": "target_session_attrs", "PGOPTIONS": "options"} + for _, entry := range environ { + key, value, _ := strings.Cut(entry, "=") + if value == "" { + continue + } + switch key { + case "PGSERVICE", "PGSERVICEFILE", "PGSSLNEGOTIATION", "PGMINPROTOCOLVERSION", "PGMAXPROTOCOLVERSION", "PGTZ": + return "", invalid + } + if name := envKeys[key]; name != "" { + settings[name] = value + } + if key == "PGPORT" && u.Port() == "" { + settings["port"] = value + } + } + if port := u.Port(); port != "" { + if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 { + return "", invalid + } + settings["port"] = port + } + if password, ok := u.User.Password(); ok { + settings["password"] = password + } + query, err := url.ParseQuery(u.RawQuery) + if err != nil { + return "", invalid + } + for key, values := range query { + if len(values) != 1 || values[0] == "" { + return "", invalid + } + switch key { + case "pool_max_conns", "pool_min_conns", "pool_min_idle_conns", "pool_max_conn_lifetime", "pool_max_conn_idle_time", "pool_health_check_period", "pool_max_conn_lifetime_jitter": + continue // pgx-only pool controls; main has already validated them. + case "sslmode", "sslcert", "sslkey", "sslrootcert", "sslpassword", "sslsni", "passfile", "connect_timeout", "application_name", "target_session_attrs", "options": + settings[key] = values[0] + default: + return "", invalid + } + } + var result strings.Builder + result.WriteString("[beacon_backup]\n") + keys := make([]string, 0, len(settings)) + for key, value := range settings { + // libpq service files have line-based, unquoted values. Reject anything + // their parser would trim or interpret as another entry. + if strings.TrimSpace(value) != value || strings.IndexFunc(value, unicode.IsControl) >= 0 || len(value) > 1000 { + return "", invalid + } + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + result.WriteString(key + "=" + settings[key] + "\n") + } + return result.String(), nil +} diff --git a/internal/backup/connection_test.go b/internal/backup/connection_test.go new file mode 100644 index 00000000..d640a385 --- /dev/null +++ b/internal/backup/connection_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "strings" + "testing" +) + +func TestConnectionService(t *testing.T) { + service, err := connectionService("postgresql://user:p%23ass%3Dword@[::1]:5544/db%20name?sslmode=verify-full&sslrootcert=%2Fprivate%2Fca.pem&passfile=%2Fprivate%2Fpass&pool_max_conns=2", []string{ + "PGHOST=wrong", "PGDATABASE=wrong", "PGUSER=wrong", "PGPASSWORD=wrong", "PGSSLMODE=disable", "PGPASSFILE=/wrong", "PGSSLKEY=/private/key.pem"}) + if err != nil { + t.Fatal(err) + } + for _, line := range []string{"[beacon_backup]", "host=::1", "port=5544", "user=user", "password=p#ass=word", "dbname=db name", + "sslmode=verify-full", "sslrootcert=/private/ca.pem", "passfile=/private/pass", "sslkey=/private/key.pem"} { + if !strings.Contains(service, line+"\n") { + t.Fatalf("missing expected setting %q", line) + } + } + if strings.Contains(service, "wrong") || strings.Contains(service, "pool_") { + t.Fatal("unrelated target or pool settings leaked into libpq") + } + service, err = connectionService("postgres://user@host/db", []string{"PGPASSWORD=from-environment", "PGPORT=5555", "PGSSLMODE=require"}) + if err != nil || !strings.Contains(service, "password=from-environment\n") || !strings.Contains(service, "port=5555\n") || !strings.Contains(service, "sslmode=require\n") { + t.Fatal("supported environment defaults were not captured") + } +} + +func TestConnectionServiceRejectsAmbiguousSettings(t *testing.T) { + for _, dsn := range []string{ + "host=host user=user dbname=db", "postgres://host/db", "postgres://user@host/", "postgres://user@a,b/db", + "postgres://user@host/db?service=other", "postgres://user@host/db?host=other", "postgres://user@host/db?sslmode=require&sslmode=disable", + "postgres://user@host/db?unknown=x", "postgres://user:secret%0Ahost%3Dother@host/db", "postgres://user:%20secret@host/db", + "postgres://user@host:99999/db", "postgres://user@host/db#fragment", "postgres://user@host/db?sslnegotiation=direct", + } { + if _, err := connectionService(dsn, nil); err == nil || strings.Contains(err.Error(), "secret") { + t.Fatal("unsupported connection must fail without echoing input") + } + } + for _, env := range []string{"PGSERVICE=other", "PGSERVICEFILE=/private", "PGSSLNEGOTIATION=direct", "PGMINPROTOCOLVERSION=3.0", "PGTZ=UTC"} { + if _, err := connectionService("postgres://user@host/db", []string{env}); err == nil { + t.Fatal("unsupported ambient connection settings accepted") + } + } +} diff --git a/internal/backup/export.go b/internal/backup/export.go index 63abf68c..e7b1a56d 100644 --- a/internal/backup/export.go +++ b/internal/backup/export.go @@ -17,6 +17,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" ) @@ -36,6 +37,9 @@ type Options struct { MaxBytes int64 // Maximum uncompressed database dump size. Timeout time.Duration Version string + // ConnectionService overrides PG* using a private service file. Construct it + // with ConnectionService at startup; empty retains the standalone PG* mode. + ConnectionService string } // Manifest describes format 1; hashes cover the uncompressed archive members. @@ -72,7 +76,7 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex if opts.ConfigPath == "" || opts.OutputPath == "" || opts.MaxBytes <= 0 || opts.MaxBytes > 1<<40 || opts.Timeout <= 0 { return errors.New("config, output, positive timeout and max-bytes (at most 1 TiB) are required") } - if os.Getenv("PGDATABASE") == "" { + if opts.ConnectionService == "" && os.Getenv("PGDATABASE") == "" { return errors.New("PGDATABASE must explicitly name the database; POSTGRES_DSN is not used") } ctx, cancel := context.WithTimeout(ctx, opts.Timeout) @@ -104,6 +108,21 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex hash := sha256.New() output := &limitedWriter{ctx: ctx, cancel: cancel, dst: io.MultiWriter(dump, hash), remaining: opts.MaxBytes} cmd := command(ctx) + if opts.ConnectionService != "" { + servicePath := filepath.Join(temp, "pg_service.conf") + if err := os.WriteFile(servicePath, []byte(opts.ConnectionService), 0600); err != nil { + return errors.New("cannot prepare private backup connection") + } + // Do not let unrelated PG* values redirect the client, and never mutate + // the Beacon process environment while another request is running. + cmd.Env = nil + for _, entry := range os.Environ() { + if !strings.HasPrefix(strings.ToUpper(entry), "PG") { + cmd.Env = append(cmd.Env, entry) + } + } + cmd.Env = append(cmd.Env, "PGSERVICEFILE="+servicePath, "PGSERVICE=beacon_backup") + } cmd.Stdout, cmd.Stderr = output, io.Discard cmd.WaitDelay = time.Second err = cmd.Run() diff --git a/internal/config/config.go b/internal/config/config.go index b5fc09a2..61786364 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -18,6 +18,7 @@ import ( // Config is the top-level structure of the Beacon config file. type Config struct { Auth AuthConfig `yaml:"auth"` + Backup BackupConfig `yaml:"backup"` Log LogConfig `yaml:"log"` Server ServerConfig `yaml:"server"` IATAs map[string]IATAConfig `yaml:"iatas"` @@ -38,6 +39,11 @@ type Config struct { Observers ObserversConfig `yaml:"observers"` } +// BackupConfig enables protected downloads. Disabled by default. +type BackupConfig struct { + Enabled bool `yaml:"enabled"` +} + // AuthConfig holds the operator key for the protected admin subtree. // The key is excluded from JSON; it must not be exposed by configuration APIs. type AuthConfig struct { From 8a831a7317e7ce50bc1161b75f951ab2f9e51020 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 13:45:21 -0400 Subject: [PATCH 04/12] fix(backup): reject ambiguous inherited ports --- internal/backup/connection.go | 6 +++--- internal/backup/connection_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/backup/connection.go b/internal/backup/connection.go index 7bb8969f..59fc2fbc 100644 --- a/internal/backup/connection.go +++ b/internal/backup/connection.go @@ -53,11 +53,11 @@ func connectionService(dsn string, environ []string) (string, error) { } } if port := u.Port(); port != "" { - if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 { - return "", invalid - } settings["port"] = port } + if port, err := strconv.Atoi(settings["port"]); err != nil || port < 1 || port > 65535 { + return "", invalid + } if password, ok := u.User.Password(); ok { settings["password"] = password } diff --git a/internal/backup/connection_test.go b/internal/backup/connection_test.go index d640a385..e3c0b118 100644 --- a/internal/backup/connection_test.go +++ b/internal/backup/connection_test.go @@ -40,7 +40,7 @@ func TestConnectionServiceRejectsAmbiguousSettings(t *testing.T) { t.Fatal("unsupported connection must fail without echoing input") } } - for _, env := range []string{"PGSERVICE=other", "PGSERVICEFILE=/private", "PGSSLNEGOTIATION=direct", "PGMINPROTOCOLVERSION=3.0", "PGTZ=UTC"} { + for _, env := range []string{"PGSERVICE=other", "PGSERVICEFILE=/private", "PGSSLNEGOTIATION=direct", "PGMINPROTOCOLVERSION=3.0", "PGTZ=UTC", "PGPORT=5432,5433"} { if _, err := connectionService("postgres://user@host/db", []string{env}); err == nil { t.Fatal("unsupported ambient connection settings accepted") } From 8d7274e680124ed7060cdf6aea01eef005003719 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 20 Sep 2026 12:04:14 -0400 Subject: [PATCH 05/12] fix(backup): isolate startup failures and check client compatibility --- cmd/beacon/backup.go | 45 ++++++++++++++++++++++++++++ cmd/beacon/backup_test.go | 26 ++++++++++++++++ cmd/beacon/main.go | 14 +-------- config.yaml.example | 6 ++++ docs/backup-export.md | 17 +++++++---- docs/docs.go | 9 ++++++ docs/swagger.json | 9 ++++++ docs/swagger.yaml | 6 ++++ internal/api/handlers/backup.go | 8 ++++- internal/api/handlers/backup_test.go | 5 +++- internal/backup/client.go | 40 +++++++++++++++++++++++++ internal/backup/client_test.go | 31 +++++++++++++++++++ 12 files changed, 196 insertions(+), 20 deletions(-) create mode 100644 cmd/beacon/backup.go create mode 100644 cmd/beacon/backup_test.go create mode 100644 internal/backup/client.go create mode 100644 internal/backup/client_test.go diff --git a/cmd/beacon/backup.go b/cmd/beacon/backup.go new file mode 100644 index 00000000..be4601a0 --- /dev/null +++ b/cmd/beacon/backup.go @@ -0,0 +1,45 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/jackc/pgx/v5/pgxpool" +) + +// A failed optional export prerequisite must not stop ingest or the public API. +// Empty options leave the authenticated backup endpoint unavailable (503). +func configureBackup(ctx context.Context, cfg *config.Config, pool *pgxpool.Pool, dsn, configPath string) backup.Options { + if !cfg.Backup.Enabled { + return backup.Options{} + } + if cfg.Auth.APIKey == "" { + slog.Error("backup unavailable: configure an admin bearer key", "component", "backup") + return backup.Options{} + } + service, err := backup.ConnectionService(dsn) + if err != nil { + slog.Error("backup unavailable: POSTGRES_DSN must be a supported single-host PostgreSQL URL; keyword DSNs remain valid for Beacon", "component", "backup") + return backup.Options{} + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + var serverVersion int + if err := pool.QueryRow(ctx, "SHOW server_version_num").Scan(&serverVersion); err != nil { + slog.Error("backup unavailable: could not determine PostgreSQL server version", "component", "backup") + return backup.Options{} + } + if err := backup.CheckClient(ctx, serverVersion); err != nil { + // CheckClient never returns process output or connection settings. + slog.Error("backup unavailable", "component", "backup", "error", err) + return backup.Options{} + } + return backup.Options{ConfigPath: configPath, MaxBytes: backup.DefaultMaxBytes, + Timeout: backup.DefaultTimeout, Version: version, ConnectionService: service} +} diff --git a/cmd/beacon/backup_test.go b/cmd/beacon/backup_test.go new file mode 100644 index 00000000..cb32d896 --- /dev/null +++ b/cmd/beacon/backup_test.go @@ -0,0 +1,26 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestBackupPreconditionsDegradeWithoutDatabaseOrExit(t *testing.T) { + cfg := &config.Config{} + if got := configureBackup(context.Background(), cfg, nil, "", "saved.yaml"); got.ConnectionService != "" { + t.Fatal("disabled backup configured") + } + cfg.Backup.Enabled = true + if got := configureBackup(context.Background(), cfg, nil, "", "saved.yaml"); got.ConnectionService != "" { + t.Fatal("missing admin key configured") + } + cfg.Auth.APIKey = "test-only-key" + if got := configureBackup(context.Background(), cfg, nil, "host=localhost user=test dbname=test", "saved.yaml"); got.ConnectionService != "" { + t.Fatal("keyword connection configured for backup") + } +} diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 6c735701..7c54b7c1 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -10,7 +10,6 @@ import ( "log/slog" "net/http" "os" - "os/exec" "os/signal" "strconv" "syscall" @@ -22,7 +21,6 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/api/router" "github.com/MeshCore-Beacon/beacon-server/internal/background" - "github.com/MeshCore-Beacon/beacon-server/internal/backup" "github.com/MeshCore-Beacon/beacon-server/internal/cache" "github.com/MeshCore-Beacon/beacon-server/internal/config" "github.com/MeshCore-Beacon/beacon-server/internal/hub" @@ -128,17 +126,6 @@ func main() { defer cancel() dsn := getEnv("POSTGRES_DSN") - var backupOpts backup.Options - if cfg.Backup.Enabled { - service, connectionErr := backup.ConnectionService(dsn) - _, clientErr := exec.LookPath("pg_dump") - if cfg.Auth.APIKey == "" || connectionErr != nil || clientErr != nil { - slog.Error("backup download requires an admin key, a supported PostgreSQL URL and pg_dump in this runtime", "component", "startup") - os.Exit(1) - } - backupOpts = backup.Options{ConfigPath: configPath, MaxBytes: backup.DefaultMaxBytes, - Timeout: backup.DefaultTimeout, Version: version, ConnectionService: service} - } pool, err := pgxpool.New(ctx, dsn) if err != nil { // Parse errors can embed the complete DSN, including its password. @@ -146,6 +133,7 @@ func main() { os.Exit(1) } defer pool.Close() + backupOpts := configureBackup(ctx, cfg, pool, dsn, configPath) if err := db.RunMigrations(ctx, pool); err != nil { slog.Error("migrations failed", "component", "startup", "error", err) diff --git a/config.yaml.example b/config.yaml.example index 8f955b36..dd7bfac4 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -112,6 +112,12 @@ telemetry: retention: 672h # 4 weeks resolution: 1h # store one snapshot per observer per hour +# Optional authenticated database/config downloads. Disabled unless explicitly enabled. +# Requires auth.api_key, a supported PostgreSQL URL and a compatible pg_dump client. +# A failed prerequisite leaves this feature unavailable; ingest and other APIs continue. +#backup: +# enabled: false + packets: retention: 720h # 30 days diff --git a/docs/backup-export.md b/docs/backup-export.md index 5061061c..160561f6 100644 --- a/docs/backup-export.md +++ b/docs/backup-export.md @@ -40,9 +40,12 @@ This is an operator API, not account login or a browser admin panel. Keep it off on public previews that do not need private exports. The server requires `pg_dump` on its own PATH when this option is enabled. The -Docker image includes PostgreSQL 16's client by default; build argument -`POSTGRES_CLIENT_MAJOR` selects another available Alpine package. Match the source -database major version. A client on the host or in a different container is not +Docker image includes PostgreSQL 16's client by default. Its Alpine 3.19 base does +not provide PostgreSQL 17/18 packages: changing `POSTGRES_CLIENT_MAJOR` alone is +not an upgrade path. For a newer database, supply a runtime with a compatible +client. Startup compares `pg_dump --version` with `server_version_num` and leaves +backup unavailable if the client is missing, unrecognized or older than the server. +Match the source database major version. A client on the host or in a different container is not sufficient. No Docker socket or host-command bridge is used. The download uses the startup `POSTGRES_DSN`, not the standalone export's ambient @@ -61,7 +64,9 @@ edge whitespace are rejected rather than silently changing targets or TLS policy Ambient `PGSERVICE`, `PGSERVICEFILE`, `PGSSLNEGOTIATION`, `PGMINPROTOCOLVERSION`, `PGMAXPROTOCOLVERSION` and `PGTZ` are also unsupported for this adapter. The standalone command retains native PG* support for those deployments. Errors do -not echo the DSN. Native connection settings live only in a 0600 service file in +not echo the DSN. Any failed backup prerequisite logs its reason and leaves only +this feature unavailable; Beacon still starts ingest, WebSocket and other APIs. +Fix the prerequisite and restart to enable downloads. Native connection settings live only in a 0600 service file in private staging; process arguments contain no connection string or credential, and no request changes Beacon's environment. @@ -72,7 +77,9 @@ deadline. Temporary files are removed after completion/failure/cancellation; abrupt process termination can leave private staging. Responses use `no-store` and a fixed attachment filename. A failed export returns an error without any partial archive. Authentication missing/invalid is 401, unconfigured access is -503, invalid input is 400, export failure is 500 and timeout is 504. +503, invalid input is 400, export failure is 500, timeout is 504 and exceeding the +configured export size limit is 507. Other export failures retain sanitized +operator diagnostics in the server log; raw pg_dump stderr remains discarded. Provision a private writable temporary directory (`TMPDIR` on Unix) with space for roughly twice the SQL limit plus overhead. A small read-only-container tmpfs diff --git a/docs/docs.go b/docs/docs.go index 931f6049..5988c94b 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -419,6 +419,15 @@ const docTemplate = `{ "$ref": "#/definitions/internal_api_handlers.APIError" } } + }, + "507": { + "description": "Insufficient Storage", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } } } } diff --git a/docs/swagger.json b/docs/swagger.json index 785f690e..b9a03899 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -417,6 +417,15 @@ "$ref": "#/definitions/internal_api_handlers.APIError" } } + }, + "507": { + "description": "Insufficient Storage", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } } } } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index c3f1f423..971bc30f 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1577,6 +1577,12 @@ paths: additionalProperties: $ref: '#/definitions/internal_api_handlers.APIError' type: object + "507": + description: Insufficient Storage + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object security: - AdminKey: [] summary: Download a private database and saved-config backup diff --git a/internal/api/handlers/backup.go b/internal/api/handlers/backup.go index db3fe779..67f9641d 100644 --- a/internal/api/handlers/backup.go +++ b/internal/api/handlers/backup.go @@ -7,6 +7,7 @@ import ( "context" "errors" "io" + "log/slog" "net/http" "os" "path/filepath" @@ -42,6 +43,7 @@ func BackupRouter(opts backup.Options) http.Handler { // @Failure 500 {object} map[string]APIError // @Failure 503 {object} map[string]APIError // @Failure 504 {object} map[string]APIError +// @Failure 507 {object} map[string]APIError // @Router /admin/backup [get] func backupDownload(opts backup.Options, export func(context.Context, backup.Options) error) http.HandlerFunc { var busy atomic.Bool @@ -69,9 +71,13 @@ func backupDownload(opts backup.Options, export func(context.Context, backup.Opt requestOpts := opts requestOpts.OutputPath = filepath.Join(dir, "backup.tar.gz") if err := export(r.Context(), requestOpts); err != nil { - if errors.Is(err, context.DeadlineExceeded) { + if errors.Is(err, backup.ErrTooLarge) { + respondError(w, http.StatusInsufficientStorage, "backup exceeds the configured export size limit") + } else if errors.Is(err, context.DeadlineExceeded) { respondError(w, 504, "backup export timed out") } else if r.Context().Err() == nil { + // Export discards pg_dump stderr and never returns connection settings. + slog.Error("backup export failed", "component", "backup", "error", err) respondError(w, 500, "backup export failed; check private client configuration and capacity") } return diff --git a/internal/api/handlers/backup_test.go b/internal/api/handlers/backup_test.go index 5243a3d0..64c77247 100644 --- a/internal/api/handlers/backup_test.go +++ b/internal/api/handlers/backup_test.go @@ -49,7 +49,7 @@ func TestBackupDownload(t *testing.T) { if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { t.Fatal("download staging retained") } - for _, failure := range []error{errors.New("PRIVATE_CANARY"), context.DeadlineExceeded} { + for _, failure := range []error{errors.New("PRIVATE_CANARY"), context.DeadlineExceeded, backup.ErrTooLarge} { handler = backupDownload(opts, func(_ context.Context, got backup.Options) error { output = got.OutputPath return failure @@ -60,6 +60,9 @@ func TestBackupDownload(t *testing.T) { if errors.Is(failure, context.DeadlineExceeded) { want = 504 } + if errors.Is(failure, backup.ErrTooLarge) { + want = 507 + } if w.Code != want || strings.Contains(w.Body.String(), "PRIVATE_CANARY") || w.Header().Get("Content-Disposition") != "" { t.Fatal("failure leaked diagnostics or published a download") } diff --git a/internal/backup/client.go b/internal/backup/client.go new file mode 100644 index 00000000..55e6f975 --- /dev/null +++ b/internal/backup/client.go @@ -0,0 +1,40 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "context" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" +) + +var dumpVersion = regexp.MustCompile(`^pg_dump \(PostgreSQL\) ([0-9]+)(?:\.[0-9]+)+(?:[ \r\n]|$)`) + +// CheckClient checks the client inside this runtime before enabling downloads. +// pg_dump cannot export a server newer than its own major version. +func CheckClient(ctx context.Context, serverVersion int) error { + output, err := exec.CommandContext(ctx, "pg_dump", "--version").Output() + if err != nil { + return errors.New("cannot run pg_dump --version; install the PostgreSQL client in this runtime") + } + return checkClientVersion(output, serverVersion) +} + +func checkClientVersion(output []byte, serverVersion int) error { + match := dumpVersion.FindSubmatch(output) + if match == nil || serverVersion < 100000 { + return errors.New("cannot verify pg_dump and PostgreSQL server major versions") + } + client, err := strconv.Atoi(string(match[1])) + if err != nil { + return errors.New("cannot verify pg_dump major version") + } + if client < serverVersion/10000 { + return fmt.Errorf("pg_dump major %d is older than PostgreSQL server major %d; install a compatible client and restart Beacon", client, serverVersion/10000) + } + return nil +} diff --git a/internal/backup/client_test.go b/internal/backup/client_test.go new file mode 100644 index 00000000..605921dd --- /dev/null +++ b/internal/backup/client_test.go @@ -0,0 +1,31 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "strings" + "testing" +) + +func TestCheckClientVersion(t *testing.T) { + for _, tc := range []struct { + output string + server int + valid bool + }{ + {"pg_dump (PostgreSQL) 16.10\n", 160010, true}, + {"pg_dump (PostgreSQL) 17.6 (Debian 17.6)\n", 160010, true}, + {"pg_dump (PostgreSQL) 16.10\n", 170006, false}, + {"pg_dump (PostgreSQL) 16.10\n", 180000, false}, + {"pg_dump (PostgreSQL) 18.0\n", 180000, true}, + {"PRIVATE_CANARY", 160010, false}, + {"pg_dump (PostgreSQL) 16.bad", 160010, false}, + {"pg_dump (PostgreSQL) 16.10\n", 0, false}, + } { + err := checkClientVersion([]byte(tc.output), tc.server) + if (err == nil) != tc.valid || (err != nil && strings.Contains(err.Error(), "PRIVATE_CANARY")) { + t.Fatalf("server %d: %v", tc.server, err) + } + } +} From 4cbde1889f934d78a908438376aa92dfecc3cea8 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 20 Sep 2026 12:36:49 -0400 Subject: [PATCH 06/12] fix(backup): read a typed server version and test startup with PostgreSQL --- .github/workflows/ci.yml | 5 +++++ cmd/beacon/backup.go | 2 +- cmd/beacon/backup_test.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32cce6c5..49b0f24e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,11 @@ jobs: - name: Test run: go test ./... + - name: Verify backup startup against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./cmd/beacon -run '^TestBackupConfigurationPostgres$' -count=1 -v + - name: Verify backup command against PostgreSQL 16 run: | CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup diff --git a/cmd/beacon/backup.go b/cmd/beacon/backup.go index be4601a0..ee3bc148 100644 --- a/cmd/beacon/backup.go +++ b/cmd/beacon/backup.go @@ -31,7 +31,7 @@ func configureBackup(ctx context.Context, cfg *config.Config, pool *pgxpool.Pool ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() var serverVersion int - if err := pool.QueryRow(ctx, "SHOW server_version_num").Scan(&serverVersion); err != nil { + if err := pool.QueryRow(ctx, "SELECT current_setting('server_version_num')::integer").Scan(&serverVersion); err != nil { slog.Error("backup unavailable: could not determine PostgreSQL server version", "component", "backup") return backup.Options{} } diff --git a/cmd/beacon/backup_test.go b/cmd/beacon/backup_test.go index cb32d896..3a49a44a 100644 --- a/cmd/beacon/backup_test.go +++ b/cmd/beacon/backup_test.go @@ -5,9 +5,14 @@ package main import ( "context" + "os" + "path/filepath" + "runtime" "testing" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/jackc/pgx/v5/pgxpool" ) func TestBackupPreconditionsDegradeWithoutDatabaseOrExit(t *testing.T) { @@ -24,3 +29,30 @@ func TestBackupPreconditionsDegradeWithoutDatabaseOrExit(t *testing.T) { t.Fatal("keyword connection configured for backup") } } + +func TestBackupConfigurationPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" || runtime.GOOS == "windows" { + t.Skip("requires BEACON_TEST_POSTGRES_DSN and a Unix test client") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal("test database configuration unavailable") + } + defer pool.Close() + bin := t.TempDir() + t.Setenv("PATH", bin) + cfg := &config.Config{} + cfg.Backup.Enabled, cfg.Auth.APIKey = true, "test-only-key" + for _, major := range []string{"999", "15"} { + if err := os.WriteFile(filepath.Join(bin, "pg_dump"), []byte("#!/bin/sh\necho 'pg_dump (PostgreSQL) "+major+".0'\n"), 0700); err != nil { + t.Fatal(err) + } + got := configureBackup(ctx, cfg, pool, dsn, "saved.yaml") + if (got.ConnectionService != "") != (major == "999") { + t.Fatalf("unexpected backup availability for test client major %s", major) + } + } +} From 4b3aabd84ed94ebf6ac3424d4a9b89832e245047 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 21:35:12 -0400 Subject: [PATCH 07/12] feat(stats): aggregate reception signal distributions and trends --- .github/workflows/ci.yml | 5 + db/queries/signal.sql | 32 +++++ db/signal.go | 68 ++++++++++ db/signal_integration_test.go | 121 +++++++++++++++++ db/signal_test.go | 21 +++ db/sqlc/mock/querier.go | 15 +++ db/sqlc/querier.go | 4 + db/sqlc/signal.sql.go | 94 +++++++++++++ docs/docs.go | 152 ++++++++++++++++++++++ docs/swagger.json | 152 ++++++++++++++++++++++ docs/swagger.yaml | 106 +++++++++++++++ internal/api/handlers/signal.go | 76 +++++++++++ internal/api/handlers/signal_test.go | 100 ++++++++++++++ internal/api/handlers/stats.go | 1 + internal/api/handlers/stub_reader_test.go | 4 + internal/api/reader.go | 3 + internal/api/signal.go | 41 ++++++ internal/cache/cache_test.go | 4 + internal/cache/reader.go | 1 + internal/cache/signal.go | 25 ++++ internal/cache/signal_test.go | 48 +++++++ 21 files changed, 1073 insertions(+) create mode 100644 db/queries/signal.sql create mode 100644 db/signal.go create mode 100644 db/signal_integration_test.go create mode 100644 db/signal_test.go create mode 100644 db/sqlc/signal.sql.go create mode 100644 internal/api/handlers/signal.go create mode 100644 internal/api/handlers/signal_test.go create mode 100644 internal/api/signal.go create mode 100644 internal/cache/signal.go create mode 100644 internal/cache/signal_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49b0f24e..9b6b62a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,11 @@ jobs: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable run: go test ./cmd/beacon -run '^TestBackupConfigurationPostgres$' -count=1 -v + - name: Verify signal aggregates against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./db -run '^TestSignalPostgres$' -count=1 -v + - name: Verify backup command against PostgreSQL 16 run: | CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup diff --git a/db/queries/signal.sql b/db/queries/signal.sql new file mode 100644 index 00000000..d247ea58 --- /dev/null +++ b/db/queries/signal.sql @@ -0,0 +1,32 @@ +-- name: GetSignalStats :many +-- One reception scan supplies totals, hourly means, and two independent histograms. +-- The zero/zero pair is Beacon's existing unavailable-reading sentinel. A real +-- zero SNR with nonzero RSSI remains valid. Non-finite SNR never reaches JSON. +WITH samples AS ( + SELECT heard_at, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM packet_observations + WHERE heard_at >= @since::timestamptz AND heard_at < @until::timestamptz + AND (COALESCE(cardinality(@iatas::text[]), 0) = 0 OR iata = ANY(@iatas::text[])) +), binned AS ( + SELECT date_trunc('hour', heard_at, 'UTC') AS hour, snr, rssi, + width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT grouping(hour, snr_bin, rssi_bin)::integer AS kind, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(avg(snr), 0)::double precision AS snr_average, + count(rssi)::bigint AS rssi_samples, + COALESCE(avg(rssi), 0)::double precision AS rssi_average +FROM binned +GROUP BY GROUPING SETS ((), (hour), (snr_bin), (rssi_bin)) +ORDER BY kind, hour, snr_bin, rssi_bin; diff --git a/db/signal.go b/db/signal.go new file mode 100644 index 00000000..da346751 --- /dev/null +++ b/db/signal.go @@ -0,0 +1,68 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + rows, err := s.q.GetSignalStats(ctx, sqlc.GetSignalStatsParams{ + Since: pgtype.Timestamptz{Time: since, Valid: true}, Until: pgtype.Timestamptz{Time: until, Valid: true}, Iatas: iatas, + }) + if err != nil { + return nil, err + } + stats := &api.SignalStats{Since: since.UnixMilli(), Until: until.UnixMilli(), + SNR: api.SignalMetric{Histogram: signalBins(-30, 5, 12)}, RSSI: api.SignalMetric{Histogram: signalBins(-140, 10, 14)}, Hourly: []api.SignalHour{}} + for _, row := range rows { + // GROUPING bits distinguish real null buckets from dimensions not in the set. + switch row.Kind { + case 7: + stats.Receptions = row.Receptions + stats.SNR.Samples, stats.SNR.Average = row.SnrSamples, signalAverage(row.SnrAverage, row.SnrSamples) + stats.RSSI.Samples, stats.RSSI.Average = row.RssiSamples, signalAverage(row.RssiAverage, row.RssiSamples) + case 3: + stats.Hourly = append(stats.Hourly, api.SignalHour{Hour: row.Hour.Time.UnixMilli(), Receptions: row.Receptions, + SNRSamples: row.SnrSamples, SNRAverage: signalAverage(row.SnrAverage, row.SnrSamples), + RSSISamples: row.RssiSamples, RSSIAverage: signalAverage(row.RssiAverage, row.RssiSamples)}) + case 5: + if row.SnrBin >= 0 { + stats.SNR.Histogram[row.SnrBin].Count = row.Receptions + } + case 6: + if row.RssiBin >= 0 { + stats.RSSI.Histogram[row.RssiBin].Count = row.Receptions + } + } + } + return stats, nil +} + +func signalAverage(value float64, count int64) *float64 { + if count == 0 { + return nil + } + return &value +} + +func signalBins(lower, width float64, count int) []api.SignalBin { + bins := make([]api.SignalBin, count+2) + for i := range bins { + if i > 0 { + v := lower + float64(i-1)*width + bins[i].Lower = &v + } + if i <= count { + v := lower + float64(i)*width + bins[i].Upper = &v + } + } + return bins +} diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go new file mode 100644 index 00000000..371e6bbb --- /dev/null +++ b/db/signal_integration_test.go @@ -0,0 +1,121 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "math" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +// Only temporary tables are written; no migrated schema or production rows are needed. +func TestSignalPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + _, err = tx.Exec(ctx, ` +SET LOCAL TIME ZONE 'America/Vancouver'; +CREATE TEMP TABLE packet_observations (heard_at timestamptz NOT NULL, iata char(3) NOT NULL, snr real, rssi smallint) ON COMMIT DROP; +INSERT INTO packet_observations +SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FROM (VALUES + ('00:30:00','YVR','0',-100), ('00:45:00','YVR','0',0), ('00:59:00','YVR',NULL,NULL), + ('01:00:00','YVR','-30',-140), ('01:10:00','YVR','-30.25',-141), ('01:20:00','YVR','30',0), + ('01:30:00','YVR','5',NULL), ('01:40:00','YVR',NULL,-90), ('01:50:00','YVR','NaN',-80), + ('02:00:00','YVR','Infinity',-70), ('02:10:00','YVR','-Infinity',-60), + ('02:20:00','YYJ','10',-50), ('03:00:00','YYZ','-5',-110), + ('00:29:59','YVR','20',-40), ('04:30:00','YVR','20',-40), + ('03:10:00','YVR','0',NULL), ('03:20:00','YVR',NULL,0) +) v(at,iata,snr,rssi);`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + since := time.Date(2026, 1, 1, 0, 30, 0, 0, time.UTC) + until := since.Add(4 * time.Hour) + for _, tc := range []struct { + name string + iatas []string + receptions, snr, rssi int64 + }{ + {"global", nil, 15, 7, 10}, {"empty filter", []string{}, 15, 7, 10}, {"YVR", []string{"YVR"}, 13, 5, 8}, + {"duplicate/overlapping IATAs", []string{"YVR", "YYJ", "YVR"}, 14, 6, 9}, + {"unknown IATA", []string{"ZZZ"}, 0, 0, 0}, {"empty region", []string{""}, 0, 0, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := store.GetSignalStats(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + if got.Receptions != tc.receptions || got.SNR.Samples != tc.snr || got.RSSI.Samples != tc.rssi { + t.Fatalf("counts: %+v", got) + } + for _, metric := range []api.SignalMetric{got.SNR, got.RSSI} { + var total int64 + for _, bin := range metric.Histogram { + total += bin.Count + } + if total != metric.Samples || (metric.Average == nil) != (metric.Samples == 0) { + t.Fatalf("histogram/average: %+v", metric) + } + } + var receptions, snr, rssi int64 + for i, hour := range got.Hourly { + if hour.Hour%3600000 != 0 || hour.Hour < since.Truncate(time.Hour).UnixMilli() || hour.Hour >= until.UnixMilli() || (i > 0 && hour.Hour <= got.Hourly[i-1].Hour) { + t.Fatalf("UTC order/window: %+v", got.Hourly) + } + receptions += hour.Receptions + snr += hour.SNRSamples + rssi += hour.RSSISamples + } + if receptions != got.Receptions || snr != got.SNR.Samples || rssi != got.RSSI.Samples { + t.Fatal("hourly totals differ") + } + if _, err := json.Marshal(got); err != nil { + t.Fatal(err) + } + if tc.name == "YVR" { + if math.Abs(*got.SNR.Average-(-5.05)) > 1e-9 || *got.RSSI.Average != -85.125 { + t.Fatalf("unweighted/missing readings: %+v", got) + } + if got.SNR.Histogram[0].Count != 1 || got.SNR.Histogram[1].Count != 1 || got.SNR.Histogram[7].Count != 1 || got.SNR.Histogram[8].Count != 1 || got.SNR.Histogram[13].Count != 1 { + t.Fatalf("SNR boundaries/zero: %+v", got.SNR.Histogram) + } + if got.RSSI.Histogram[0].Count != 1 || got.RSSI.Histogram[1].Count != 1 || got.RSSI.Histogram[15].Count != 1 { + t.Fatalf("RSSI boundaries: %+v", got.RSSI.Histogram) + } + last := got.Hourly[len(got.Hourly)-1] + if last.Receptions != 2 || last.SNRAverage != nil || last.RSSIAverage != nil { + t.Fatalf("missing-only hour: %+v", last) + } + } + }) + } + if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { + t.Fatal(err) + } + got, err := store.GetSignalStats(ctx, since, until, nil) + if err != nil || got.Receptions != 0 || got.Hourly == nil || len(got.Hourly) != 0 || got.SNR.Average != nil { + t.Fatalf("empty: %+v %v", got, err) + } +} diff --git a/db/signal_test.go b/db/signal_test.go new file mode 100644 index 00000000..35b24f69 --- /dev/null +++ b/db/signal_test.go @@ -0,0 +1,21 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import "testing" + +func TestSignalBinsAndMissingAverage(t *testing.T) { + bins := signalBins(-30, 5, 12) + if len(bins) != 14 || bins[0].Lower != nil || *bins[0].Upper != -30 || *bins[13].Lower != 30 || bins[13].Upper != nil { + t.Fatalf("bounds: %+v", bins) + } + for i := 1; i < 13; i++ { + if *bins[i].Upper-*bins[i].Lower != 5 || *bins[i].Lower != *bins[i-1].Upper { + t.Fatal("non-contiguous buckets") + } + } + if signalAverage(0, 0) != nil || *signalAverage(0, 1) != 0 { + t.Fatal("zero must be distinct from no samples") + } +} diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index bb46595c..843fdc22 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -697,6 +697,21 @@ func (mr *MockQuerierMockRecorder) GetScopesByIATAs(ctx, dollar_1 any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopesByIATAs", reflect.TypeOf((*MockQuerier)(nil).GetScopesByIATAs), ctx, dollar_1) } +// GetSignalStats mocks base method. +func (m *MockQuerier) GetSignalStats(ctx context.Context, arg db.GetSignalStatsParams) ([]db.GetSignalStatsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSignalStats", ctx, arg) + ret0, _ := ret[0].([]db.GetSignalStatsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSignalStats indicates an expected call of GetSignalStats. +func (mr *MockQuerierMockRecorder) GetSignalStats(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSignalStats", reflect.TypeOf((*MockQuerier)(nil).GetSignalStats), ctx, arg) +} + // GetStatsClockDrift mocks base method. func (m *MockQuerier) GetStatsClockDrift(ctx context.Context, arg db.GetStatsClockDriftParams) ([]db.GetStatsClockDriftRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 7496e554..1a7c6fc0 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -90,6 +90,10 @@ type Querier interface { // whose observations have expired; the filtered aggregates are empty in that case. GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) + // One reception scan supplies totals, hourly means, and two independent histograms. + // The zero/zero pair is Beacon's existing unavailable-reading sentinel. A real + // zero SNR with nonzero RSSI remains valid. Non-finite SNR never reaches JSON. + GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds // the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's // latest measured drift, not an aggregate over a period. diff --git a/db/sqlc/signal.sql.go b/db/sqlc/signal.sql.go new file mode 100644 index 00000000..32dd4064 --- /dev/null +++ b/db/sqlc/signal.sql.go @@ -0,0 +1,94 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: signal.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getSignalStats = `-- name: GetSignalStats :many +WITH samples AS ( + SELECT heard_at, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM packet_observations + WHERE heard_at >= $1::timestamptz AND heard_at < $2::timestamptz + AND (COALESCE(cardinality($3::text[]), 0) = 0 OR iata = ANY($3::text[])) +), binned AS ( + SELECT date_trunc('hour', heard_at, 'UTC') AS hour, snr, rssi, + width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT grouping(hour, snr_bin, rssi_bin)::integer AS kind, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(avg(snr), 0)::double precision AS snr_average, + count(rssi)::bigint AS rssi_samples, + COALESCE(avg(rssi), 0)::double precision AS rssi_average +FROM binned +GROUP BY GROUPING SETS ((), (hour), (snr_bin), (rssi_bin)) +ORDER BY kind, hour, snr_bin, rssi_bin +` + +type GetSignalStatsParams struct { + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetSignalStatsRow struct { + Kind int32 `json:"kind"` + Hour pgtype.Timestamptz `json:"hour"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrAverage float64 `json:"snr_average"` + RssiSamples int64 `json:"rssi_samples"` + RssiAverage float64 `json:"rssi_average"` +} + +// One reception scan supplies totals, hourly means, and two independent histograms. +// The zero/zero pair is Beacon's existing unavailable-reading sentinel. A real +// zero SNR with nonzero RSSI remains valid. Non-finite SNR never reaches JSON. +func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) { + rows, err := q.db.Query(ctx, getSignalStats, arg.Since, arg.Until, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetSignalStatsRow{} + for rows.Next() { + var i GetSignalStatsRow + if err := rows.Scan( + &i.Kind, + &i.Hour, + &i.SnrBin, + &i.RssiBin, + &i.Receptions, + &i.SnrSamples, + &i.SnrAverage, + &i.RssiSamples, + &i.RssiAverage, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/docs/docs.go b/docs/docs.go index 5988c94b..7be9b2e9 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2674,6 +2674,78 @@ const docTemplate = `{ } } }, + "/stats/signal": { + "get": { + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. UTC hourly buckets are clipped to the requested window; absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Reception signal distributions and hourly trends", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/top-advertisers": { "get": { "produces": [ @@ -4626,6 +4698,86 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "lower": { + "type": "number" + }, + "upper": { + "type": "number" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "rssiAverage": { + "type": "number" + }, + "rssiSamples": { + "type": "integer" + }, + "snrAverage": { + "type": "number" + }, + "snrSamples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric": { + "type": "object", + "properties": { + "average": { + "type": "number" + }, + "histogram": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin" + } + }, + "samples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats": { + "type": "object", + "properties": { + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour" + } + }, + "receptions": { + "type": "integer" + }, + "rssi": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "since": { + "type": "integer" + }, + "snr": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index b9a03899..6369a8a0 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2672,6 +2672,78 @@ } } }, + "/stats/signal": { + "get": { + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. UTC hourly buckets are clipped to the requested window; absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Reception signal distributions and hourly trends", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/top-advertisers": { "get": { "produces": [ @@ -4624,6 +4696,86 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "lower": { + "type": "number" + }, + "upper": { + "type": "number" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "rssiAverage": { + "type": "number" + }, + "rssiSamples": { + "type": "integer" + }, + "snrAverage": { + "type": "number" + }, + "snrSamples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric": { + "type": "object", + "properties": { + "average": { + "type": "number" + }, + "histogram": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin" + } + }, + "samples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats": { + "type": "object", + "properties": { + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour" + } + }, + "receptions": { + "type": "integer" + }, + "rssi": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "since": { + "type": "integer" + }, + "snr": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 971bc30f..9e478176 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1105,6 +1105,58 @@ definitions: description: distinct packets matched to this scope type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin: + properties: + count: + type: integer + lower: + type: number + upper: + type: number + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour: + properties: + hour: + type: integer + receptions: + type: integer + rssiAverage: + type: number + rssiSamples: + type: integer + snrAverage: + type: number + snrSamples: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric: + properties: + average: + type: number + histogram: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin' + type: array + samples: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats: + properties: + hourly: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour' + type: array + receptions: + type: integer + rssi: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric' + since: + type: integer + snr: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric' + until: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview: properties: activeIatas: @@ -3101,6 +3153,60 @@ paths: summary: Scope statistics tags: - Stats + /stats/signal: + get: + description: Aggregates stored observer receptions in [since, until), at most + 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero + unavailable sentinel are excluded per metric; actual zero SNR with nonzero + RSSI remains valid. Averages are null without samples. Histogram bounds are + lower-inclusive/upper-exclusive, with null for unbounded ends. UTC hourly + buckets are clipped to the requested window; absent hours are omitted. These + last-hop readings do not measure end-to-end quality or packet loss. + parameters: + - description: Inclusive start, epoch milliseconds (0 through 253402300799999) + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; at most 30 days after since + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Reception signal distributions and hourly trends + tags: + - Stats /stats/top-advertisers: get: parameters: diff --git a/internal/api/handlers/signal.go b/internal/api/handlers/signal.go new file mode 100644 index 00000000..b09f9999 --- /dev/null +++ b/internal/api/handlers/signal.go @@ -0,0 +1,76 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strconv" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// getSignalStats godoc +// +// @Summary Reception signal distributions and hourly trends +// @Description Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. UTC hourly buckets are clipped to the requested window; absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss. +// @Tags Stats +// @Produce json +// @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" +// @Param until query int true "Exclusive end, epoch milliseconds; at most 30 days after since" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.SignalStats +// @Failure 400 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/signal [get] +func getSignalStats(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + for _, key := range []string{"since", "until"} { + if len(q[key]) != 1 || q.Get(key) == "" { + respondError(w, http.StatusBadRequest, key+" must be supplied exactly once") + return + } + } + since, errSince := strconv.ParseInt(q.Get("since"), 10, 64) + until, errUntil := strconv.ParseInt(q.Get("until"), 10, 64) + if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 || until-since > int64((30*24*time.Hour)/time.Millisecond) { + respondError(w, http.StatusBadRequest, "since and until must be epoch milliseconds between 0 and 253402300799999, with a positive window of at most 30 days") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondError(w, http.StatusBadRequest, "region not found") + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} + } + } + stats, err := reader.GetSignalStats(ctx, time.UnixMilli(since), time.UnixMilli(until), iatas) + switch { + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "signal query timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("signal stats failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, stats) + } + } +} diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go new file mode 100644 index 00000000..8625afde --- /dev/null +++ b/internal/api/handlers/signal_test.go @@ -0,0 +1,100 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type signalReader struct { + stubReader + read func(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) +} + +func (s signalReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + return s.read(ctx, since, until, iatas) +} + +func TestSignalRejectsInvalidWindow(t *testing.T) { + for _, query := range []string{"", "since=0", "until=1", "since=x&until=1", "since=-1&until=1", "since=0&until=0", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { + w := httptest.NewRecorder() + StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) + if w.Code != 400 { + t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) + } + } +} + +func TestSignalFiltersAndErrors(t *testing.T) { + for _, tc := range []struct { + name, query string + iatas []string + err error + status int + }{ + {"global", "", nil, nil, 200}, + {"IATAs", "&iatas=yvr,yyj", []string{"YVR", "YYJ"}, nil, 200}, + {"region", "®ion=west", []string{"YVR"}, nil, 200}, + {"empty region", "®ion=empty", []string{""}, nil, 200}, + {"missing region", "®ion=missing", nil, nil, 400}, + {"timeout", "", nil, context.DeadlineExceeded, 503}, + {"database error", "", nil, errors.New("private detail"), 500}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := signalReader{stubReader: stubReader{getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + switch slug { + case "empty": + return &api.Region{}, nil + case "missing": + return nil, errors.New("not found") + default: + return &api.Region{IATAs: []string{"YVR"}}, nil + } + }}, read: func(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { + t.Fatal("missing bounded query deadline") + } + if since.UnixMilli() != 0 || until.UnixMilli() != 2592000000 || !reflect.DeepEqual(iatas, tc.iatas) { + t.Fatalf("unexpected arguments: %v %v %v", since, until, iatas) + } + return &api.SignalStats{Until: until.UnixMilli(), Receptions: 5, Hourly: []api.SignalHour{}}, tc.err + }} + w := httptest.NewRecorder() + StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) + if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + if tc.status == 200 { + var got api.SignalStats + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || got.Receptions != 5 || got.SNR.Average != nil { + t.Fatalf("invalid response: %+v %v", got, err) + } + } + }) + } +} + +func TestSignalCancellationReachesReader(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + called := false + reader := signalReader{read: func(ctx context.Context, _ time.Time, _ time.Time, _ []string) (*api.SignalStats, error) { + called = true + return nil, ctx.Err() + }} + w := httptest.NewRecorder() + StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) + if !called || w.Body.Len() != 0 { + t.Fatalf("cancellation: called=%v body=%s", called, w.Body.String()) + } +} diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index f65bdc04..56b7b0a8 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -30,6 +30,7 @@ func StatsRouter(reader api.Reader) http.Handler { r := chi.NewRouter() r.Get("/overview", getStatsOverview(reader)) r.Get("/observations", getStatsObservations(reader)) + r.Get("/signal", getSignalStats(reader)) r.Get("/observer-comparison", getObserverComparison(reader)) r.Get("/payload-breakdown", getStatsPayloadBreakdown(reader)) r.Get("/top-nodes", getStatsTopNodes(reader)) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 559200a0..644d6c03 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -66,6 +66,10 @@ type stubReader struct { getNodesByIDs func(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]*api.ResolvedNode, error) } +func (s stubReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { + return nil, nil +} + func (s stubReader) GetObserverComparison(ctx context.Context, a, b uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { if s.getObserverComparison != nil { return s.getObserverComparison(ctx, a, b, since, until, iatas) diff --git a/internal/api/reader.go b/internal/api/reader.go index 1b90b75c..90593549 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -161,6 +161,9 @@ type Reader interface { // since defines the start of the window; pass zero time for default (last 7 days). GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]ObservationPoint, error) + // GetSignalStats aggregates retained reception readings in [since, until). + GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*SignalStats, error) + // GetStatsPayloadBreakdown returns observation counts grouped by payload type. // Pass nil for iatas to return stats across all IATAs. // since defines the start of the window; pass zero time for default (last 24h). diff --git a/internal/api/signal.go b/internal/api/signal.go new file mode 100644 index 00000000..f520c057 --- /dev/null +++ b/internal/api/signal.go @@ -0,0 +1,41 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +// SignalStats describes retained reception readings in [Since, Until). +// A reception is a stored observation, not a unique packet or a lost-packet estimate. +type SignalStats struct { + Since int64 `json:"since"` + Until int64 `json:"until"` + Receptions int64 `json:"receptions"` + SNR SignalMetric `json:"snr"` + RSSI SignalMetric `json:"rssi"` + Hourly []SignalHour `json:"hourly"` +} + +// SignalMetric counts available finite samples; Average is null without samples. +type SignalMetric struct { + Samples int64 `json:"samples"` + Average *float64 `json:"average"` + Histogram []SignalBin `json:"histogram"` +} + +// SignalBin includes Lower and excludes Upper. Null denotes an unbounded end. +// SNR uses 5 dB bins from -30 to 30; RSSI uses 10 dBm bins from -140 to 0, +// both with underflow and overflow bins. These are display bins, not quality ratings. +type SignalBin struct { + Lower *float64 `json:"lower"` + Upper *float64 `json:"upper"` + Count int64 `json:"count"` +} + +// SignalHour is a UTC hour clipped to the requested window. Missing hours are omitted. +type SignalHour struct { + Hour int64 `json:"hour"` + Receptions int64 `json:"receptions"` + SNRSamples int64 `json:"snrSamples"` + SNRAverage *float64 `json:"snrAverage"` + RSSISamples int64 `json:"rssiSamples"` + RSSIAverage *float64 `json:"rssiAverage"` +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index e39ccc03..4d7ec1b6 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -31,6 +31,10 @@ type stubReader struct { calls int } +func (s *stubReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { + return nil, s.err +} + func (s *stubReader) ListIATAs(_ context.Context) ([]api.IATA, error) { s.calls++ return s.iatas, s.err diff --git a/internal/cache/reader.go b/internal/cache/reader.go index aa4f402c..5d846c6b 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -29,6 +29,7 @@ const ( keyScopeByNamePrefix = "beacon:scope:name:" keyStatsOverviewPrefix = "beacon:stats:overview:" keyStatsObservationsPrefix = "beacon:stats:observations:" + keySignalStatsPrefix = "beacon:stats:signal:" keyStatsBreakdownPrefix = "beacon:stats:breakdown:" keyStatsTopNodesPrefix = "beacon:stats:top-nodes:" keyStatsTopObsPrefix = "beacon:stats:top-observers:" diff --git a/internal/cache/signal.go b/internal/cache/signal.go new file mode 100644 index 00000000..d1dfaa51 --- /dev/null +++ b/internal/cache/signal.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func (cr *CachedReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + sorted := append([]string{}, iatas...) + slices.Sort(sorted) + sorted = slices.Compact(sorted) + segment, _ := json.Marshal(sorted) // [] and [""] must remain distinct (global vs empty region). + key := fmt.Sprintf("%s%s:%d:%d", keySignalStatsPrefix, segment, since.UnixMilli(), until.UnixMilli()) + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.SignalStats, error) { + return cr.inner.GetSignalStats(ctx, since, until, iatas) + }) +} diff --git a/internal/cache/signal_test.go b/internal/cache/signal_test.go new file mode 100644 index 00000000..89b739b6 --- /dev/null +++ b/internal/cache/signal_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type signalStatsReader struct { + api.Reader + calls int64 +} + +func (r *signalStatsReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { + r.calls++ + return &api.SignalStats{Receptions: r.calls}, nil +} + +func TestSignalCacheSeparatesWindowAndRegion(t *testing.T) { + c, mr := newTestClient(t) + inner := &signalStatsReader{} + r := NewCachedReader(inner, c, CacheTTLs{Stats: time.Minute}) + for _, tc := range []struct { + iatas []string + since, until, want int64 + }{ + {nil, 0, 10, 1}, {[]string{}, 0, 10, 1}, {[]string{""}, 0, 10, 2}, {[]string{"YVR"}, 0, 10, 3}, + {[]string{"YVR", "YYJ"}, 0, 10, 4}, {[]string{"YYJ", "YVR", "YYJ"}, 0, 10, 4}, + {nil, 1, 10, 5}, {nil, 0, 11, 6}, {nil, 0, 10, 1}, + } { + before := slices.Clone(tc.iatas) + got, err := r.GetSignalStats(context.Background(), time.UnixMilli(tc.since), time.UnixMilli(tc.until), tc.iatas) + if err != nil || got.Receptions != tc.want || !slices.Equal(before, tc.iatas) { + t.Fatalf("%+v: %+v %v", tc, got, err) + } + } + mr.FastForward(time.Minute) + got, err := r.GetSignalStats(context.Background(), time.UnixMilli(0), time.UnixMilli(10), nil) + if err != nil || got.Receptions != 7 { + t.Fatalf("expiry: %+v %v", got, err) + } +} From d4ee5f69a155193ffa8c1a576f9983f3ee7162f1 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 21:44:33 -0400 Subject: [PATCH 08/12] perf(stats): retain regional signal index access for prepared queries --- db/queries/signal.sql | 16 ++++++++++++---- db/signal_integration_test.go | 9 +++++++++ db/sqlc/querier.go | 2 ++ db/sqlc/signal.sql.go | 16 ++++++++++++---- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/db/queries/signal.sql b/db/queries/signal.sql index d247ea58..44fc9f2d 100644 --- a/db/queries/signal.sql +++ b/db/queries/signal.sql @@ -2,16 +2,24 @@ -- One reception scan supplies totals, hourly means, and two independent histograms. -- The zero/zero pair is Beacon's existing unavailable-reading sentinel. A real -- zero SNR with nonzero RSSI remains valid. Non-finite SNR never reaches JSON. -WITH samples AS ( +-- Separate global/filtered branches retain index use after pgx adopts a generic +-- prepared plan. Match the indexed bpchar column without casting each stored IATA. +WITH readings AS ( + SELECT heard_at, snr, rssi FROM packet_observations + WHERE heard_at >= @since::timestamptz AND heard_at < @until::timestamptz + AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT heard_at, snr, rssi FROM packet_observations + WHERE heard_at >= @since::timestamptz AND heard_at < @until::timestamptz + AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) +), samples AS ( SELECT heard_at, CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) AND snr > '-Infinity'::real AND snr < 'Infinity'::real THEN snr::double precision END AS snr, CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) THEN rssi::double precision END AS rssi - FROM packet_observations - WHERE heard_at >= @since::timestamptz AND heard_at < @until::timestamptz - AND (COALESCE(cardinality(@iatas::text[]), 0) = 0 OR iata = ANY(@iatas::text[])) + FROM readings ), binned AS ( SELECT date_trunc('hour', heard_at, 'UTC') AS hour, snr, rssi, width_bucket(snr, -30, 30, 12) AS snr_bin, diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go index 371e6bbb..05440e6f 100644 --- a/db/signal_integration_test.go +++ b/db/signal_integration_test.go @@ -111,6 +111,15 @@ SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FRO } }) } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err = tx.Exec(ctx, "SET LOCAL plan_cache_mode="+mode); err != nil { + t.Fatal(err) + } + got, err := store.GetSignalStats(ctx, since, until, []string{"YVR"}) + if err != nil || got.Receptions != 13 || got.SNR.Samples != 5 || got.RSSI.Samples != 8 { + t.Fatalf("%s: %+v %v", mode, got, err) + } + } if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { t.Fatal(err) } diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 1a7c6fc0..3b80fe3a 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -93,6 +93,8 @@ type Querier interface { // One reception scan supplies totals, hourly means, and two independent histograms. // The zero/zero pair is Beacon's existing unavailable-reading sentinel. A real // zero SNR with nonzero RSSI remains valid. Non-finite SNR never reaches JSON. + // Separate global/filtered branches retain index use after pgx adopts a generic + // prepared plan. Match the indexed bpchar column without casting each stored IATA. GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds // the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's diff --git a/db/sqlc/signal.sql.go b/db/sqlc/signal.sql.go index 32dd4064..dd7e3eaf 100644 --- a/db/sqlc/signal.sql.go +++ b/db/sqlc/signal.sql.go @@ -12,16 +12,22 @@ import ( ) const getSignalStats = `-- name: GetSignalStats :many -WITH samples AS ( +WITH readings AS ( + SELECT heard_at, snr, rssi FROM packet_observations + WHERE heard_at >= $1::timestamptz AND heard_at < $2::timestamptz + AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT heard_at, snr, rssi FROM packet_observations + WHERE heard_at >= $1::timestamptz AND heard_at < $2::timestamptz + AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) +), samples AS ( SELECT heard_at, CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) AND snr > '-Infinity'::real AND snr < 'Infinity'::real THEN snr::double precision END AS snr, CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) THEN rssi::double precision END AS rssi - FROM packet_observations - WHERE heard_at >= $1::timestamptz AND heard_at < $2::timestamptz - AND (COALESCE(cardinality($3::text[]), 0) = 0 OR iata = ANY($3::text[])) + FROM readings ), binned AS ( SELECT date_trunc('hour', heard_at, 'UTC') AS hour, snr, rssi, width_bucket(snr, -30, 30, 12) AS snr_bin, @@ -63,6 +69,8 @@ type GetSignalStatsRow struct { // One reception scan supplies totals, hourly means, and two independent histograms. // The zero/zero pair is Beacon's existing unavailable-reading sentinel. A real // zero SNR with nonzero RSSI remains valid. Non-finite SNR never reaches JSON. +// Separate global/filtered branches retain index use after pgx adopts a generic +// prepared plan. Match the indexed bpchar column without casting each stored IATA. func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) { rows, err := q.db.Query(ctx, getSignalStats, arg.Since, arg.Until, arg.Iatas) if err != nil { From ddff0798d95cceea683b30102f493a1274491b0a Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 21:48:02 -0400 Subject: [PATCH 09/12] test(stats): exercise signal HTTP response against PostgreSQL --- db/signal_integration_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go index 05440e6f..acb9037a 100644 --- a/db/signal_integration_test.go +++ b/db/signal_integration_test.go @@ -7,12 +7,14 @@ import ( "context" "encoding/json" "math" + "net/http/httptest" "os" "testing" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/jackc/pgx/v5" ) @@ -120,6 +122,13 @@ SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FRO t.Fatalf("%s: %+v %v", mode, got, err) } } + w := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/signal?since=1767227400000&until=1767241800000&iatas=YVR", nil).WithContext(ctx) + handlers.StatsRouter(store).ServeHTTP(w, request) + var response api.SignalStats + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 13 || response.SNR.Samples != 5 { + t.Fatalf("PostgreSQL HTTP response: status=%d body=%s error=%v", w.Code, w.Body.String(), err) + } if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { t.Fatal(err) } From cc3fa7ce225e1ed3e94734c7df97b536d3701090 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 20 Sep 2026 12:10:12 -0400 Subject: [PATCH 10/12] perf(stats): serve signal analytics from hourly materialized views --- db/migrations/035_mv_signal_stats.sql | 32 ++++++++++++++ db/queries/signal.sql | 55 ++++++++++------------- db/signal.go | 8 +++- db/signal_integration_test.go | 24 +++++++--- db/sqlc/mock/querier.go | 14 ++++++ db/sqlc/models.go | 13 ++++++ db/sqlc/querier.go | 8 ++-- db/sqlc/signal.sql.go | 61 ++++++++++++-------------- db/stats_migration_integration_test.go | 33 ++++++++++++++ docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 8 ++-- internal/api/handlers/regions.go | 19 ++++++++ internal/api/handlers/signal.go | 18 +++----- internal/api/handlers/signal_test.go | 13 ++++++ internal/api/signal.go | 5 ++- internal/background/tasks.go | 2 + internal/background/tasks_test.go | 9 +++- 18 files changed, 228 insertions(+), 98 deletions(-) create mode 100644 db/migrations/035_mv_signal_stats.sql create mode 100644 db/stats_migration_integration_test.go diff --git a/db/migrations/035_mv_signal_stats.sql b/db/migrations/035_mv_signal_stats.sql new file mode 100644 index 00000000..59a9943e --- /dev/null +++ b/db/migrations/035_mv_signal_stats.sql @@ -0,0 +1,32 @@ +-- Reception-IATA rollups include legacy NULL payload types and exclude non-finite +-- SNR. mv_observer_activity_hourly has a different grain/filter, so reusing its +-- means would disagree with these histograms and the reception coverage counts. +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_signal_stats_hourly AS +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_signal_stats_hourly + ON mv_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin); +CREATE INDEX IF NOT EXISTS idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly (hour); diff --git a/db/queries/signal.sql b/db/queries/signal.sql index 44fc9f2d..24bb800c 100644 --- a/db/queries/signal.sql +++ b/db/queries/signal.sql @@ -1,40 +1,31 @@ -- name: GetSignalStats :many --- One reception scan supplies totals, hourly means, and two independent histograms. --- The zero/zero pair is Beacon's existing unavailable-reading sentinel. A real --- zero SNR with nonzero RSSI remains valid. Non-finite SNR never reaches JSON. --- Separate global/filtered branches retain index use after pgx adopts a generic --- prepared plan. Match the indexed bpchar column without casting each stored IATA. +-- Read compact hourly snapshots, never observations on an HTTP request. +-- Weight averages by sample counts instead of averaging regional/hourly means. WITH readings AS ( - SELECT heard_at, snr, rssi FROM packet_observations - WHERE heard_at >= @since::timestamptz AND heard_at < @until::timestamptz + SELECT * FROM mv_signal_stats_hourly + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 UNION ALL - SELECT heard_at, snr, rssi FROM packet_observations - WHERE heard_at >= @since::timestamptz AND heard_at < @until::timestamptz + SELECT * FROM mv_signal_stats_hourly + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) -), samples AS ( - SELECT heard_at, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM readings -), binned AS ( - SELECT date_trunc('hour', heard_at, 'UTC') AS hour, snr, rssi, - width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples ) -SELECT grouping(hour, snr_bin, rssi_bin)::integer AS kind, +SELECT (3 + 4 * grouping(hour))::integer AS kind, COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(avg(snr), 0)::double precision AS snr_average, - count(rssi)::bigint AS rssi_samples, - COALESCE(avg(rssi), 0)::double precision AS rssi_average -FROM binned -GROUP BY GROUPING SETS ((), (hour), (snr_bin), (rssi_bin)) + -1::integer AS snr_bin, -1::integer AS rssi_bin, + COALESCE(sum(receptions), 0)::bigint AS receptions, + COALESCE(sum(snr_samples), 0)::bigint AS snr_samples, + COALESCE(sum(snr_sum) / NULLIF(sum(snr_samples), 0), 0)::double precision AS snr_average, + COALESCE(sum(rssi_samples), 0)::bigint AS rssi_samples, + COALESCE(sum(rssi_sum) / NULLIF(sum(rssi_samples), 0), 0)::double precision AS rssi_average +FROM readings WHERE kind = 3 +GROUP BY GROUPING SETS ((), (hour)) +UNION ALL +SELECT (kind + 4)::integer, 'epoch'::timestamptz, snr_bin, rssi_bin, + sum(receptions)::bigint, 0::bigint, 0::double precision, 0::bigint, 0::double precision +FROM readings WHERE kind IN (1, 2) +GROUP BY kind, snr_bin, rssi_bin ORDER BY kind, hour, snr_bin, rssi_bin; + +-- name: RefreshSignalStats :exec +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_signal_stats_hourly; diff --git a/db/signal.go b/db/signal.go index da346751..9a7b1a66 100644 --- a/db/signal.go +++ b/db/signal.go @@ -33,11 +33,11 @@ func (s *Store) GetSignalStats(ctx context.Context, since, until time.Time, iata SNRSamples: row.SnrSamples, SNRAverage: signalAverage(row.SnrAverage, row.SnrSamples), RSSISamples: row.RssiSamples, RSSIAverage: signalAverage(row.RssiAverage, row.RssiSamples)}) case 5: - if row.SnrBin >= 0 { + if row.SnrBin >= 0 && int(row.SnrBin) < len(stats.SNR.Histogram) { stats.SNR.Histogram[row.SnrBin].Count = row.Receptions } case 6: - if row.RssiBin >= 0 { + if row.RssiBin >= 0 && int(row.RssiBin) < len(stats.RSSI.Histogram) { stats.RSSI.Histogram[row.RssiBin].Count = row.Receptions } } @@ -45,6 +45,10 @@ func (s *Store) GetSignalStats(ctx context.Context, since, until time.Time, iata return stats, nil } +func (s *Store) RefreshSignalStats(ctx context.Context) error { + return s.q.RefreshSignalStats(ctx) +} + func signalAverage(value float64, count int64) *float64 { if count == 0 { return nil diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go index acb9037a..8af97f8e 100644 --- a/db/signal_integration_test.go +++ b/db/signal_integration_test.go @@ -6,6 +6,7 @@ package db import ( "context" "encoding/json" + "fmt" "math" "net/http/httptest" "os" @@ -18,7 +19,7 @@ import ( "github.com/jackc/pgx/v5" ) -// Only temporary tables are written; no migrated schema or production rows are needed. +// Real view DDL and fixture rows are isolated in a rolled-back private schema. func TestSignalPostgres(t *testing.T) { dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") if dsn == "" { @@ -36,9 +37,10 @@ func TestSignalPostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) + isolateStatsSchema(t, ctx, tx) _, err = tx.Exec(ctx, ` SET LOCAL TIME ZONE 'America/Vancouver'; -CREATE TEMP TABLE packet_observations (heard_at timestamptz NOT NULL, iata char(3) NOT NULL, snr real, rssi smallint) ON COMMIT DROP; +CREATE TABLE packet_observations (heard_at timestamptz NOT NULL, iata char(3) NOT NULL, snr real, rssi smallint); INSERT INTO packet_observations SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FROM (VALUES ('00:30:00','YVR','0',-100), ('00:45:00','YVR','0',0), ('00:59:00','YVR',NULL,NULL), @@ -46,14 +48,19 @@ SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FRO ('01:30:00','YVR','5',NULL), ('01:40:00','YVR',NULL,-90), ('01:50:00','YVR','NaN',-80), ('02:00:00','YVR','Infinity',-70), ('02:10:00','YVR','-Infinity',-60), ('02:20:00','YYJ','10',-50), ('03:00:00','YYZ','-5',-110), - ('00:29:59','YVR','20',-40), ('04:30:00','YVR','20',-40), + ('04:00:00','YVR','20',-40), ('04:30:00','YVR','20',-40), ('03:10:00','YVR','0',NULL), ('03:20:00','YVR',NULL,0) ) v(at,iata,snr,rssi);`) if err != nil { t.Fatal(err) } + since := time.Now().UTC().Truncate(24 * time.Hour).Add(-48 * time.Hour) + if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=heard_at+($1::timestamptz-'2026-01-01 00:00+00'::timestamptz)", since); err != nil { + t.Fatal(err) + } + applyStatsMigration(t, ctx, tx, "035_mv_signal_stats.sql") + applyStatsMigration(t, ctx, tx, "035_mv_signal_stats.sql") // interrupted journal retry store := &Store{q: sqlc.New(tx)} - since := time.Date(2026, 1, 1, 0, 30, 0, 0, time.UTC) until := since.Add(4 * time.Hour) for _, tc := range []struct { name string @@ -123,7 +130,7 @@ SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FRO } } w := httptest.NewRecorder() - request := httptest.NewRequest("GET", "/signal?since=1767227400000&until=1767241800000&iatas=YVR", nil).WithContext(ctx) + request := httptest.NewRequest("GET", fmt.Sprintf("/signal?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx) handlers.StatsRouter(store).ServeHTTP(w, request) var response api.SignalStats if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 13 || response.SNR.Samples != 5 { @@ -132,6 +139,13 @@ SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FRO if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { t.Fatal(err) } + stale, err := store.GetSignalStats(ctx, since, until, nil) + if err != nil || stale.Receptions != 15 { + t.Fatalf("snapshot lost: %+v %v", stale, err) + } + if err := store.RefreshSignalStats(ctx); err != nil { + t.Fatal(err) + } got, err := store.GetSignalStats(ctx, since, until, nil) if err != nil || got.Receptions != 0 || got.Hourly == nil || len(got.Hourly) != 0 || got.SNR.Average != nil { t.Fatalf("empty: %+v %v", got, err) diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 843fdc22..add69431 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -1290,6 +1290,20 @@ func (mr *MockQuerierMockRecorder) RefreshRadioPresets(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshRadioPresets", reflect.TypeOf((*MockQuerier)(nil).RefreshRadioPresets), ctx) } +// RefreshSignalStats mocks base method. +func (m *MockQuerier) RefreshSignalStats(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RefreshSignalStats", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// RefreshSignalStats indicates an expected call of RefreshSignalStats. +func (mr *MockQuerierMockRecorder) RefreshSignalStats(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshSignalStats", reflect.TypeOf((*MockQuerier)(nil).RefreshSignalStats), ctx) +} + // RefreshTopAdvertisers mocks base method. func (m *MockQuerier) RefreshTopAdvertisers(ctx context.Context) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 80ab12bb..074994d4 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -112,6 +112,19 @@ type MvRadioPreset struct { Count int64 `json:"count"` } +type MvSignalStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrSum float64 `json:"snr_sum"` + RssiSamples int64 `json:"rssi_samples"` + RssiSum float64 `json:"rssi_sum"` +} + type MvTopAdvertisersByIatum struct { Iata string `json:"iata"` NodeID uuid.UUID `json:"node_id"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 3b80fe3a..4256c4e1 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -90,11 +90,8 @@ type Querier interface { // whose observations have expired; the filtered aggregates are empty in that case. GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) - // One reception scan supplies totals, hourly means, and two independent histograms. - // The zero/zero pair is Beacon's existing unavailable-reading sentinel. A real - // zero SNR with nonzero RSSI remains valid. Non-finite SNR never reaches JSON. - // Separate global/filtered branches retain index use after pgx adopts a generic - // prepared plan. Match the indexed bpchar column without casting each stored IATA. + // Read compact hourly snapshots, never observations on an HTTP request. + // Weight averages by sample counts instead of averaging regional/hourly means. GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds // the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's @@ -220,6 +217,7 @@ type Querier interface { RefreshObserverActivity(ctx context.Context) error RefreshPayloadBreakdown(ctx context.Context) error RefreshRadioPresets(ctx context.Context) error + RefreshSignalStats(ctx context.Context) error RefreshTopAdvertisers(ctx context.Context) error RefreshTopNodes(ctx context.Context) error RefreshTopObservers(ctx context.Context) error diff --git a/db/sqlc/signal.sql.go b/db/sqlc/signal.sql.go index dd7e3eaf..8a490490 100644 --- a/db/sqlc/signal.sql.go +++ b/db/sqlc/signal.sql.go @@ -13,38 +13,29 @@ import ( const getSignalStats = `-- name: GetSignalStats :many WITH readings AS ( - SELECT heard_at, snr, rssi FROM packet_observations - WHERE heard_at >= $1::timestamptz AND heard_at < $2::timestamptz + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM mv_signal_stats_hourly + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND COALESCE(cardinality($3::bpchar[]), 0) = 0 UNION ALL - SELECT heard_at, snr, rssi FROM packet_observations - WHERE heard_at >= $1::timestamptz AND heard_at < $2::timestamptz + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM mv_signal_stats_hourly + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) -), samples AS ( - SELECT heard_at, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM readings -), binned AS ( - SELECT date_trunc('hour', heard_at, 'UTC') AS hour, snr, rssi, - width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples ) -SELECT grouping(hour, snr_bin, rssi_bin)::integer AS kind, +SELECT (3 + 4 * grouping(hour))::integer AS kind, COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(avg(snr), 0)::double precision AS snr_average, - count(rssi)::bigint AS rssi_samples, - COALESCE(avg(rssi), 0)::double precision AS rssi_average -FROM binned -GROUP BY GROUPING SETS ((), (hour), (snr_bin), (rssi_bin)) + -1::integer AS snr_bin, -1::integer AS rssi_bin, + COALESCE(sum(receptions), 0)::bigint AS receptions, + COALESCE(sum(snr_samples), 0)::bigint AS snr_samples, + COALESCE(sum(snr_sum) / NULLIF(sum(snr_samples), 0), 0)::double precision AS snr_average, + COALESCE(sum(rssi_samples), 0)::bigint AS rssi_samples, + COALESCE(sum(rssi_sum) / NULLIF(sum(rssi_samples), 0), 0)::double precision AS rssi_average +FROM readings WHERE kind = 3 +GROUP BY GROUPING SETS ((), (hour)) +UNION ALL +SELECT (kind + 4)::integer, 'epoch'::timestamptz, snr_bin, rssi_bin, + sum(receptions)::bigint, 0::bigint, 0::double precision, 0::bigint, 0::double precision +FROM readings WHERE kind IN (1, 2) +GROUP BY kind, snr_bin, rssi_bin ORDER BY kind, hour, snr_bin, rssi_bin ` @@ -66,11 +57,8 @@ type GetSignalStatsRow struct { RssiAverage float64 `json:"rssi_average"` } -// One reception scan supplies totals, hourly means, and two independent histograms. -// The zero/zero pair is Beacon's existing unavailable-reading sentinel. A real -// zero SNR with nonzero RSSI remains valid. Non-finite SNR never reaches JSON. -// Separate global/filtered branches retain index use after pgx adopts a generic -// prepared plan. Match the indexed bpchar column without casting each stored IATA. +// Read compact hourly snapshots, never observations on an HTTP request. +// Weight averages by sample counts instead of averaging regional/hourly means. func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) { rows, err := q.db.Query(ctx, getSignalStats, arg.Since, arg.Until, arg.Iatas) if err != nil { @@ -100,3 +88,12 @@ func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) } return items, nil } + +const refreshSignalStats = `-- name: RefreshSignalStats :exec +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_signal_stats_hourly +` + +func (q *Queries) RefreshSignalStats(ctx context.Context) error { + _, err := q.db.Exec(ctx, refreshSignalStats) + return err +} diff --git a/db/stats_migration_integration_test.go b/db/stats_migration_integration_test.go new file mode 100644 index 00000000..cfe3f94f --- /dev/null +++ b/db/stats_migration_integration_test.go @@ -0,0 +1,33 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// Views cannot reference temporary tables. Keep fixtures and real migration DDL +// in a unique schema inside the caller's rolled-back transaction instead. +func isolateStatsSchema(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + schema := pgx.Identifier{"stats_test_" + uuid.NewString()}.Sanitize() + if _, err := tx.Exec(ctx, "CREATE SCHEMA "+schema+"; SET LOCAL search_path TO "+schema); err != nil { + t.Fatal(err) + } +} + +func applyStatsMigration(t *testing.T, ctx context.Context, tx pgx.Tx, name string) { + t.Helper() + migration, err := migrationFiles.ReadFile("migrations/" + name) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, string(migration)); err != nil { + t.Fatal(err) + } +} diff --git a/docs/docs.go b/docs/docs.go index 7be9b2e9..95a95b60 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2676,7 +2676,7 @@ const docTemplate = `{ }, "/stats/signal": { "get": { - "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. UTC hourly buckets are clipped to the requested window; absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", "produces": [ "application/json" ], diff --git a/docs/swagger.json b/docs/swagger.json index 6369a8a0..2460fe3d 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2674,7 +2674,7 @@ }, "/stats/signal": { "get": { - "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. UTC hourly buckets are clipped to the requested window; absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", "produces": [ "application/json" ], diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9e478176..be2d5b2b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -3159,9 +3159,11 @@ paths: 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are - lower-inclusive/upper-exclusive, with null for unbounded ends. UTC hourly - buckets are clipped to the requested window; absent hours are omitted. These - last-hop readings do not measure end-to-end quality or packet loss. + lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries + round down to UTC hours and the response reports that effective window. Reads + use hourly materialized snapshots refreshed by background.view_refresh; the + current partial hour is excluded and absent hours are omitted. These last-hop + readings do not measure end-to-end quality or packet loss. parameters: - description: Inclusive start, epoch milliseconds (0 through 253402300799999) in: query diff --git a/internal/api/handlers/regions.go b/internal/api/handlers/regions.go index b9bfe5c5..0b0f4008 100644 --- a/internal/api/handlers/regions.go +++ b/internal/api/handlers/regions.go @@ -9,6 +9,7 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" @@ -74,6 +75,24 @@ func getRegion(reader api.Reader) http.HandlerFunc { } } +// parseStatsWindow rounds both endpoints down to UTC hours. Polls within an +// hour share a cache key; the response reports the effective window. A valid +// sub-hour request can contain no complete buckets and return an empty result. +func parseStatsWindow(r *http.Request) (time.Time, time.Time, error) { + q := r.URL.Query() + for _, key := range []string{"since", "until"} { + if len(q[key]) != 1 || q.Get(key) == "" { + return time.Time{}, time.Time{}, fmt.Errorf("%s must be supplied exactly once", key) + } + } + since, errSince := strconv.ParseInt(q.Get("since"), 10, 64) + until, errUntil := strconv.ParseInt(q.Get("until"), 10, 64) + if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 || until-since > int64((30*24*time.Hour)/time.Millisecond) { + return time.Time{}, time.Time{}, fmt.Errorf("since and until must be epoch milliseconds between 0 and 253402300799999, with a positive window of at most 30 days") + } + return time.UnixMilli(since).UTC().Truncate(time.Hour), time.UnixMilli(until).UTC().Truncate(time.Hour), nil +} + // parseIATAs splits a comma-separated iatas query param and uppercases each value. func parseIATAs(r *http.Request) []string { raw := r.URL.Query().Get("iatas") diff --git a/internal/api/handlers/signal.go b/internal/api/handlers/signal.go index b09f9999..b1029a98 100644 --- a/internal/api/handlers/signal.go +++ b/internal/api/handlers/signal.go @@ -8,7 +8,6 @@ import ( "errors" "log/slog" "net/http" - "strconv" "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" @@ -17,7 +16,7 @@ import ( // getSignalStats godoc // // @Summary Reception signal distributions and hourly trends -// @Description Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. UTC hourly buckets are clipped to the requested window; absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss. +// @Description Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss. // @Tags Stats // @Produce json // @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" @@ -33,16 +32,9 @@ import ( func getSignalStats(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() - for _, key := range []string{"since", "until"} { - if len(q[key]) != 1 || q.Get(key) == "" { - respondError(w, http.StatusBadRequest, key+" must be supplied exactly once") - return - } - } - since, errSince := strconv.ParseInt(q.Get("since"), 10, 64) - until, errUntil := strconv.ParseInt(q.Get("until"), 10, 64) - if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 || until-since > int64((30*24*time.Hour)/time.Millisecond) { - respondError(w, http.StatusBadRequest, "since and until must be epoch milliseconds between 0 and 253402300799999, with a positive window of at most 30 days") + since, until, err := parseStatsWindow(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) return } ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) @@ -59,7 +51,7 @@ func getSignalStats(reader api.Reader) http.HandlerFunc { iatas = []string{""} } } - stats, err := reader.GetSignalStats(ctx, time.UnixMilli(since), time.UnixMilli(until), iatas) + stats, err := reader.GetSignalStats(ctx, since, until, iatas) switch { case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): respondError(w, http.StatusServiceUnavailable, "signal query timed out; try a shorter time period") diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go index 8625afde..405db1ca 100644 --- a/internal/api/handlers/signal_test.go +++ b/internal/api/handlers/signal_test.go @@ -98,3 +98,16 @@ func TestSignalCancellationReachesReader(t *testing.T) { t.Fatalf("cancellation: called=%v body=%s", called, w.Body.String()) } } + +func TestStatsWindowSnapsPollingTimes(t *testing.T) { + for _, query := range []string{"since=123&until=86400123", "since=3599999&until=89999999"} { + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?"+query, nil)) + if err != nil || since.UnixMilli() != 0 || until.UnixMilli() != 86400000 { + t.Fatalf("%v %v %v", since, until, err) + } + } + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?since=1&until=2", nil)) + if err != nil || !since.Equal(until) { + t.Fatal("sub-hour normalization") + } +} diff --git a/internal/api/signal.go b/internal/api/signal.go index f520c057..1514dce4 100644 --- a/internal/api/signal.go +++ b/internal/api/signal.go @@ -3,7 +3,8 @@ package api -// SignalStats describes retained reception readings in [Since, Until). +// SignalStats describes a materialized reception snapshot in [Since, Until). +// Since and Until are the effective UTC-hour boundaries, rounded down from input. // A reception is a stored observation, not a unique packet or a lost-packet estimate. type SignalStats struct { Since int64 `json:"since"` @@ -30,7 +31,7 @@ type SignalBin struct { Count int64 `json:"count"` } -// SignalHour is a UTC hour clipped to the requested window. Missing hours are omitted. +// SignalHour is a complete UTC bucket in the effective window. Missing hours are omitted. type SignalHour struct { Hour int64 `json:"hour"` Receptions int64 `json:"receptions"` diff --git a/internal/background/tasks.go b/internal/background/tasks.go index b29760ad..0290df8a 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -22,6 +22,7 @@ type viewRefresher interface { RefreshTopAdvertisers(context.Context) error RefreshRadioPresets(context.Context) error RefreshObserverActivity(context.Context) error + RefreshSignalStats(context.Context) error } // ViewRefreshTask returns a Task that refreshes all materialized views. @@ -43,6 +44,7 @@ func ViewRefreshTask(store viewRefresher, interval time.Duration) Task { {"top advertisers", store.RefreshTopAdvertisers}, {"radio presets", store.RefreshRadioPresets}, {"observer activity", store.RefreshObserverActivity}, + {"signal stats", store.RefreshSignalStats}, } { if err := view.refresh(ctx); err != nil { errs = append(errs, fmt.Errorf("%s: %w", view.name, err)) diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go index af1dae22..2c4526dd 100644 --- a/internal/background/tasks_test.go +++ b/internal/background/tasks_test.go @@ -119,6 +119,10 @@ func (s *refreshStub) RefreshObserverActivity(ctx context.Context) error { return s.refresh(ctx, "observer activity") } +func (s *refreshStub) RefreshSignalStats(ctx context.Context) error { + return s.refresh(ctx, "signal stats") +} + func TestViewRefreshTask(t *testing.T) { first, second := errors.New("first failure"), errors.New("second failure") for _, tc := range []struct { @@ -127,13 +131,14 @@ func TestViewRefreshTask(t *testing.T) { }{ {"success", nil}, {"partial failure", map[string]error{"hourly stats": first, "top talkers": second}}, + {"signal failure", map[string]error{"signal stats": first}}, } { t.Run(tc.name, func(t *testing.T) { store := &refreshStub{errs: tc.errs} task := ViewRefreshTask(store, time.Minute) err := task.Run(context.Background()) - if len(store.calls) != 8 { - t.Fatalf("refreshed %d views, want 8", len(store.calls)) + if len(store.calls) != 9 { + t.Fatalf("refreshed %d views, want 9", len(store.calls)) } if len(tc.errs) == 0 && err != nil { t.Fatal(err) From 274bf16b9bbf7af37fd99ad5b159e0777d0c11cc Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 20 Sep 2026 12:20:21 -0400 Subject: [PATCH 11/12] test(stats): keep dated signal fixtures independent of daylight saving --- db/signal_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go index 8af97f8e..7eef6bbf 100644 --- a/db/signal_integration_test.go +++ b/db/signal_integration_test.go @@ -55,7 +55,7 @@ SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FRO t.Fatal(err) } since := time.Now().UTC().Truncate(24 * time.Hour).Add(-48 * time.Hour) - if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=heard_at+($1::timestamptz-'2026-01-01 00:00+00'::timestamptz)", since); err != nil { + if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { t.Fatal(err) } applyStatsMigration(t, ctx, tx, "035_mv_signal_stats.sql") From 0ba847f2ce1f90afc36a2ee087cfca04c2a6888b Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 20 Sep 2026 12:25:28 -0400 Subject: [PATCH 12/12] fix(stats): materialize only complete UTC hours --- db/migrations/035_mv_signal_stats.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/migrations/035_mv_signal_stats.sql b/db/migrations/035_mv_signal_stats.sql index 59a9943e..14b3dd3f 100644 --- a/db/migrations/035_mv_signal_stats.sql +++ b/db/migrations/035_mv_signal_stats.sql @@ -10,7 +10,8 @@ WITH samples AS ( CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) THEN rssi::double precision END AS rssi FROM packet_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') ), binned AS ( SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, width_bucket(rssi, -140, 0, 14) AS rssi_bin