diff --git a/.build/Dockerfile b/.build/Dockerfile index 19e41ba7..7703603a 100644 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -9,7 +9,8 @@ COPY . . RUN go build -o beacon ./cmd/beacon FROM alpine:3.19 -RUN apk add --no-cache ca-certificates tzdata +ARG POSTGRES_CLIENT_MAJOR=16 +RUN apk add --no-cache ca-certificates tzdata postgresql${POSTGRES_CLIENT_MAJOR}-client WORKDIR /app COPY --from=builder /app/beacon . CMD ["./beacon"] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fc14c65d..b8732e9d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,5 +1,5 @@ # Default owners for everything -* @446564 +* @MrAlders0n # EXAMPLE Specific ownership by directory diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 64ec8161..c7bf29c4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -34,5 +34,5 @@ paste logs here ## Additional context -Any other context — config snippets (redact credentials), MQTT broker details, +Any other context: config snippets (redact credentials), MQTT broker details, mesh topology, etc. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 433c4232..f2fdb7f3 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -15,7 +15,7 @@ related to an existing issue or limitation, link it here. ## Proposed solution -How you'd like it to work. API shape, config options, WS event format — as much +How you'd like it to work. API shape, config options, WS event format, as much detail as you have. ## Alternatives considered diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 16463f3b..ef6d6c03 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -23,7 +23,7 @@ issues: `Closes #123` or `Related to #456`. - [ ] API changes include swagger annotations and `swag init` has been run - [ ] `docs/` is committed if swagger was regenerated - [ ] New dependencies are added to `SHOULDERS.md` -- [ ] I have read [CONTRIBUTING.md](CONTRIBUTING.md) +- [ ] I have read [CONTRIBUTING.md](../CONTRIBUTING.md) ## Testing notes diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdb2a3b7..95527aa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,18 @@ on: jobs: build: runs-on: ubuntu-latest + services: + backup-postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: backup-ci-only + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 steps: - uses: actions/checkout@v4 @@ -35,6 +47,37 @@ jobs: - name: Test run: go test ./... + - name: Verify scope refresh concurrency + run: go test -race ./internal/meshmapper ./internal/scopestore + + - name: Verify backup startup against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./cmd/beacon -run '^TestBackupConfigurationPostgres$' -count=1 -v + + - name: Verify stats and endpoint queries against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v + + - name: Verify backup command against PostgreSQL 16 + run: | + CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup + CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup.test" ./internal/backup + CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup-http.test" ./internal/api/router + docker run --rm --network host \ + -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ + -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable \ + -e BEACON_BACKUP_TEST_POSTGRES=1 -e BEACON_BACKUP_TEST_BINARY=/beacon-backup \ + -v "$RUNNER_TEMP/beacon-backup:/beacon-backup:ro" \ + -v "$RUNNER_TEMP/backup.test:/backup.test:ro" \ + postgres:16-alpine /backup.test -test.run '^TestExportPostgres$' -test.v + docker run --rm --network host \ + -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ + -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable -e BEACON_BACKUP_TEST_POSTGRES=1 \ + -v "$RUNNER_TEMP/backup-http.test:/backup-http.test:ro" \ + postgres:16-alpine /backup-http.test -test.run '^TestBackupDownloadPostgres$' -test.v + - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest @@ -54,4 +97,3 @@ jobs: exit 1 fi rm docs/swagger.json.bak - diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 48f459db..70f27ec4 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,10 +38,12 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} labels: | org.opencontainers.image.title=beacon-server - org.opencontainers.image.description=Beacon — MeshCore LoRa mesh observation server + org.opencontainers.image.description=Beacon: MeshCore LoRa mesh observation server org.opencontainers.image.licenses=AGPL-3.0-or-later + flavor: | + latest=false tags: | - type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} type=raw,value=dev,enable=${{ github.ref == 'refs/heads/dev' }} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c8610310..e50faa7d 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -25,7 +25,7 @@ Examples of behaviour that contributes to a positive environment: Examples of unacceptable behaviour: -- Hate speech or discriminatory language or jokes of any kind — including but +- Hate speech or discriminatory language or jokes of any kind, including but not limited to those targeting race, ethnicity, nationality, gender identity or expression, sexual orientation, disability, religion, age, or socioeconomic background @@ -34,7 +34,7 @@ Examples of unacceptable behaviour: - Trolling, insulting or derogatory comments, and personal or political attacks - Public or private harassment of any kind - Deliberate misgendering or use of rejected names -- Publishing others' private information — such as a physical or email address — +- Publishing others' private information, such as a physical or email address, without their explicit permission - Threats of violence or incitement of violence toward any individual or group - Dismissing or minimising reports of harassment or discrimination @@ -55,15 +55,15 @@ when appropriate. ## Scope -This Code of Conduct applies within all project spaces — GitHub issues, pull -requests, discussions, and the MeshCore Canada Discord server — and also when an +This Code of Conduct applies within all project spaces (GitHub issues, pull +requests, discussions, and the MeshCore Canada Discord server) and also when an individual is representing the project in public spaces. ## Reporting Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported by contacting the maintainers directly on the MeshCore Canada Discord -server at https://discord.gg/Gz3KvJx2hf — reach out to **dedskelly** privately. +server at https://discord.gg/Gz3KvJx2hf; reach out to **dedskelly** privately. All reports will be handled with discretion and confidentiality. Maintainers are obligated to respect the privacy and safety of the reporter. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7edf1eb0..46702834 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,55 +6,31 @@ before opening a PR. --- -## Before you start - -**Open or comment on an issue before starting work.** This avoids duplicate -effort and lets maintainers flag if something is already in progress or out of -scope. For small bug fixes a brief comment is fine; for larger features please -discuss the approach first. - -**One thing per PR.** Each pull request should cover one logical change — a bug -fix, a new endpoint, a refactor, a new test. PRs that touch many unrelated parts -of the codebase are hard to review and hard to revert if something goes wrong. - -**No fully AI-generated contributions.** We welcome developers who use AI tools -to assist their work, but PRs should reflect the author's own understanding and -judgement. PRs that appear to be unreviewed AI output may be closed without -further comment. - ---- - -## Branches - -- `main` — stable releases only, protected. Never target this directly. -- `dev` — active development. All PRs target `dev`. - ## Workflow -1. Fork or create a branch from `dev` -2. Make your changes -3. Run the checklist below -4. Open a pull request against `dev` with a clear description of what changed - and why, referencing any related issues - ---- +Branches, the pull request flow, commit message format and the release process are the same +for every Beacon repo and live in +[beacon-docs/CONTRIBUTING.md](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/CONTRIBUTING.md). +In short: open or comment on an issue first, branch from `dev`, keep each PR to one change, +write conventional commit messages, and open the PR against `dev`. Running beacon-web against +a local server is described there too. -## Checklist before opening a PR +Before opening a PR here: ``` go build ./... # must compile gofmt -l . # must be empty (no unformatted files) go vet ./... # no warnings go test ./... # all tests pass -swag init # if you changed any handler or api type (see below) +govulncheck ./... # no reachable vulnerabilities +swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency + # if you changed any handler or api type (see below) ``` --- ## Code style -- Run `gofmt -w .` before committing — CI will fail on unformatted files -- Run `go vet ./...` — no warnings - Follow the existing patterns in each package before introducing new ones - Keep functions small and single-purpose - Prefer explicit error handling over panic @@ -67,26 +43,47 @@ swag init # if you changed any handler or api type (see below) no side effects) should have unit tests. See `internal/hub/hub_test.go`, `internal/api/nodes_test.go`, and `internal/keystore/keystore_test.go` for examples of the style we use. -- Integration tests (requiring a real DB) are not yet required but are welcome. - They will be gated before release. -- Run `go test ./...` before opening a PR. All tests must pass. +- Integration tests (`*_integration_test.go`) need a real PostgreSQL 16 and skip + unless `BEACON_TEST_POSTGRES_DSN` is set. CI runs a selected set; run the ones + touching your change locally: + `BEACON_TEST_POSTGRES_DSN=postgres://... go test ./db/ -run Postgres` - If you are fixing a bug, add a test that would have caught it. --- +## Backup export tests + +CI runs export, restore round-trip, offline verify and truncated-archive rejection against +PostgreSQL 16. Locally, with `PG*` pointing at a throwaway server whose role can create +databases: + +```sh +go build -o "$PWD/beacon-backup" ./cmd/beacon-backup +BEACON_BACKUP_TEST_POSTGRES=1 BEACON_BACKUP_TEST_BINARY="$PWD/beacon-backup" \ + go test ./internal/backup -run '^TestExportPostgres$' -v +``` + +The test creates, migrates, restores and drops two randomly named databases. The export +format and operator usage are documented in +[Backup and export](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/backup-export.md). + +--- + ## Database changes All schema changes must include a proper migration path: - Add a new migration file to `db/migrations/` following the existing naming - convention (e.g. `002_add_observation_count.sql`). Do not modify existing - migration files — append only via new files. + convention (e.g. `002_add_observation_count.sql`). `001_baseline.sql` is the + 2.0.0 schema; do not modify existing migration files; append only via new files. - Update `db/queries/queries.sql` with any new or modified queries - Re-run `sqlc generate` to regenerate `db/sqlc/` - Update the store layer in `db/` to expose the new functionality - Update `internal/api/reader.go` if the change needs to be exposed via the API +- Historical stats read hourly rollups, never raw observations; follow + [docs/historical-stats.md](docs/historical-stats.md) to add a metric -Never edit files under `db/sqlc/` by hand — they are generated by sqlc and will +Never edit files under `db/sqlc/` by hand; they are generated by sqlc and will be overwritten. If you need to work around a sqlc limitation, document it clearly in the query comment. @@ -100,19 +97,32 @@ sqlc generate ## API changes +### Router configuration and optional admin endpoints + +Keep `router.New`'s signature stable. Pass startup settings by name in +`router.Options`; a new optional setting must not require unrelated callers and +tests to supply another positional argument. + +An optional admin feature owns its subrouter, dependencies and HTTP methods. +Register it once through `Options.AdminRoutes` in the startup wiring, using a +literal path such as `/accounts`. The parent admin router applies bearer +authentication to the entire subtree; the feature must not mount itself beside +that boundary. `/config` remains owned by the shared admin router. No global +registration or `init` side effects are needed. + +When concurrent PRs touch the same startup wiring, stack them in an explicit +merge order, regenerate Swagger rather than hand-merging it, and build the +combined result even when GitHub says it merges cleanly. + Any new or modified REST endpoint must have swagger annotations and regenerated docs: - Add or update `// @Summary`, `// @Param`, `// @Success`, `// @Failure`, and `// @Router` comments on the handler function -- Response types are defined in `internal/api/` — add new types there, not +- Response types are defined in `internal/api/`; add new types there, not inline in handlers -- After changing any handler or API type, regenerate the swagger docs and commit - the updated `docs/` directory alongside your changes: - -```bash -swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency -``` +- After changing any handler or API type, rerun the `swag init` command from the + checklist and commit the updated `docs/` directory Install swag if you don't have it: @@ -198,44 +208,37 @@ AIRPORTS_CSV=/path/to/airports.csv go run ./internal/iatadb/gen --- -## Commit messages - -Use the conventional commits format: - -``` -feat(routes): add observation_count to known routes response -fix(ingest): correct lat/lon divisor for advert payloads -refactor(hub): collapse RegionIATAs into IATAs in Scope -test(api): add unit tests for NodeTypeName and NodeTypeFromString -chore: update airports.csv -docs: expand CONTRIBUTING.md -``` - -Scopes are optional but helpful for larger codebases. Common scopes: `api`, -`db`, `ingest`, `hub`, `ws`, `handlers`, `config`, `keystore`. - ---- - ## Project structure ``` -cmd/beacon/ — main entry point, wiring, startup -db/ — store layer: sqlc-generated code + thin mapping layer - migrations/ — SQL schema (single file, append only) - queries/ — SQL queries (input to sqlc) - sqlc/ — generated Go code (do not edit by hand) +cmd/beacon/ main entry point, wiring, startup +cmd/beacon-backup/ standalone backup export/verify command +db/ store layer: sqlc-generated code + thin mapping layer + migrations/ SQL schema (2.0.0 baseline + numbered files, append only) + queries/ SQL queries (input to sqlc) + sqlc/ generated Go code (do not edit by hand) internal/ - api/ — response types and Reader interface - handlers/ — HTTP handlers (validation, routing, response) - router/ — chi router wiring - config/ — config loading and scope key derivation - hub/ — WebSocket fan-out broker - ingest/ — MQTT packet ingestion and side effects - iatadb/ — in-memory IATA airport lookup - keystore/ — channel key lookup - scopestore/ — transport scope key lookup - ws/ — WebSocket connection handling -docs/ — generated swagger docs (do not edit by hand) + api/ response types and Reader interface + handlers/ HTTP handlers (validation, routing, response) + middleware/ auth, rate limiting, logging + router/ chi router wiring + background/ periodic maintenance (retention, rollups, age-out) + backup/ database + config bundle export and verification + borders/ foreign-repeater classification against IATA borders + cache/ Redis caching layer over the Reader + config/ config loading and scope key derivation + hub/ WebSocket fan-out broker + iatadb/ in-memory IATA airport lookup + ingest/ MQTT packet ingestion and side effects + keystore/ channel key lookup + logging/ slog setup + lora/ LoRa airtime calculation + meshmapper/ MeshMapper scope and zone imports + presence/ coalesced last-seen writes + profiling/ opt-in CPU profiling + scopestore/ transport scope key lookup + ws/ WebSocket connection handling +docs/ generated swagger and design notes (swagger: do not edit by hand) ``` Key patterns to understand before contributing: @@ -245,7 +248,7 @@ Key patterns to understand before contributing: put business logic here. - **Ingest layer** (`internal/ingest/`): processes raw MQTT packets, calls the store, and broadcasts hub events. The `DB` interface in `ingest.go` defines - exactly what the ingest layer needs from the store — keep it minimal. + exactly what the ingest layer needs from the store; keep it minimal. - **Hub** (`internal/hub/`): pure fan-out broker. Events are pre-serialised JSON before entering the hub so the broadcast loop never touches encoding. - **Reader interface** (`internal/api/reader.go`): defines everything the API @@ -255,54 +258,15 @@ Key patterns to understand before contributing: ## Releases -Merges from `dev` to `main` are done by maintainers and represent a versioned -release. Do not open PRs directly against `main`. - -### Cutting a release - -> NOTE: release are done manually, not from GH. Commits for releases should be -> signed. - -1. Ensure all changes are committed and CI is green on `dev` -2. Bump the version string in the `@version` swagger annotation in - `cmd/beacon/main.go` -3. Regenerate swagger docs: +The flow is in +[Releases and versioning](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/releases.md#cutting-a-release). +The server-specific step is bumping `@version` in `cmd/beacon/main.go` and regenerating Swagger: ```bash swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency ``` -4. Commit the version bump and updated docs: - -``` -chore: bump version to vX.Y.Z -``` - -5. Merge `dev` into `main` (fast-forward only): - -```bash -git checkout main -git merge --ff-only dev -``` - -6. Tag the release and push: - -```bash -git tag vX.Y.Z -git push origin main --tags -``` - -Pushing the tag triggers the release CI workflow, which builds binaries for all -supported platforms and attaches them to a draft GitHub release. - -7. Open the draft release on GitHub, paste the release notes, and publish. - -8. Rebase `dev` on `main` to keep histories in sync for future releases: - -```bash -git checkout dev -git rebase main -``` +--- ## Recognition diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 929596ed..0f7e5d98 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -4,7 +4,7 @@ Beacon is built by its contributors. Thank you to everyone who has helped. ## Core -- [ded](https://hackers.town/@ded) — co-founder, lead developer +- [ded](https://hackers.town/@ded), co-founder, lead developer ## Contributors diff --git a/README.md b/README.md index 54251ba6..dca6275e 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ in PostgreSQL, and streams live events to WebSocket clients. - Subscribes to MeshCore MQTT brokers and decodes incoming LoRa packets using [meshcore-go](https://github.com/meshcore-go/meshcore-go) - Stores packets, observations, nodes, observers, traces, routes and channel - messages in PostgreSQL (more backends to come) -- Deduplicates observations across multiple brokers (same packet heard by two + messages in PostgreSQL +- Deduplicates observations across multiple brokers (the same packet heard by two brokers is one observation per observer) - Decrypts group text messages for known channel keys - Detects firmware capability flags from path hash sizes @@ -25,9 +25,9 @@ in PostgreSQL, and streams live events to WebSocket clients. - Seeds regions, IATA display names, and channel keys from a YAML config file on startup -For deployment instructions including the frontend app, see the deployment docs. - ---- +This repo is the code. Deploying, configuring and operating Beacon is documented in +[beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs); see +[Documentation](#documentation) below. ## Stack @@ -44,363 +44,70 @@ For deployment instructions including the frontend app, see the deployment docs. | Config | YAML via gopkg.in/yaml.v3 | | Env | godotenv | ---- - -## Getting started - -### Prerequisites +## Running it locally -- Go 1.26+ -- Docker and Docker Compose - -### 1. Clone and configure +You need Go 1.26+ and a PostgreSQL 16 database. Docker is the easy way to get the database. ```bash -git clone https://github.com/MeshCore-Beacon/beacon-server.git -cd beacon-server +git clone https://github.com/MeshCore-Beacon/beacon-server.git && cd beacon-server cp env.example .env cp config.yaml.example config.yaml -``` - -Edit `.env` with your broker credentials and database DSN. Edit `config.yaml` to -define your regions, IATA display names, channel keys, and retention settings. - -### 2. Start PostgreSQL - -```bash -docker compose up postgres -d -``` - -Database migrations are applied automatically on startup. - -### 3. Run - -```bash +docker run -d --name beacon-postgres -p 5432:5432 \ + -e POSTGRES_USER=beacon -e POSTGRES_PASSWORD=beacon -e POSTGRES_DB=beacon postgres:16-alpine go run ./cmd/beacon ``` -Or pull and run the Docker image: - -```bash -docker pull ghcr.io/meshcore-beacon/beacon-server:latest -``` - -The image is public on GitHub Container Registry — no `docker login` required. - -Beacon will: - -- Load `.env` and `config.yaml` -- Connect to PostgreSQL and seed config data -- Connect to the configured MQTT brokers -- Start the HTTP server on `LISTEN_ADDR` (default `:8080`) - -### Cold start and path resolution - -Path resolution, firmware capability detection, and known route storage all -depend on nodes having advertised at least once to a local observer. On a fresh -deployment `resolvedPath` will show `"confidence": "none"` for all hops and -`supportsMultibytePaths` will be `false` for all nodes until advert traffic -arrives and populates `node_short_ids`. This is expected behaviour — resolution -improves automatically as the mesh is observed over time. - ---- - -## Configuration - -### Environment variables (`.env`) - -| Variable | Default | Description | -| ------------------------ | ------------- | ------------------------------------------------------------ | -| `LISTEN_ADDR` | `:8080` | HTTP listen address | -| `POSTGRES_DSN` | — | PostgreSQL connection string | -| `REDIS_ADDR` | — | Redis address (`host:port`). Leave unset to disable caching. | -| `REDIS_PASSWORD` | — | Redis password (optional) | -| `REDIS_DB` | `0` | Redis database index | -| `CONFIG_PATH` | `config.yaml` | Path to YAML config file | -| `MQTT_BROKER_1_URL` | — | Broker 1 WebSocket URL (e.g. `wss://mqtt1.example.com:443`) | -| `MQTT_BROKER_1_USERNAME` | — | Broker 1 username | -| `MQTT_BROKER_1_PASSWORD` | — | Broker 1 password | -| `MQTT_BROKER_2_URL` | — | Broker 2 WebSocket URL | -| `MQTT_BROKER_2_USERNAME` | — | Broker 2 username | -| `MQTT_BROKER_2_PASSWORD` | — | Broker 2 password | - -### Config file (`config.yaml`) - -```yaml -# Optional IATA overrides — auto-created on first packet arrival, -# only needed if you want to customise display name or coordinates. -iatas: - YVR: - name: Vancouver International - lat: 49.1967 - lng: -123.1815 - -# Super-regions grouping multiple IATAs. -regions: - - slug: western-canada - name: Western Canada - display_order: 1 - center_lat: 51.0 - center_lng: -114.0 - zoom_level: 5 - iatas: [YVR, YYJ, YYC, YEG] - -# Channel keys for decrypting group messages. -channel_keys: - # Hashtag channels: Beacon derives the PSK from the tag name automatically. - # secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0] - # Tag names should be provided without the # prefix. - hashtags: - - meshcore - - # Explicit keys: channel hash (hex) and key (hex), with optional display name. - # The public MeshCore channel key is included in config.yaml.example. - keys: - "11": - key: "8b3387e9c5cdea6ac9e5edbaa115cd72" - name: "Public" - -# Regional transport scopes for matching TRANSPORT_FLOOD packets. -# Plain names have # prepended automatically (e.g. "bc" → "#bc"). -scopes: - - name: bc - - name: "#west" - -# Observer telemetry storage settings. -telemetry: - retention: 672h # how long to keep telemetry snapshots (default: 4 weeks) - resolution: 1h # snapshot frequency per observer; duplicates within window are dropped (default: 1h) - -# Packet and observation retention. -packets: - retention: 720h # how long to keep packets and observations (default: 30 days) - -# Presence write coalescing. -# Observer last_seen and packet last_heard_at bumps are batched in memory and -# flushed on an interval instead of writing one row per observation. An -# unclean shutdown loses at most one interval of presence freshness. -presence: - flush_interval: 30s # how often coalesced bumps are flushed (default: 30s) - packet_ttl: 30s # how long a quiet packet stays coalesced before writing through again (default: 30s) - -# WebSocket settings. -websocket: - max_connections_per_ip: 5 # default: 5 - -# Redis caching layer (optional). -# Caches read-heavy, slow-changing responses to reduce PostgreSQL load. -# Connection details (address, password, database) are set via environment -# variables. Leave REDIS_ADDR unset to disable caching entirely. -# TTLs are duration strings e.g. "30m", "1h". Per-category TTLs override -# the global ttl. Any unset category inherits ttl. Default: 1h. -cache: - ttl: "1h" - ttls: - stats: "1h" # stats endpoints (backed by materialized views) - reference: "1h" # IATAs, regions, scopes - nodes: "1h" # node detail (also explicitly invalidated on upsert) - observers: "1h" # observer detail (also explicitly invalidated on upsert) - -# Geographic ingest filter (optional). -# Drop packets from observers outside the specified area. -# Country codes are ISO 3166-1 alpha-2. Continent codes: AF AN AS EU NA OC SA. -# If both are set an IATA passes if it matches either (OR semantics). -# Omit entirely to accept all IATAs (default). -ingest: - allow_countries: [CA, US] # only store packets from these countries - allow_continents: [NA] # or: accept all of North America -``` - -IATAs are auto-created on first packet arrival. The config file adds display -names and coordinates. Regions and channel keys must be defined here — they are -not auto-created. - ---- - -## Authentication - -API authentication is not yet implemented. Beacon is intended for trusted -internal network or reverse-proxy deployments. Do not expose it directly to the -public internet without an authentication layer in front of it. - ---- - -## WebSocket API - -Connect to `ws://host:8080/ws`. - -On connect the server sends a `hello`: - -```json -{ "v": 1, "type": "hello", "serverTime": 1234567890000, "connectionId": "uuid" } -``` - -The connection closes after 90 seconds of inactivity. Clients should send a -`ping` every 30 seconds. - -### Client → Server messages - -**Subscribe** — add a filter to this connection. Multiple subscriptions are -unioned (OR semantics): an event matches if it satisfies any active -subscription. The server replies with a `subscriptionId` to use for -unsubscribing. - -```json -{ - "v": 1, - "type": "subscribe", - "id": "sub-1", - "scope": { - "iatas": ["YOW", "YYZ"], - "regionIds": ["1"], - "regionSlugs": ["western-canada"], - "payloadTypes": [4, 5], - "channelHashes": ["11"], - "events": ["packetObservation", "channelMessage"] - } -} -``` - -All scope fields are optional. Omitted means no filter on that dimension (match -everything). Empty array means match nothing on that dimension. `regionIds` and -`regionSlugs` are both expanded to their member IATAs server-side. +Fill in `POSTGRES_DSN` and your MQTT broker credentials in `.env`. Every variable is described +in [Configuration](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/configuration.md), +and the fully annotated `config.yaml` lives in beacon-docs as well; the copy here is a working +starter. Migrations run on startup. The API listens on `LISTEN_ADDR` (default `:8080`) and +Swagger is at `http://localhost:8080/swagger/index.html`. -**Unsubscribe** — remove a specific subscription by ID. +To run the web frontend against it, see +[Running the full stack locally](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/CONTRIBUTING.md#running-the-full-stack-locally). -```json -{ - "v": 1, - "type": "unsubscribe", - "id": "unsub-1", - "subscriptionId": "" -} -``` +The published image is `ghcr.io/meshcore-beacon/beacon-server`. `latest` tracks stable +releases, `dev` follows the development branch, and each release is also tagged `X.Y.Z` and +`X.Y`. -**Configure** — toggle connection-wide settings. Currently just `resolvePath`, -which enables per-hop node resolution on `packetObservation` events (see -below). Unlike `subscribe`, this is a single flag for the whole connection, -not additive across calls — each `configure` sets it to exactly the value -sent, and it can be flipped on or off as many times as you like for the life -of the connection. Default is `false`. +### What you see on an empty database -```json -{ "v": 1, "type": "configure", "id": "cfg-1", "resolvePath": true } -``` +Path resolution, capability detection and known routes depend on nodes having advertised to a +local observer. On a fresh database every hop shows `"confidence": "none"` and +`supportsMultibytePaths` is `false` until adverts populate `node_short_ids`. That is expected +and fills in as the mesh is observed. -The server replies: +GRP_TXT packets whose channel key is not known yet are stored hash-only. After adding the key to +`config.yaml`, the next restart decrypts the matching history; look for +`config: backfilled N previously-undecrypted channel message(s)` in the log. -```json -{ "v": 1, "type": "configured", "id": "cfg-1", "resolvePath": true } -``` +## Documentation -**Ping** +In [beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs): -```json -{ "v": 1, "type": "ping", "id": "ping-1" } -``` +- [Deploy with Docker](https://github.com/MeshCore-Beacon/beacon-docs#deploy-the-all-in-one-stack) +- [Getting packets in](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/getting-packets-in.md): brokers, the subscriber account, topics +- [Configuration](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/configuration.md): environment variables and `config.yaml` +- [API contract](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/api-contract.md): REST, WebSocket, admin endpoints +- [Reverse proxy and rate limits](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/reverse-proxy.md) +- [Upgrading](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/upgrading.md) +- [Operations](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/operations.md), [backup and export](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/backup-export.md) and [CPU profiling](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/profiling.md) +- [High level design](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/high-level-design.md) +- [Releases and versioning](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/releases.md) -### Server → Client events +In this repo: [Historical stats](docs/historical-stats.md) explains the hourly rollups behind +the stats endpoints, for anyone changing them. `docs/swagger.yaml` is the generated OpenAPI +description. -| Type | Description | -| ------------------- | --------------------------------------------------- | -| `packetObservation` | New observation written to DB | -| `observerStatus` | Observer status update | -| `nodeUpdate` | Node upserted from advert | -| `channelMessage` | Decrypted channel message (scope must include hash) | +## Contributing -When `resolvePath` is enabled via `configure`, `packetObservation` events -include a `resolvedPath` array: one entry per hop in the packet's path, each -with a `confidence` (`"high"` for exactly one candidate node, `"ambiguous"` -for multiple, `"none"` for zero) and the matching node(s)' id, name, -lat/lng, and public key. Without `resolvePath` enabled, `resolvedPath` is -`null` — this keeps the default event payload small; only opt in if you're -actually rendering path connections (e.g. drawing hops on a map). - -### Backpressure - -The server write buffer per connection is bounded at 256 events. If a client -falls behind, the server drops the oldest queued events and sends a `lagged` -notice: - -```json -{ "v": 1, "type": "lagged", "droppedCount": 12, "since": 1234567890000 } -``` - -Clients should respond by re-fetching the relevant REST endpoint using `afterId` -to backfill missed events, then resume streaming. - -### Reconnection - -Subscriptions are not persisted — they exist only for the lifetime of the -connection. On any disconnect the client should reconnect with backoff, re-issue -all subscriptions, and backfill via REST using -`afterId=`. - -### Connection limits - -By default a maximum of 5 concurrent WebSocket connections are allowed per IP -address. Connections beyond this limit receive `HTTP 429`. The limit is -configurable via `websocket.max_connections_per_ip` in `config.yaml`. - ---- - -## REST API - -Base path: `/api/v1` - -All list endpoints support cursor-based pagination via `cursor` and `limit` -query params. See the Swagger UI at `http://localhost:8080/swagger/index.html` -for full parameter documentation. - -### Authentication - -Not yet implemented — see the Authentication section above. - -### Endpoints - -| Method | Path | Description | -| ------ | ----------------------------------- | -------------------------------------------------------------------------------------------------- | -| `GET` | `/brokers` | List MQTT brokers and connection status | -| `GET` | `/channels` | List channels (optional: `?hash=&iata=&limit=50`) | -| `GET` | `/channels/{id}` | Get channel detail by integer ID | -| `GET` | `/channels/{id}/messages` | List messages for a channel (optional: `?since=&iata=&limit=50`) | -| `GET` | `/iatas` | List all known IATA codes | -| `GET` | `/iatas/{iata}` | Get a single IATA code | -| `GET` | `/messages` | List all messages (optional: `?channelId=&channelHash=&iata=&since=&limit=50`) | -| `GET` | `/messages/backfill` | Backfill messages after a given message ID | -| `GET` | `/nodes` | List nodes | -| `GET` | `/nodes/{nodeId}` | Get node detail | -| `GET` | `/nodes/{nodeId}/neighbors` | List neighboring nodes observed in the mesh | -| `GET` | `/nodes/{nodeId}/observations` | List observations for a node | -| `GET` | `/observers` | List observers (optional: `?iata=&type=&broker=&status=online\|offline`) | -| `GET` | `/observers/{observerId}` | Get observer detail including broker last-seen timestamps | -| `GET` | `/observers/{observerId}/adverts` | Adverts heard by observer | -| `GET` | `/observers/{observerId}/telemetry` | Observer telemetry history (optional: `?range=24h&interval=1h\|6h\|24h`) | -| `GET` | `/packets` | List packets with filters | -| `GET` | `/packets/backfill` | Backfill packets after a given observation ID | -| `GET` | `/packets/{packetHash}` | Get packet with all observations | -| `GET` | `/regions` | List all regions (summary) | -| `GET` | `/regions/{id}` | Get a single region with IATA list | -| `GET` | `/routes` | List known routes (all hops high confidence) | -| `GET` | `/routes/search` | Search routes by source and destination hash | -| `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | -| `GET` | `/scopes` | List transport scopes | -| `GET` | `/scopes/{name}` | Get scope detail | -| `GET` | `/stats/observations` | Hourly observation time series (last 7 days by default) | -| `GET` | `/stats/overview` | Network overview stats | -| `GET` | `/stats/payload-breakdown` | Observation counts by payload type (last 24h by default) | -| `GET` | `/stats/scopes` | Configured region scopes and breakdown of packets, nodes, observers | -| `GET` | `/stats/top-nodes` | Top N nodes by observation count (from materialized view) | -| `GET` | `/stats/top-observers` | Top N observers by observation count (last 24h by default) | -| `GET` | `/traces` | List trace tags with filters (optional: ?type=TRACE\|PING) | -| `GET` | `/traces/{tag}` | Get full trace detail with resolved routes | - ---- +[CONTRIBUTING.md](CONTRIBUTING.md) covers code style, tests, and database and API changes. +Branches, commits and the release flow are shared across the Beacon repos and live in +[beacon-docs/CONTRIBUTING.md](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/CONTRIBUTING.md). +Please read the [Code of Conduct](CODE_OF_CONDUCT.md). Security reports go through +[SECURITY.md](SECURITY.md). ## Acknowledgements -See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people who have helped build -Beacon. - -Beacon stands on the shoulders of giants. See [SHOULDERS.md](SHOULDERS.md) for -the full list of open source projects that make this possible. +See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people who have helped build Beacon, and +[SHOULDERS.md](SHOULDERS.md) for the open source projects it stands on. diff --git a/SECURITY.md b/SECURITY.md index 397b3071..12a5e817 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,7 +5,7 @@ Please do not report security vulnerabilities through public GitHub issues. Instead, contact the maintainers directly via the MeshCore Canada Discord -server: [MeshCore Canada Discord](https://discord.gg/Gz3KvJx2hf) — reach out to +server: [MeshCore Canada Discord](https://discord.gg/Gz3KvJx2hf); reach out to **dedskelly** directly. Include as much detail as possible: the nature of the issue, steps to reproduce, and any potential impact. diff --git a/SHOULDERS.md b/SHOULDERS.md index 5948cf42..51a98568 100644 --- a/SHOULDERS.md +++ b/SHOULDERS.md @@ -5,33 +5,48 @@ projects that make this possible: ## Core dependencies -- [meshcore-go](https://github.com/meshcore-go/meshcore-go) — MeshCore packet +- [meshcore-go](https://github.com/meshcore-go/meshcore-go): MeshCore packet decoding -- [paho.mqtt.golang](https://github.com/eclipse/paho.mqtt.golang) — MQTT client -- [pgx](https://github.com/jackc/pgx) — PostgreSQL driver and connection pool -- [sqlc](https://sqlc.dev) — Type-safe SQL code generation -- [chi](https://github.com/go-chi/chi) — HTTP router -- [go-chi/cors](https://github.com/go-chi/cors) — CORS middleware for chi -- [go-redis](https://github.com/redis/go-redis) — Redis client -- [coder/websocket](https://github.com/coder/websocket) — WebSocket +- [paho.mqtt.golang](https://github.com/eclipse/paho.mqtt.golang): MQTT client +- [pgx](https://github.com/jackc/pgx): PostgreSQL driver and connection pool +- [sqlc](https://sqlc.dev): Type-safe SQL code generation +- [chi](https://github.com/go-chi/chi): HTTP router +- [go-chi/cors](https://github.com/go-chi/cors): CORS middleware for chi +- [go-chi/httprate](https://github.com/go-chi/httprate): Per-client HTTP + sliding-window rate limits and retry headers +- [go-redis](https://github.com/redis/go-redis): Redis client +- [coder/websocket](https://github.com/coder/websocket): WebSocket implementation -- [swaggo/swag](https://github.com/swaggo/swag) — OpenAPI documentation +- [swaggo/swag](https://github.com/swaggo/swag): OpenAPI documentation generation -- [godotenv](https://github.com/joho/godotenv) — .env file loading -- [yaml.v3](https://github.com/go-yaml/yaml) — YAML config parsing -- [google/uuid](https://github.com/google/uuid) — UUID generation +- [godotenv](https://github.com/joho/godotenv): .env file loading +- [yaml.v3](https://github.com/go-yaml/yaml): YAML config parsing +- [google/uuid](https://github.com/google/uuid): UUID generation +- [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync): singleflight for + collapsing concurrent cache misses +- [paulmach/orb](https://github.com/paulmach/orb): GeoJSON border geometry and + point-in-polygon checks +- [swaggo/http-swagger](https://github.com/swaggo/http-swagger): Swagger UI + handler + +## Testing + +- [go.uber.org/mock](https://github.com/uber-go/mock): gomock mocks for the + generated queries +- [miniredis](https://github.com/alicebob/miniredis): In-memory Redis for cache + tests ## Data -- [OurAirports](https://ourairports.com/data/) — IATA airport geographic data +- [OurAirports](https://ourairports.com/data/): IATA airport geographic data (CC0 public domain) ## Infrastructure -- [PostgreSQL](https://www.postgresql.org) — Primary data store -- [Redis](https://redis.io) — Optional caching layer -- [Caddy](https://caddyserver.com) — Reverse proxy +- [PostgreSQL](https://www.postgresql.org): Primary data store +- [Redis](https://redis.io): Optional caching layer +- [Caddy](https://caddyserver.com): Reverse proxy ## Language -- [Go](https://go.dev) — The language that makes this all possible +- [Go](https://go.dev): The language that makes this all possible diff --git a/cmd/beacon-backup/main.go b/cmd/beacon-backup/main.go new file mode 100644 index 00000000..6484382a --- /dev/null +++ b/cmd/beacon-backup/main.go @@ -0,0 +1,92 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// beacon-backup creates or verifies a private database and saved-config export. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "syscall" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" +) + +var version = "dev" + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + code := run(ctx, os.Args[1:], os.Stdout, os.Stderr) + stop() + os.Exit(code) +} + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) int { + var opts backup.Options + var verifyPath string + flags := flag.NewFlagSet("beacon-backup", flag.ContinueOnError) + flags.SetOutput(stderr) + flags.StringVar(&opts.ConfigPath, "config", "config.yaml", "saved YAML file to export verbatim (may contain secrets)") + flags.StringVar(&opts.OutputPath, "output", "", "new private .tar.gz export destination (never overwritten)") + flags.StringVar(&verifyPath, "verify", "", "verify an existing .tar.gz without extraction or database access") + flags.Int64Var(&opts.MaxBytes, "max-bytes", backup.DefaultMaxBytes, "maximum uncompressed database dump size") + flags.DurationVar(&opts.Timeout, "timeout", backup.DefaultTimeout, "maximum export or verification duration") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if flags.NArg() != 0 || opts.Timeout <= 0 { + fmt.Fprintln(stderr, "positive timeout and no positional arguments required") + return 2 + } + verifyMode, exportFlag := false, false + flags.Visit(func(f *flag.Flag) { + verifyMode = verifyMode || f.Name == "verify" + exportFlag = exportFlag || f.Name == "config" || f.Name == "output" + }) + if verifyMode { + if verifyPath == "" || exportFlag { + fmt.Fprintln(stderr, "verify requires an archive path and cannot be combined with config or output") + return 2 + } + // Reject directories/devices/FIFOs before opening, and check the opened + // descriptor too. A symlink to a regular, operator-selected file is fine. + info, err := os.Stat(verifyPath) + if err != nil || !info.Mode().IsRegular() { + fmt.Fprintln(stderr, "verification requires a readable regular archive file") + return 1 + } + file, err := os.Open(verifyPath) + if err != nil { + fmt.Fprintln(stderr, "cannot open backup archive") + return 1 + } + defer file.Close() + if info, err = file.Stat(); err != nil || !info.Mode().IsRegular() { + fmt.Fprintln(stderr, "verification requires a regular archive file") + return 1 + } + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) + defer cancel() + if _, err := backup.Verify(ctx, file, opts.MaxBytes); err != nil { + fmt.Fprintln(stderr, "verification failed:", err) + return 1 + } + fmt.Fprintln(stdout, "Archive structure and checksums verified. No files extracted or SQL executed.") + return 0 + } + opts.Version = version + if err := backup.Export(ctx, opts); err != nil { + fmt.Fprintln(stderr, "backup failed:", err) + return 1 + } + fmt.Fprintln(stdout, "Backup complete. Store this bundle privately; it may contain keys and message data.") + return 0 +} diff --git a/cmd/beacon-backup/main_test.go b/cmd/beacon-backup/main_test.go new file mode 100644 index 00000000..e55caf6a --- /dev/null +++ b/cmd/beacon-backup/main_test.go @@ -0,0 +1,82 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" +) + +func TestVerifyCommandIsReadOnlyAndOffline(t *testing.T) { + dir := t.TempDir() + t.Setenv("PATH", dir) // no pg_dump, psql or shell helper is available + t.Setenv("PGDATABASE", "") + archive := filepath.Join(dir, "private.tar.gz") + m := backup.Manifest{FormatVersion: 1, CreatedAt: time.Now().UTC(), DatabaseFormat: "postgresql-plain-sql", ToolVersion: "PRIVATE_CANARY"} + data := map[string][]byte{"database.sql": []byte("SELECT 'PRIVATE_CANARY';"), "config.yaml": {}} + for _, name := range []string{"database.sql", "config.yaml"} { + hash := sha256.Sum256(data[name]) + m.Files = append(m.Files, backup.File{Name: name, Size: int64(len(data[name])), SHA256: hex.EncodeToString(hash[:])}) + } + data["manifest.json"], _ = json.Marshal(m) + var raw bytes.Buffer + gz := gzip.NewWriter(&raw) + tw := tar.NewWriter(gz) + for _, name := range []string{"manifest.json", "database.sql", "config.yaml"} { + if err := tw.WriteHeader(&tar.Header{Name: name, Size: int64(len(data[name])), Mode: 0600}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(data[name]); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(archive, raw.Bytes(), 0600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + args []string + code int + }{ + {[]string{"-verify", archive}, 0}, + {[]string{"-verify", archive, "-max-bytes", "1"}, 1}, + {[]string{"-verify", archive, "-timeout", "0"}, 2}, + {[]string{"-verify", archive, "-output", filepath.Join(dir, "must-not-exist")}, 2}, + {[]string{"-verify", archive, "-config", "missing.yaml"}, 2}, + {[]string{"-verify", "", "-output", filepath.Join(dir, "must-not-exist")}, 2}, + {[]string{"-verify", dir}, 1}, + {[]string{"-verify", filepath.Join(dir, "PRIVATE_CANARY-missing")}, 1}, + {[]string{"-verify", archive, "unexpected"}, 2}, + } { + var stdout, stderr bytes.Buffer + if got := run(context.Background(), tc.args, &stdout, &stderr); got != tc.code || strings.Contains(stdout.String()+stderr.String(), "PRIVATE_CANARY") { + t.Fatalf("exit %d, want %d; diagnostic %s", got, tc.code, stderr.String()) + } + } + files, err := os.ReadDir(dir) + if err != nil || len(files) != 1 || files[0].Name() != "private.tar.gz" { + t.Fatal("verification extracted files or created output") + } + unchanged, err := os.ReadFile(archive) + if err != nil || !bytes.Equal(unchanged, raw.Bytes()) { + t.Fatal("input archive changed") + } +} diff --git a/cmd/beacon/backup.go b/cmd/beacon/backup.go new file mode 100644 index 00000000..ee3bc148 --- /dev/null +++ b/cmd/beacon/backup.go @@ -0,0 +1,45 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/jackc/pgx/v5/pgxpool" +) + +// A failed optional export prerequisite must not stop ingest or the public API. +// Empty options leave the authenticated backup endpoint unavailable (503). +func configureBackup(ctx context.Context, cfg *config.Config, pool *pgxpool.Pool, dsn, configPath string) backup.Options { + if !cfg.Backup.Enabled { + return backup.Options{} + } + if cfg.Auth.APIKey == "" { + slog.Error("backup unavailable: configure an admin bearer key", "component", "backup") + return backup.Options{} + } + service, err := backup.ConnectionService(dsn) + if err != nil { + slog.Error("backup unavailable: POSTGRES_DSN must be a supported single-host PostgreSQL URL; keyword DSNs remain valid for Beacon", "component", "backup") + return backup.Options{} + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + var serverVersion int + if err := pool.QueryRow(ctx, "SELECT current_setting('server_version_num')::integer").Scan(&serverVersion); err != nil { + slog.Error("backup unavailable: could not determine PostgreSQL server version", "component", "backup") + return backup.Options{} + } + if err := backup.CheckClient(ctx, serverVersion); err != nil { + // CheckClient never returns process output or connection settings. + slog.Error("backup unavailable", "component", "backup", "error", err) + return backup.Options{} + } + return backup.Options{ConfigPath: configPath, MaxBytes: backup.DefaultMaxBytes, + Timeout: backup.DefaultTimeout, Version: version, ConnectionService: service} +} diff --git a/cmd/beacon/backup_test.go b/cmd/beacon/backup_test.go new file mode 100644 index 00000000..3a49a44a --- /dev/null +++ b/cmd/beacon/backup_test.go @@ -0,0 +1,58 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestBackupPreconditionsDegradeWithoutDatabaseOrExit(t *testing.T) { + cfg := &config.Config{} + if got := configureBackup(context.Background(), cfg, nil, "", "saved.yaml"); got.ConnectionService != "" { + t.Fatal("disabled backup configured") + } + cfg.Backup.Enabled = true + if got := configureBackup(context.Background(), cfg, nil, "", "saved.yaml"); got.ConnectionService != "" { + t.Fatal("missing admin key configured") + } + cfg.Auth.APIKey = "test-only-key" + if got := configureBackup(context.Background(), cfg, nil, "host=localhost user=test dbname=test", "saved.yaml"); got.ConnectionService != "" { + t.Fatal("keyword connection configured for backup") + } +} + +func TestBackupConfigurationPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" || runtime.GOOS == "windows" { + t.Skip("requires BEACON_TEST_POSTGRES_DSN and a Unix test client") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal("test database configuration unavailable") + } + defer pool.Close() + bin := t.TempDir() + t.Setenv("PATH", bin) + cfg := &config.Config{} + cfg.Backup.Enabled, cfg.Auth.APIKey = true, "test-only-key" + for _, major := range []string{"999", "15"} { + if err := os.WriteFile(filepath.Join(bin, "pg_dump"), []byte("#!/bin/sh\necho 'pg_dump (PostgreSQL) "+major+".0'\n"), 0700); err != nil { + t.Fatal(err) + } + got := configureBackup(ctx, cfg, pool, dsn, "saved.yaml") + if (got.ConnectionService != "") != (major == "999") { + t.Fatalf("unexpected backup availability for test client major %s", major) + } + } +} diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 1067408e..dc87e2bf 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -6,29 +6,36 @@ package main import ( "context" "encoding/hex" + "encoding/json" "fmt" - "log" + "log/slog" "net/http" "os" "os/signal" "strconv" + "sync" "syscall" "time" "github.com/MeshCore-Beacon/beacon-server/db" _ "github.com/MeshCore-Beacon/beacon-server/docs" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/api/router" "github.com/MeshCore-Beacon/beacon-server/internal/background" + "github.com/MeshCore-Beacon/beacon-server/internal/borders" "github.com/MeshCore-Beacon/beacon-server/internal/cache" "github.com/MeshCore-Beacon/beacon-server/internal/config" "github.com/MeshCore-Beacon/beacon-server/internal/hub" "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "github.com/MeshCore-Beacon/beacon-server/internal/logging" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" "github.com/MeshCore-Beacon/beacon-server/internal/presence" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" "github.com/joho/godotenv" ) @@ -38,8 +45,10 @@ import ( var version = "dev" // @title MeshCore Beacon API -// @version 1.6.0 +// @version 2.0.0 // @description MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket. +// @description REST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget. +// @description WebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open. // @termsOfService https://github.com/MeshCore-Beacon/beacon-server // @contact.name MeshCore Beacon @@ -52,6 +61,11 @@ var version = "dev" // @schemes http https +// @securityDefinitions.apikey AdminKey +// @in header +// @name Authorization +// @description Enter Bearer followed by the configured operator key. Use HTTPS. + // @tag.name IATAs // @tag.description Airport/location codes that group observers and packets // @tag.name Regions @@ -71,7 +85,6 @@ var version = "dev" // @tag.name Stats // @tag.description Network statistics and time series func main() { - log.Printf("beacon version %s", version) _ = godotenv.Load() addr := os.Getenv("LISTEN_ADDR") if addr == "" { @@ -85,12 +98,32 @@ func main() { cfg, err := config.Load(configPath) if err != nil { - log.Fatalf("failed to load config: %v", err) + slog.Error("failed to load config", "component", "startup", "error", err) + os.Exit(1) } - + logger, err := logging.New(os.Stderr, cfg.Log) + if err != nil { + slog.Error("invalid logging configuration", "component", "startup", "error", err) + os.Exit(1) + } + slog.SetDefault(logger) + slog.Info("beacon starting", "component", "startup", "version", version) resolved := config.Resolve(cfg) + if msg := proxyLimitWarning(resolved, len(cfg.Server.TrustedProxies)); msg != "" { + slog.Warn(msg, "component", "startup") + } + borderFiles, err := config.LoadLocalBorders(cfg) + if err != nil { + slog.Error("invalid local border configuration", "component", "startup", "error", err) + os.Exit(1) + } + var localBorders *borders.Live + if cfg.Nodes.MarkForeign { + local, _ := config.BuildLocalBorders(borderFiles, nil) // checked by LoadLocalBorders + localBorders = borders.NewLive(local) + } - log.Printf("config: loaded — %s", resolved) + slog.Info(fmt.Sprintf("config: loaded — %s", resolved), "component", "startup") // ── Hub ────────────────────────────────────────────────────────────────── h := hub.New() @@ -100,14 +133,19 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - pool, err := pgxpool.New(ctx, getEnv("POSTGRES_DSN")) + dsn := getEnv("POSTGRES_DSN") + pool, err := pgxpool.New(ctx, dsn) if err != nil { - log.Fatalf("failed to connect to postgres at %s: %v", os.Getenv("POSTGRES_DSN_HOST"), err) + // Parse errors can embed the complete DSN, including its password. + slog.Error("invalid PostgreSQL connection configuration; check POSTGRES_DSN", "component", "startup") + os.Exit(1) } defer pool.Close() + backupOpts := configureBackup(ctx, cfg, pool, dsn, configPath) if err := db.RunMigrations(ctx, pool); err != nil { - log.Fatalf("migrations failed: %v", err) + slog.Error("migrations failed", "component", "startup", "error", err) + os.Exit(1) } store := db.New(pool, resolved.ClockDriftThreshold, resolved.NodeStaleThreshold) @@ -134,35 +172,93 @@ func main() { }(), ) if err := redisClient.Ping(ctx); err != nil { - log.Printf("warning: redis unavailable at %s, caching disabled: %v", redisAddr, err) + slog.Warn(fmt.Sprintf("warning: redis unavailable at %s, caching disabled", redisAddr), "component", "startup", "error", err) } else { ttls := cache.ResolveTTLs(cfg.Cache) reader = cache.NewCachedReader(store, redisClient, ttls) defer redisClient.Close() - log.Printf("cache: Redis connected at %s (stats=%s reference=%s nodes=%s observers=%s)", - redisAddr, ttls.Stats, ttls.Reference, ttls.Nodes, ttls.Observers) + slog.Info(fmt.Sprintf("cache: Redis connected at %s (stats=%s reference=%s nodes=%s observers=%s)", redisAddr, ttls.Stats, ttls.Reference, ttls.Nodes, ttls.Observers), "component", "startup") } } // ── Seed config data ───────────────────────────────────────────────────── if err := config.Seed(ctx, cfg, store); err != nil { - log.Fatalf("failed to seed config: %v", err) + slog.Error("failed to seed config", "component", "startup", "error", err) + os.Exit(1) } // ── Build transport scope keystore ─────────────────────────────────────── scopes := scopestore.New() scopeEntries, err := store.GetTransportScopes(ctx) if err != nil { - log.Fatalf("failed to load transport scopes: %v", err) + slog.Error("failed to load transport scopes", "component", "startup", "error", err) + os.Exit(1) } scopes.Load(scopeEntries) - log.Printf("loaded %d transport scopes", len(scopeEntries)) + scopes.SetManualMembers(cfg.ManualScopeMembers()) + slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") + directory := meshmapper.NewDirectory(store) + if cfg.MeshMapper.Scopes.Enabled || cfg.MeshMapper.Zones.Enabled || cfg.MeshMapper.Channels.Enabled { + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + err = directory.Restore(restoreCtx) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper zone lists", "component", "startup", "error", err) + os.Exit(1) + } + } + var scopeImporter *meshmapper.Importer + if cfg.MeshMapper.Scopes.Enabled { + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, directory, scopes, scopeEntries) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper scope catalogues", "component", "startup", "error", err) + os.Exit(1) + } + } + + // Always restored, so disabling the import prunes it and file borders return. + zones := meshmapper.NewZones(cfg.MeshMapper.Zones, store, directory) + var detailed []string + for iata, d := range cfg.IATAs { + if d.Name != "" || d.Lat != nil || d.Lng != nil { + detailed = append(detailed, iata) + } + } + zones.SetConfiguredIATAs(detailed) + if cr, ok := reader.(*cache.CachedReader); ok { + zones.OnChange(cr.InvalidateIATABorder) + zones.OnRegionsChange(cr.InvalidateRegions) + zones.OnIATAsChange(cr.InvalidateIATAs) + } + if localBorders != nil { + zones.OnUpdate(func(imported map[string]json.RawMessage) { + local, err := config.BuildLocalBorders(borderFiles, imported) + if err != nil { + slog.Error("MeshMapper boundaries rejected for foreign marking", "component", "meshmapper.zones", "error", err) + return + } + localBorders.Store(local) + }) + } + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + err = zones.Restore(restoreCtx) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper boundaries", "component", "startup", "error", err) + os.Exit(1) + } + if localBorders != nil && !localBorders.Ready() { + slog.Warn("nodes.mark_foreign has no boundaries yet; waiting for MeshMapper", "component", "startup") + } // ── Build channel keystore ────────────────────────────────────────────── entries := make(map[string][]keystore.Entry) // Hashtag-derived channels: secret = SHA256("#tag")[:16], hash = SHA256(secret)[0] - for _, tag := range cfg.ChannelKeys.Hashtags { + for _, hashtag := range cfg.ChannelKeys.Hashtags { + tag := hashtag.Name secret, channelHash, fingerprint := keystore.DeriveHashtagKey(tag) hashHex := fmt.Sprintf("%02x", channelHash) entry := keystore.Entry{ @@ -173,7 +269,7 @@ func main() { } if !keystore.EntryExists(entries[hashHex], entry) { entries[hashHex] = append(entries[hashHex], entry) - log.Printf("config: loaded hashtag channel #%s (hash=%s)", tag, hashHex) + slog.Info(fmt.Sprintf("config: loaded hashtag channel #%s (hash=%s)", tag, hashHex), "component", "startup") } } @@ -181,7 +277,7 @@ func main() { for hashHex, keyCfg := range cfg.ChannelKeys.Keys { key, err := hex.DecodeString(keyCfg.Key) if err != nil { - log.Printf("warning: invalid channel key for hash %s, skipping: %v", hashHex, err) + slog.Warn(fmt.Sprintf("warning: invalid channel key for hash %s, skipping", hashHex), "component", "startup", "error", err) continue } entry := keystore.Entry{ @@ -191,35 +287,56 @@ func main() { } if !keystore.EntryExists(entries[hashHex], entry) { entries[hashHex] = append(entries[hashHex], entry) - log.Printf("config: loaded explicit channel key for hash %s name=%q", hashHex, keyCfg.Name) + slog.Info(fmt.Sprintf("config: loaded explicit channel key for hash %s name=%q", hashHex, keyCfg.Name), "component", "startup") } } keys := keystore.NewMapKeyStore(entries) + // Restored before the boot backfill so saved MeshMapper keys decrypt history too. + restoreCtx, cancelRestore = context.WithTimeout(ctx, 10*time.Second) + channelImporter, err := meshmapper.NewChannels(restoreCtx, cfg.MeshMapper.Channels, store, directory, keys) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper channels", "component", "startup", "error", err) + os.Exit(1) + } + channelImporter.OnNewKeys(func(hashes [][]byte) { + // Off the refresh tick: a backfill can outlast it, and history isn't urgent. + go func() { + backfillCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + if n, err := ingest.BackfillChannelHashes(backfillCtx, store, keys, hashes); err != nil { + slog.Error("MeshMapper channel backfill failed", "component", "meshmapper.channels", "error", err) + } else if n > 0 { + slog.Info(fmt.Sprintf("backfilled %d channel message(s) for imported keys", n), "component", "meshmapper.channels") + } + }() + }) + // ── Backfill channel messages ──────────────────────────────────────────── // Packets whose channel key wasn't yet configured at ingest time were stored as // hash-only channels and never decrypted. Retry them now against the keystore we just // built, so adding a channel key to the config surfaces its history on the next boot // instead of leaving it stranded in the DB indefinitely. if n, err := ingest.BackfillChannelMessages(ctx, store, keys); err != nil { - log.Printf("config: channel message backfill failed: %v", err) + slog.Error("config: channel message backfill failed", "component", "startup", "error", err) } else if n > 0 { - log.Printf("config: backfilled %d previously-undecrypted channel message(s)", n) + slog.Info(fmt.Sprintf("config: backfilled %d previously-undecrypted channel message(s)", n), "component", "startup") } // ── Build geographic ingest filter ─────────────────────────────────────────────────────────── allowedIATAs := iatadb.BuildAllowedSet(cfg.Ingest.AllowCountries, cfg.Ingest.AllowContinents) if allowedIATAs != nil { - log.Printf("config: ingest filter active — %d allowed IATAs (countries=%v continents=%v)", - len(allowedIATAs), cfg.Ingest.AllowCountries, cfg.Ingest.AllowContinents) + slog.Info(fmt.Sprintf("config: ingest filter active — %d allowed IATAs (countries=%v continents=%v)", len(allowedIATAs), cfg.Ingest.AllowCountries, cfg.Ingest.AllowContinents), "component", "startup") } else { - log.Printf("config: ingest filter inactive — accepting all IATAs") + slog.Info("config: ingest filter inactive — accepting all IATAs", "component", "startup") } broker1 := ingest.New( ingest.Config{ BrokerName: "mqtt1", + LocalBorders: localBorders, URL: getEnv("MQTT_BROKER_1_URL"), Username: getEnv("MQTT_BROKER_1_USERNAME"), Password: getEnv("MQTT_BROKER_1_PASSWORD"), @@ -235,6 +352,7 @@ func main() { broker2 := ingest.New( ingest.Config{ BrokerName: "mqtt2", + LocalBorders: localBorders, URL: getEnv("MQTT_BROKER_2_URL"), Username: getEnv("MQTT_BROKER_2_USERNAME"), Password: getEnv("MQTT_BROKER_2_PASSWORD"), @@ -248,32 +366,86 @@ func main() { ) if cr, ok := reader.(*cache.CachedReader); ok { + if scopeImporter != nil { + scopeImporter.SetCacheInvalidator(cr.InvalidateScopeNames) + } broker1.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver) broker2.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver) } - go broker1.Start(ctx) - go broker2.Start(ctx) + var ingestWorkers sync.WaitGroup + ingestWorkers.Go(func() { broker1.Start(ctx) }) + ingestWorkers.Go(func() { broker2.Start(ctx) }) - scheduler := background.New([]background.Task{ + rollup := background.NewRollup(store, resolved.PacketRetention, resolved.RollupRetention) + go func() { + if err := rollup.CatchUp(ctx); err != nil { + slog.Error("analytics rollup catch-up failed", "component", "background", "error", err) + } + }() + tasks := []background.Task{ + rollup.Task(), background.ViewRefreshTask(store, resolved.ViewRefreshInterval), - background.CleanupTask(store, resolved.TelemetryRetention, resolved.PacketRetention, resolved.NodeDeleteAfter, resolved.CleanupInterval), - background.ReconfirmTask(store, resolved.ReconfirmInterval), - }) + background.CleanupTask(store, background.CleanupConfig{ + TelemetryRetention: resolved.TelemetryRetention, + PacketRetention: resolved.PacketRetention, + RollupRetention: resolved.RollupRetention, + NodeDeleteAfter: resolved.NodeDeleteAfter, + Interval: resolved.CleanupInterval, + }), + background.ReconfirmTask(store, resolved.RouteRetention, resolved.RouteGrace, int64(resolved.RouteMinObservations), resolved.ReconfirmInterval), + } + if resolved.ObserverDeleteAfter > 0 { + var onDelete func(context.Context, uuid.UUID) + if cr, ok := reader.(*cache.CachedReader); ok { + onDelete = cr.InvalidateObserver + } + tasks = append(tasks, background.ObserverCleanupTask(coalescer, resolved.ObserverDeleteAfter, resolved.CleanupInterval, onDelete)) + } + if scopeImporter != nil { + tasks = append(tasks, background.Task{Name: "meshmapper.scopes", Interval: meshmapper.PollInterval, Run: scopeImporter.Refresh}) + } + if cfg.MeshMapper.Zones.Enabled { + tasks = append(tasks, background.Task{Name: "meshmapper.zones", Interval: meshmapper.PollInterval, Run: zones.Refresh}) + } + if cfg.MeshMapper.Channels.Enabled { + tasks = append(tasks, background.Task{Name: "meshmapper.channels", Interval: meshmapper.PollInterval, Run: channelImporter.Refresh}) + } + profiles := configureProfiling(ctx, pool) + defer profiles.Stop() + for i := range tasks { + tasks[i].Run = profiles.WrapTask(tasks[i].Name, tasks[i].Run) + } + scheduler := background.New(tasks) go scheduler.Start(ctx) // ── HTTP server ────────────────────────────────────────────────────────── - r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, cfg.CORS) + // Wrap after wiring cache invalidators and cleanup callbacks to the actual + // CachedReader. Only response projections receive the geographic annotation. + reader = api.WithLocalBorders(reader, localBorders) + r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, router.Options{ + MaxConnsPerIP: resolved.MaxConnsPerIP, + MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, + WSAllowedOrigins: cfg.WebSocket.AllowedOrigins, + CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, + Scopes: scopes, RollupRetention: resolved.RollupRetention, + AdminRoutes: map[string]http.Handler{ + "/accounts": handlers.AccountsRouter(store), + "/backup": handlers.BackupRouter(backupOpts, ctx), + }, + }) srv := &http.Server{ - Addr: addr, - Handler: r, + Addr: addr, + Handler: r, + ErrorLog: slog.NewLogLogger(slog.Default().With("component", "http").Handler(), slog.LevelError), } go func() { - fmt.Printf("Beacon listening on %s\n", addr) + slog.Info(fmt.Sprintf("Beacon listening on %s", addr), "component", "startup") if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Fatalf("server error: %v", err) + slog.Error("server error", "component", "startup", "error", err) + os.Exit(1) } }() @@ -282,23 +454,36 @@ func main() { signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) <-quit - log.Println("shutting down...") + slog.Info("shutting down...", "component", "startup") cancel() // stops ingest workers shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) defer shutdownCancel() if err := srv.Shutdown(shutdownCtx); err != nil { - log.Printf("server shutdown error: %v", err) + slog.Error("server shutdown error", "component", "startup", "error", err) } + ingestWorkers.Wait() coalescer.Flush(shutdownCtx) } +// proxyLimitWarning explains why per-IP limits go site-wide behind an unlisted reverse proxy. +func proxyLimitWarning(r config.ResolvedConfig, trustedProxies int) string { + if trustedProxies > 0 { + return "" + } + limits := "the WebSocket connect limit" + if r.RateLimit.Enabled { + limits = "the REST rate limit and the WebSocket connect limit" + } + return fmt.Sprintf("warning: server.trusted_proxies is empty, so %s key on the direct peer; behind a reverse proxy every visitor shares one budget. List the proxy in server.trusted_proxies and have it set X-Real-IP", limits) +} + // getEnv returns the value of an env var and logs a warning if it is unset. // Callers that require the value to be non-empty should fatal themselves; // ingest workers tolerate missing broker config and will fail on connect instead. func getEnv(key string) string { v := os.Getenv(key) if v == "" { - log.Printf("warning: %s is not set", key) + slog.Warn(fmt.Sprintf("warning: %s is not set", key), "component", "startup") } return v } diff --git a/cmd/beacon/main_test.go b/cmd/beacon/main_test.go new file mode 100644 index 00000000..88f8ee40 --- /dev/null +++ b/cmd/beacon/main_test.go @@ -0,0 +1,76 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "bytes" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestProxyLimitWarning(t *testing.T) { + limited := config.ResolvedConfig{RateLimit: config.ResolvedRateLimitConfig{Enabled: true}, MaxConnectsPerMinute: 10} + if got := proxyLimitWarning(limited, 0); !strings.Contains(got, "server.trusted_proxies") || !strings.Contains(got, "X-Real-IP") { + t.Errorf("rate limited without proxies: %q", got) + } + wsOnly := config.ResolvedConfig{MaxConnectsPerMinute: 10} + if got := proxyLimitWarning(wsOnly, 0); !strings.Contains(got, "WebSocket") || strings.Contains(got, "REST") { + t.Errorf("WebSocket limit only: %q", got) + } + if got := proxyLimitWarning(limited, 1); got != "" { + t.Errorf("proxies configured: %q", got) + } +} + +func TestStartupLogging(t *testing.T) { + if os.Getenv("BEACON_TEST_LOG_STARTUP") == "1" { + main() + return + } + for _, level := range []string{"debug", "info", "warn", "error"} { + t.Run(level, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte("log: {level: debug, format: text}\n"), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(os.Args[0], "-test.run=^TestStartupLogging$") + cmd.Dir = dir + // Invalid port fails during parsing, before any database/network connection. + cmd.Env = append(os.Environ(), "BEACON_TEST_LOG_STARTUP=1", "CONFIG_PATH="+path, "LOG_LEVEL="+level, "LOG_FORMAT=json", "POSTGRES_DSN=postgres://test:do-not-log-this@localhost:invalid/beacon") + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatal("startup unexpectedly succeeded") + } + if strings.Contains(string(out), "do-not-log-this") { + t.Fatal("DSN credential leaked") + } + foundError := false + for _, line := range bytes.Split(bytes.TrimSpace(out), []byte("\n")) { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatalf("non-JSON startup output: %s", line) + } + if record["level"] == "ERROR" { + foundError = true + } + if level == "error" && record["level"] != "ERROR" { + t.Fatal("error threshold ignored") + } + if level == "warn" && record["level"] != "WARN" && record["level"] != "ERROR" { + t.Fatal("warn threshold ignored") + } + } + if !foundError { + t.Fatal("startup failure was silent") + } + }) + } +} diff --git a/cmd/beacon/profiling.go b/cmd/beacon/profiling.go new file mode 100644 index 00000000..df65d4cb --- /dev/null +++ b/cmd/beacon/profiling.go @@ -0,0 +1,31 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "log/slog" + "os" + + "github.com/MeshCore-Beacon/beacon-server/internal/profiling" + "github.com/jackc/pgx/v5/pgxpool" +) + +func configureProfiling(ctx context.Context, pool *pgxpool.Pool) *profiling.Recorder { + r, err := profiling.Start(ctx, os.Getenv("BEACON_CPU_PROFILE_DIR"), os.Getenv("BEACON_CPU_PROFILE_UNTIL"), []string{"reconfirm"}, func() any { + s := pool.Stat() + return struct { + Acquired, Idle, Total, Max int32 + Acquires, EmptyAcquires, CanceledAcquires int64 + AcquireDurationNS int64 + }{ + s.AcquiredConns(), s.IdleConns(), s.TotalConns(), s.MaxConns(), + s.AcquireCount(), s.EmptyAcquireCount(), s.CanceledAcquireCount(), int64(s.AcquireDuration()), + } + }) + if err != nil { + slog.Warn("CPU profiling unavailable", "component", "profiling", "error", err) + } + return r +} diff --git a/config.yaml.example b/config.yaml.example index d99f4de1..6044840e 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,11 +1,119 @@ -# Beacon configuration file -# Copy to config.yaml and adjust as needed. +# Beacon configuration file: local development starter. Copy to config.yaml and adjust. +# The fully annotated reference, with every key and its default, is +# https://github.com/MeshCore-Beacon/beacon-docs/blob/main/app_config/config.yaml.example +# Durations are Go strings (30s, 1h, 168h); negative values prevent startup +# (observers.delete_after excepted, where nonpositive disables it). + +# Public reads stay available. No key means admin routes return 503. +# Prefer BEACON_API_KEY in the service environment; never commit a real key. +# At least 16 characters with no inner whitespace (surrounding whitespace is +# trimmed); a shorter key prevents startup. +auth: + api_key: "" + +log: + # Environment LOG_LEVEL / LOG_FORMAT override these values. + level: info # debug, info, warn, error + format: text # text or json + +server: + # Only these direct proxy peers may set the client IP through X-Real-IP. + # The proxy must overwrite that header, not pass through client input. + # Leave empty for direct access (logs a startup warning). Invalid/empty CIDR + # entries prevent startup. A single host needs /32 for IPv4 or /128 for IPv6; + # bare IP addresses are not accepted. + # For Docker, list the actual proxy/compose network CIDR (e.g. 172.20.0.0/24). + # For a local host proxy, use ["127.0.0.1/32", "::1/128"]. + # Direct-client proxy examples; X-Forwarded-For alone is not used: + # Apache 2.4.10+ (mod_headers, in the proxy virtual host): + # RequestHeader set X-Real-IP "expr=%{REMOTE_ADDR}" + # nginx (in the proxy location): + # proxy_set_header X-Real-IP $remote_addr; + # Caddy (inside the reverse_proxy block): + # header_up X-Real-IP {http.request.remote.host} + # With additional proxy hops, first configure their trust at the edge proxy + # and pass its validated client address rather than a client-supplied header. + trusted_proxies: [] + +# Applies to /api/v1/* only. Both counters are local to this Beacon instance. +# Set trusted_proxies correctly before using a reverse proxy, or clients share +# the proxy's budget. IPv6 clients share a /64 budget to prevent address rotation. +ratelimit: + enabled: true + requests_per_minute: 300 + # Maximum requests in a one-second sliding window, not token-bucket capacity. + # Omitted/zero defaults to requests_per_minute. Negative limits prevent startup. + # A 429 returns error.code=rate_limited and Retry-After (1 or 60 seconds). + burst: 300 + +# MeshMapper integrations (optional, no API key). Import transport scopes and +# IATA border outlines from MeshMapper instead of maintaining scopes: and +# iatas.*.borderFile by hand. +# +# Scope-name discovery. Manual scopes remain authoritative. +# Every known IATA, including ones auto-created from traffic, is looked up in its +# country's zone list (shared with zones below) and its site's get_scopes.php +# catalogue is imported; IATAs MeshMapper doesn't list are skipped. No sources +# to configure; a legacy sources: list is ignored with a warning. +# Catalogues covering more than one IATA are rejected. +# 64 names per catalogue, 64 KiB per response; no page scraping. +# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. +# Refresh success/failure/freshness is logged under component=meshmapper.scopes. +# Turning scopes off deactivates imported matching keys on restart; +# historical scope identities and recorded evidence are retained. +# Region-filtered scope lists and scope stats show imported names only under the +# IATAs whose catalogue lists them; IATAs without a catalogue show manual scopes only. +#meshmapper: +# scopes: +# enabled: false +# refresh_interval: 1h # 1h-24h (MeshMapper allows one get_scopes.php per region per 55m); one source per 15s tick +# +# Region boundaries (Zones API). Every known IATA, including ones auto-created +# from traffic, is looked up in its country's zone list, then its get_geojson.php +# outline is imported; new IATAs are picked up while running. An imported boundary +# overrides that IATA's borderFile on the border map and for nodes.mark_foreign. +# A missing or failed boundary keeps the last good import, or the borderFile when +# there is none. Disabling drops every import on restart. Logged under +# component=meshmapper.zones. +# Each country's zone list is fetched at most once a day (MeshMapper allows one +# get_zones.php per country per 23.5h) and saved, so restarts don't refetch it. +# Failed or rate-limited requests also wait out the limit, or Retry-After if longer. +# Listed IATAs also take their display name and approximate location from the +# zone list, unless iatas: below sets a name or lat/lng for them. +# import_groups turns MeshMapper zone groups (e.g. ONQC: YOW, YQB, YUL) into +# regions, from the zone lists already fetched. A group is imported when at least +# one member is heard by an observer or configured here, with its full member list (members Beacon hasn't +# heard yet are added as IATAs); slug = lowercased group code, centered on the +# members' average location, listed after the regions below. A region below with +# the same slug wins. +# Imported regions are removed when the group disappears or stops qualifying, +# and all of them when import_groups is turned off. Requires zones.enabled. +# zones: +# enabled: false +# import_groups: false +# refresh_interval: 24h # 24h-168h (one get_geojson.php per region per 23.5h); one request per 15s tick, ETag-conditional +# +# Public hashtag channels (get_channels.php). Every known IATA that MeshMapper +# lists is looked up on its site, the same way as scopes. Each key and hash is +# checked against the name before use; a list with any mismatch is rejected and +# the last good one kept. Imported channels decrypt like configured ones (history +# is backfilled when a key first appears) and are listed under that IATA when +# /channels is filtered by region. Saved in PostgreSQL across restarts; failures +# wait out the 23.5h limit. Turning it off removes the region listing on restart. +# Logged under component=meshmapper.channels. +# channels: +# enabled: false +# refresh_interval: 24h # 24h-168h (one get_channels.php per region per 23.5h) iatas: YVR: name: Vancouver International lat: 49.1967 lng: -123.1815 + # borderFile: borders/yvr.geojson # optional GeoJSON Feature (Polygon/MultiPolygon) + # # (or use meshmapper.zones above) + # # for the region border map; path is relative to + # # this file's directory; validated at boot. YYJ: name: Victoria International lat: 48.6469 @@ -27,6 +135,8 @@ iatas: lat: 43.6777 lng: -79.6248 +# Hand-written regions; optional with meshmapper.zones.import_groups. Seeding +# sets each region's members to exactly this list. regions: - slug: western-canada name: Western Canada @@ -49,8 +159,12 @@ channel_keys: # secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0] # Hashtag names should be provided without the # prefix. # Beacon prepends # automatically before deriving the key. + # A plain name is listed under every region; add region: (a configured + # region) to list it only there when /channels is filtered by region. hashtags: - meshcore + # - name: ottawa-mesh + # region: eastern-canada # Explicit keys: provide the channel hash (hex) and key (hex) directly. # An optional name can be set for display purposes. @@ -58,31 +172,104 @@ channel_keys: "11": key: "8b3387e9c5cdea6ac9e5edbaa115cd72" name: "Public" + # region: eastern-canada # optional, as for hashtags # Regional transport scopes for matching TRANSPORT_FLOOD packets. # Plain names have # prepended automatically (e.g. "bc" → "#bc"). +# region (required) is a configured region slug; region-filtered scope lists +# and scope stats show the scope under that region's IATAs. Matching stays global. +# Optional when meshmapper.scopes covers your regions. scopes: - name: bc + region: western-canada - name: "#west" + region: western-canada telemetry: - retention: 672h # 4 weeks + retention: 744h # 31 days resolution: 1h # store one snapshot per observer per hour +# Optional authenticated database/config downloads. Disabled unless explicitly enabled. +# Requires auth.api_key, a supported PostgreSQL URL and a compatible pg_dump client. +# A failed prerequisite leaves this feature unavailable; ingest and other APIs continue. +#backup: +# enabled: false + +# Also covers observations and channel messages. packets: - retention: 720h # 30 days + retention: 168h # 7 days + +# Hourly analytics rollups (stats endpoints). Rollups outlive raw packets; this is also the +# longest window /stats/series and the other historical stats endpoints accept. +analytics: + rollup_retention: 2160h # 90 days + +# Presence write coalescing: observer last_seen and packet last_heard_at bumps are +# batched in memory and flushed on an interval. An unclean shutdown loses at most +# one interval of presence freshness. +#presence: +# flush_interval: 30s # default: 30s +# packet_ttl: 30s # how long a quiet packet stays coalesced (default: 30s) + +# Known-route retention. Routes are distilled packet history (the path a packet +# took, hop by hop), so they may outlive packets.retention; there is no required +# relationship between the two. +routes: + # How long a route is kept after it was last observed. Every new observation + # of the same path resets this clock, so routes the mesh still uses never + # expire -- only paths nothing has traveled for this long age out. + retention: 336h # 14 days + # Shorter window for rarely-seen routes: one observed fewer than + # min_observations times in total is dropped once it goes unobserved for + # this long. Must be shorter than retention, or it never applies. + grace: 168h # 7 days + # Lifetime observation count a route must reach to earn the full retention + # window. Below it a route is treated as flood noise -- a path recorded once + # or twice and never confirmed -- and only kept for the grace window. The + # count never resets, so once a route crosses this bar it stays in the + # retention tier for good. + min_observations: 3 websocket: max_connections_per_ip: 5 # default: 5 + # Upgrade attempts, including failed handshakes; zero/omitted defaults to 10. + # IPv6 clients share a /64 attempt budget. Negative values prevent startup. + # Rate exhaustion rejects the handshake with 429 and Retry-After: 60. + # Within this rate budget, a full concurrent cap accepts then closes with 1013 + # before hello, allowing browsers to back off without evicting existing clients. + max_connects_per_minute: 10 + # Other sites allowed to open /ws. Exact scheme://host[:port], or + # scheme://*.domain[:port] for every subdomain of domain at any depth. The + # wildcard never matches the apex itself, and scheme and port stay exact. + #allowed_origins: + # - https://example.com + # - https://*.example.com + +# Node staleness, deletion, and clock-drift thresholds. +#nodes: +# mark_foreign: false # opt in to possiblyForeign on repeater nodes +# # uses the union of configured iatas.*.borderFile polygons, +# # with meshmapper.zones boundaries replacing them; requires +# # a valid border file or meshmapper.zones; no packet filtering +# stale_threshold: 24h # default: 24h +# delete_after: 720h # default: 30 days +# clock_drift_threshold: 5m # default: 5m + +# Optional observer age-out. Omitted or nonpositive leaves observers intact. +# Enable only with one Beacon ingest process for this database. +# At most 1000 old observers are deleted per background.cleanup interval; +# retained observations, telemetry and ownership records prevent deletion. +#observers: +# delete_after: 720h # opt in to 30 days unseen # CORS configuration. # Controls which origins, methods and headers are allowed for cross-origin requests. -# Defaults to allowing all origins with GET/HEAD/OPTIONS if omitted — appropriate +# Defaults to allowing all origins with GET/HEAD/OPTIONS if omitted, appropriate # for a public read-only API. Restrict allowed_origins if you expose write endpoints. #cors: # allowed_origins: # - "https://app.example.com" -# allowed_methods: [GET, HEAD, OPTIONS, POST] +# allowed_methods: [GET, HEAD, OPTIONS, POST, PUT, DELETE] # admin UI writes # allowed_headers: [Accept, Authorization, Content-Type] # allow_credentials: false # max_age: 300 @@ -96,7 +283,7 @@ cache: ttl: "1h" ttls: # Aggregated stats endpoints (overview, observations, breakdowns, top nodes/observers, - # radio presets, scope stats). Backed by materialized views refreshed hourly. + # radio presets, scope stats). Backed by hourly rollups; keys change when a new hour rolls. stats: "1h" # Mostly-static reference data (IATAs, regions, scopes). # Only changes when new observers arrive or config is reseeded. @@ -121,6 +308,6 @@ cache: # Shorter intervals are useful during initial deployment to confirm data is # flowing. Back off to 1h or more once stable. #background: -# view_refresh: 1h # default: 1h +# view_refresh: 1h # rebuilds radio presets; default: 1h # reconfirm: 1h # default: 1h # cleanup: 1h # default: 1h diff --git a/db/accounts.go b/db/accounts.go new file mode 100644 index 00000000..336dcc61 --- /dev/null +++ b/db/accounts.go @@ -0,0 +1,78 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +var _ api.AccountStore = (*Store)(nil) + +func (s *Store) CreateAccount(ctx context.Context, name string) (api.Account, error) { + name, err := api.NormalizeAccountName(name) + if err != nil { + return api.Account{}, err + } + row, err := s.q.CreateAccount(ctx, name) + if errors.Is(err, pgx.ErrNoRows) { + return api.Account{}, api.ErrAccountNameConflict + } + if err != nil { + return api.Account{}, err + } + return accountFromRow(row), nil +} + +func (s *Store) ListAccounts(ctx context.Context) ([]api.Account, error) { + // ponytail: unpaginated operator list; add cursors if account counts grow large. + rows, err := s.q.ListAccounts(ctx) + if err != nil { + return nil, err + } + items := make([]api.Account, 0, len(rows)) + for _, row := range rows { + items = append(items, accountFromRow(row)) + } + return items, nil +} + +func (s *Store) GetAccount(ctx context.Context, id uuid.UUID) (api.Account, error) { + row, err := s.q.GetAccount(ctx, id) + if errors.Is(err, pgx.ErrNoRows) { + return api.Account{}, api.ErrAccountNotFound + } + if err != nil { + return api.Account{}, err + } + return accountFromRow(row), nil +} + +func (s *Store) DeactivateAccount(ctx context.Context, id uuid.UUID) error { + result, err := s.q.DeactivateAccount(ctx, id) + if err != nil { + return err + } + if !result.Found { + return api.ErrAccountNotFound + } + if !result.Deactivated { + return api.ErrAccountInactive + } + return nil +} + +func accountFromRow(row sqlc.Account) api.Account { + account := api.Account{ID: row.ID, Name: row.Name, CreatedAt: row.CreatedAt.Time.UTC(), Active: !row.DeactivatedAt.Valid} + if row.DeactivatedAt.Valid { + when := row.DeactivatedAt.Time.UTC() + account.DeactivatedAt = &when + } + return account +} diff --git a/db/accounts_integration_test.go b/db/accounts_integration_test.go new file mode 100644 index 00000000..a5426270 --- /dev/null +++ b/db/accounts_integration_test.go @@ -0,0 +1,149 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestAccountsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for account regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + setup, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer setup.Close(context.Background()) + schema := "account_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + quoted := pgx.Identifier{schema}.Sanitize() + if _, err := setup.Exec(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatal(err) + } + defer func() { + if _, err := setup.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error(err) + } + }() + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.MaxConns = 4 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + store := New(pool, 0, 0) + items, err := store.ListAccounts(ctx) + if err != nil || items == nil || len(items) != 0 { + t.Fatal("empty list", err) + } + if _, err := store.CreateAccount(ctx, "\t"); !errors.Is(err, api.ErrAccountNameInvalid) { + t.Fatal("invalid name accepted", err) + } + first, err := store.CreateAccount(ctx, " Montréal 🦀 ") + if err != nil || first.Name != "Montréal 🦀" || !first.Active || first.ID == uuid.Nil || first.CreatedAt.IsZero() { + t.Fatal("create", err) + } + if _, err := store.CreateAccount(ctx, first.Name); !errors.Is(err, api.ErrAccountNameConflict) { + t.Fatal("duplicate", err) + } + if err := store.DeactivateAccount(ctx, first.ID); err != nil { + t.Fatal(err) + } + inactive, err := store.GetAccount(ctx, first.ID) + if err != nil || inactive.Active || inactive.DeactivatedAt == nil { + t.Fatal("deactivate", err) + } + if err := store.DeactivateAccount(ctx, first.ID); !errors.Is(err, api.ErrAccountInactive) { + t.Fatal("repeated deactivate", err) + } + again, _ := store.GetAccount(ctx, first.ID) + if !again.DeactivatedAt.Equal(*inactive.DeactivatedAt) { + t.Fatal("deactivation timestamp changed") + } + reused, err := store.CreateAccount(ctx, first.Name) + if err != nil || reused.ID == first.ID { + t.Fatal("name reuse", err) + } + if _, err := store.CreateAccount(ctx, "montréal 🦀"); err != nil { + t.Fatal("names must be case-sensitive", err) + } + missing := uuid.New() + if _, err := store.GetAccount(ctx, missing); !errors.Is(err, api.ErrAccountNotFound) { + t.Fatal("missing get", err) + } + if err := store.DeactivateAccount(ctx, missing); !errors.Is(err, api.ErrAccountNotFound) { + t.Fatal("missing deactivate", err) + } + items, err = store.ListAccounts(ctx) + if err != nil || len(items) != 3 { + t.Fatal("list lifecycle", err) + } + for i := 1; i < len(items); i++ { + if items[i-1].CreatedAt.Before(items[i].CreatedAt) || (items[i-1].CreatedAt.Equal(items[i].CreatedAt) && items[i-1].ID.String() < items[i].ID.String()) { + t.Fatal("unstable list order") + } + } + start := make(chan struct{}) + results := make(chan error, 8) + for i := 0; i < 8; i++ { + go func() { <-start; _, err := store.CreateAccount(ctx, "concurrent"); results <- err }() + } + close(start) + created := 0 + for i := 0; i < 8; i++ { + err := <-results + if err == nil { + created++ + } else if !errors.Is(err, api.ErrAccountNameConflict) { + t.Fatal("concurrent create", err) + } + } + if created != 1 { + t.Fatalf("created %d accounts with the same name", created) + } + for round := 0; round < 8; round++ { + account, err := store.CreateAccount(ctx, "race-deactivate") + if err != nil { + t.Fatal(err) + } + gate := make(chan struct{}) + for i := 0; i < 2; i++ { + go func() { <-gate; results <- store.DeactivateAccount(ctx, account.ID) }() + } + close(gate) + changed := 0 + for i := 0; i < 2; i++ { + err := <-results + if err == nil { + changed++ + } else if !errors.Is(err, api.ErrAccountInactive) { + t.Fatal("concurrent deactivate", err) + } + } + if changed != 1 { + t.Fatalf("deactivated %d times", changed) + } + } +} diff --git a/db/accounts_test.go b/db/accounts_test.go new file mode 100644 index 00000000..e8489ee1 --- /dev/null +++ b/db/accounts_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +func TestAccountFromRow(t *testing.T) { + when := time.Date(2026, 1, 1, 12, 0, 0, 0, time.FixedZone("test", 3600)) + row := sqlc.Account{ID: uuid.New(), Name: "test", CreatedAt: pgtype.Timestamptz{Time: when, Valid: true}} + active := accountFromRow(row) + if active.ID != row.ID || active.Name != row.Name || !active.Active || active.DeactivatedAt != nil || !active.CreatedAt.Equal(when) || active.CreatedAt.Location() != time.UTC { + t.Fatal("active account mapping") + } + row.DeactivatedAt = pgtype.Timestamptz{Time: when.Add(time.Hour), Valid: true} + inactive := accountFromRow(row) + if inactive.Active || inactive.DeactivatedAt == nil || !inactive.DeactivatedAt.Equal(row.DeactivatedAt.Time) { + t.Fatal("inactive account mapping") + } +} diff --git a/db/baseline_integration_test.go b/db/baseline_integration_test.go new file mode 100644 index 00000000..27a8a606 --- /dev/null +++ b/db/baseline_integration_test.go @@ -0,0 +1,119 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Views cannot reference temporary tables. Keep fixtures and real migration DDL +// in a unique schema inside the caller's rolled-back transaction instead. +func isolateStatsSchema(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + schema := pgx.Identifier{"stats_test_" + uuid.NewString()}.Sanitize() + if _, err := tx.Exec(ctx, "CREATE SCHEMA "+schema+"; SET LOCAL search_path TO "+schema); err != nil { + t.Fatal(err) + } +} + +// applyBaseline builds the real schema in an isolated schema, plus the IATAs fixtures use. +func applyBaseline(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + isolateStatsSchema(t, ctx, tx) + ddl, err := migrationFiles.ReadFile("migrations/" + baselineMigration) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, string(ddl)); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, "INSERT INTO iata_codes (iata) VALUES ('YVR'),('YYJ'),('YYZ'),('YOW')"); err != nil { + t.Fatal(err) + } +} + +// schemaPool returns a pool whose connections all use a fresh, dropped-on-cleanup schema. +func schemaPool(t *testing.T) (context.Context, *pgxpool.Pool) { + t.Helper() + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + setup, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + schema := "migrate_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + quoted := pgx.Identifier{schema}.Sanitize() + if _, err := setup.Exec(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := setup.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error(err) + } + setup.Close(context.Background()) + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.MaxConns = 4 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + return ctx, pool +} + +func TestRunMigrationsBaselinePostgres(t *testing.T) { + ctx, pool := schemaPool(t) + for range 2 { + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + } + rows, err := pool.Query(ctx, "SELECT filename FROM schema_migrations ORDER BY filename") + if err != nil { + t.Fatal(err) + } + ledger, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil || len(ledger) != 1 || ledger[0] != baselineMigration { + t.Fatalf("ledger %v, %v", ledger, err) + } + // Stats refresh uses CONCURRENTLY, which needs populated views. + if _, err := pool.Exec(ctx, "REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets"); err != nil { + t.Fatal(err) + } +} + +func TestRunMigrationsRefusesPreBaselinePostgres(t *testing.T) { + for name, setup := range map[string]string{ + "1.x ledger": "CREATE TABLE schema_migrations (filename text PRIMARY KEY); INSERT INTO schema_migrations VALUES ('001_initial_schema.sql')", + "pre-ledger 1.x": "CREATE TABLE packets (packet_hash bytea PRIMARY KEY)", + } { + t.Run(name, func(t *testing.T) { + ctx, pool := schemaPool(t) + if _, err := pool.Exec(ctx, setup); err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); !errors.Is(err, errPreBaseline) { + t.Fatalf("got %v, want errPreBaseline", err) + } + }) + } +} diff --git a/db/channel_cursor_integration_test.go b/db/channel_cursor_integration_test.go new file mode 100644 index 00000000..a25eb9f7 --- /dev/null +++ b/db/channel_cursor_integration_test.go @@ -0,0 +1,208 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "reflect" + "strconv" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/jackc/pgx/v5" +) + +type channelCursorQueries struct { + sqlc.DBTX + query string + args []any +} + +func (q *channelCursorQueries) Query(ctx context.Context, query string, args ...any) (pgx.Rows, error) { + q.query, q.args = query, args + return q.DBTX.Query(ctx, query, args...) +} + +func TestChannelCursorPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + exec := func(sql string) { + t.Helper() + if _, err := tx.Exec(ctx, sql); err != nil { + t.Fatal(err) + } + } + applyBaseline(t, ctx, tx) + // NULL key_known is reported as false, so keyKnown=false must include it. + exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen,key_known) VALUES + (1,'\xaa','\x01','2026-09-08 12:00:00+00',true),(2,'\xaa','\x02','2026-09-08 12:00:00+00',false), + (3,'\xaa','\x03','2026-09-08 12:00:00+00',NULL),(4,'\xbb','\x04','2026-09-08 11:59:59+00',true), + (5,'\xcc','\x05','2026-09-08 12:00:00.000321+00',true), + (6,'\xcc','\x06','2026-09-08 12:00:00.000111+00',false),(7,'\xcc','\x07','2026-09-08 12:00:00.000111+00',true); +INSERT INTO meshmapper_channel_members(iata,key_fingerprint) VALUES + ('YOW','\x01'),('YOW','\x02'),('YOW','\x03'),('YOW','\x04'),('YYZ','\x05'),('YYZ','\x06'),('YYZ','\x07');`) + queries := &channelCursorQueries{DBTX: tx} + router := handlers.ChannelsRouter(&Store{q: sqlc.New(queries)}) + // Decode the wire contract so the same regression runs against the old server. + type wirePage struct { + Items []api.ChannelSummary `json:"items"` + NextCursor *int64 `json:"nextCursor"` + NextPageCursor *string `json:"nextPageCursor"` + HasMore bool `json:"hasMore"` + } + request := func(t *testing.T, params url.Values) wirePage { + t.Helper() + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/?"+params.Encode(), nil).WithContext(ctx)) + if w.Code != http.StatusOK { + t.Fatalf("channel page HTTP %d", w.Code) + } + var page wirePage + if err := json.Unmarshal(w.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + return page + } + for _, tc := range []struct { + name, query string + want []int + }{ + {"all ties and precision", "", []int{5, 7, 6, 3, 2, 1, 4}}, + {"hash", "hash=AA", []int{3, 2, 1}}, + {"region", "iata=yow", []int{3, 2, 1, 4}}, + {"multi region", "iatas=yow,yyz", []int{5, 7, 6, 3, 2, 1, 4}}, + {"hash and region", "hash=cc&iata=yyz", []int{5, 7, 6}}, + {"no regional match", "hash=aa&iata=yyz", nil}, + {"missing region", "iata=ZZZ", nil}, + {"key known", "keyKnown=true", []int{5, 7, 1, 4}}, + {"key unknown", "keyKnown=false", []int{6, 3, 2}}, + {"key known in region", "keyKnown=true&iata=yow", []int{1, 4}}, + {"key unknown in region", "keyKnown=false&iata=yow", []int{3, 2}}, + {"key unknown hash", "keyKnown=false&hash=cc", []int{6}}, + } { + t.Run(tc.name, func(t *testing.T) { + params, _ := url.ParseQuery(tc.query) + params.Set("limit", "2") + params.Set("cursor", "0") // Clients may retain the legacy first-page value. + var got []int + for n := 0; n < 10; n++ { + page := request(t, params) + for _, item := range page.Items { + got = append(got, item.ID) + } + if !page.HasMore { + if page.NextCursor != nil || page.NextPageCursor != nil { + t.Error("final page retains a cursor") + } + break + } + if page.NextCursor == nil { + t.Fatal("legacy numeric cursor was removed") + } + if page.NextPageCursor != nil { + params.Set("pageCursor", *page.NextPageCursor) + } else { + params.Set("cursor", strconv.FormatInt(*page.NextCursor, 10)) + } + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("paged IDs %v; want %v", got, tc.want) + } + }) + } + // The integer cursor still has its original strict millisecond semantics. + legacy := request(t, url.Values{"limit": {"50"}, "cursor": {"1788868800000"}}) + if len(legacy.Items) != 1 || legacy.Items[0].ID != 4 { + t.Error("legacy cursor semantics changed") + } + t.Run("newer activity stays ahead of the boundary", func(t *testing.T) { + first := request(t, url.Values{"limit": {"2"}}) + if first.NextPageCursor == nil { + t.Fatal("precise cursor missing") + } + exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) +VALUES (8,'\xee','\x08','2026-09-08 12:00:02+00')`) + rest := request(t, url.Values{"limit": {"50"}, "pageCursor": {*first.NextPageCursor}}) + var ids []int + for _, item := range rest.Items { + ids = append(ids, item.ID) + } + if !reflect.DeepEqual(ids, []int{6, 3, 2, 1, 4}) { + t.Errorf("new activity changed older traversal: %v", ids) + } + fresh := request(t, url.Values{"limit": {"2"}}) + if fresh.Items[0].ID != 8 { + t.Error("fresh first page missed new activity") + } + exec("DELETE FROM channels WHERE id=8") + }) + + exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) +SELECT i,'\xdd',decode(lpad(to_hex(i),16,'0'),'hex'), + '2026-09-09 12:00:00+00'::timestamptz-i*interval '1 millisecond' +FROM generate_series(1001,21000) i; +ANALYZE channels;`) + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + t.Run(mode, func(t *testing.T) { + exec("SET LOCAL plan_cache_mode = '" + mode + "'") + at := time.Date(2026, 9, 9, 12, 0, 0, 0, time.UTC).Add(-20 * time.Second) + page := request(t, url.Values{"limit": {"50"}, "pageCursor": {fmt.Sprintf("v1:%d:20000", at.UnixMicro())}}) + if len(page.Items) != 50 || page.Items[0].ID != 20001 { + t.Fatal("deep cursor returned the wrong page") + } + var raw []byte + if err := tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+queries.query, queries.args...).Scan(&raw); err != nil { + t.Fatal(err) + } + var plans []map[string]any + if err := json.Unmarshal(raw, &plans); err != nil { + t.Fatal(err) + } + rows := 0.0 + indexed := false + var visit func(map[string]any) + visit = func(plan map[string]any) { + if plan["Relation Name"] == "channels" { + n, _ := plan["Actual Rows"].(float64) + removed, _ := plan["Rows Removed by Filter"].(float64) + loops, _ := plan["Actual Loops"].(float64) + rows += (n + removed) * loops + _, indexed = plan["Index Cond"] + } + children, _ := plan["Plans"].([]any) + for _, child := range children { + visit(child.(map[string]any)) + } + } + visit(plans[0]["Plan"].(map[string]any)) + if rows > 60 || !indexed { + t.Errorf("deep page examines %.0f rows, indexed=%v", rows, indexed) + } + t.Logf("%s: examined %.0f rows; execution %v ms", mode, rows, plans[0]["Execution Time"]) + }) + } +} diff --git a/db/channel_message_count_integration_test.go b/db/channel_message_count_integration_test.go new file mode 100644 index 00000000..52558163 --- /dev/null +++ b/db/channel_message_count_integration_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" +) + +func TestChannelMessageCountPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +SELECT decode(lpad(to_hex(i), 2, '0'), 'hex'), 5, 0, 1, '\x00', '\x00', now(), now() FROM generate_series(1, 2) i`); err != nil { + t.Fatal(err) + } + // Re-upserting the channel (as every decrypt does) must not touch the count. + var id int + for range 3 { + if id, err = store.UpsertChannel(ctx, []byte{0x11}, []byte{1, 2, 3, 4, 5, 6, 7, 8}, "public", ""); err != nil { + t.Fatal(err) + } + } + for _, hash := range [][]byte{{1}, {1}, {2}} { // second {1} is a duplicate + if _, err := store.InsertChannelMessage(ctx, ingest.InsertChannelMessageParams{ + ChannelID: id, PacketHash: hash, SenderName: "a", Content: "b", SentAt: time.Now(), + }); err != nil { + t.Fatal(err) + } + } + ch, err := store.GetChannel(ctx, int32(id)) + if err != nil { + t.Fatal(err) + } + if ch.MessageCount != 2 { + t.Errorf("message_count = %d, want 2", ch.MessageCount) + } +} diff --git a/db/channel_scopes_integration_test.go b/db/channel_scopes_integration_test.go new file mode 100644 index 00000000..ab9a9c46 --- /dev/null +++ b/db/channel_scopes_integration_test.go @@ -0,0 +1,185 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "net/http/httptest" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestChannelMessageScopesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"transport_scopes", "packets", "packet_observations", "channels", "channel_messages"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO transport_scopes(id,name,transport_key,key_fingerprint) VALUES +(1,'#yow',decode(repeat('00',16),'hex'),decode(repeat('01',8),'hex')), +(2,'#can',decode(repeat('02',16),'hex'),decode(repeat('03',8),'hex')); +INSERT INTO channels(id,channel_hash,key_fingerprint,name,is_hashtag,key_known) VALUES (123,'\x11','\x01020304','#fixture',true,true); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,transport_codes_present,scope_id,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),5,0,1,transport,scope_id,'\x00','\x15',NOW(),NOW() +FROM (VALUES(1,true,1),(2,true,NULL),(3,false,NULL),(4,NULL,NULL),(5,true,2),(6,false,NULL)) v(i,transport,scope_id); +INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT packet_hash,md5(n::text)::uuid,'YOW',NOW(),0,1,0 FROM packets CROSS JOIN generate_series(1,2) n; +UPDATE packets SET observation_count=2; +`) + if err != nil { + t.Fatal(err) + } + s := &Store{q: sqlc.New(tx)} + // Later matching reception must not relabel the immutable first stored packet. + scopeID := int32(1) + _, err = s.UpsertPacket(ctx, ingest.UpsertPacketParams{PacketHash: []byte{6}, PayloadType: 5, ScopeID: &scopeID, TransportCodes: []byte{1, 2, 3, 4}, RawPayload: []byte{0}, RawHeader: []byte{0}}) + if err != nil { + t.Fatal(err) + } + statuses := []api.ChannelScopeStatus{"matched", "unknown", "unscoped", "unavailable", "matched", "unscoped"} + wants := map[int64]*ingest.InsertedChannelMessage{} + for n, status := range statuses { + m := ingest.InsertChannelMessageParams{ChannelID: 123, PacketHash: []byte{byte(n + 1)}, SenderName: "Fixture", Content: "Synthetic message", SentAt: time.Now()} + inserted, err := s.InsertChannelMessage(ctx, m) + if err != nil || inserted == nil || inserted.ScopeStatus != status { + t.Fatalf("insert %d: %+v %v", n, inserted, err) + } + wants[inserted.ID] = inserted + duplicate, err := s.InsertChannelMessage(ctx, m) + if err != nil || duplicate != nil { + t.Fatal("duplicate changed live delivery", duplicate, err) + } + } + check := func(items []api.ChannelMessage) { + t.Helper() + if len(items) != 6 { + t.Fatalf("messages=%d", len(items)) + } + for _, m := range items { + want := wants[m.ID] + if want == nil || m.ScopeStatus != want.ScopeStatus { + t.Fatal("history differs from insertion", m) + } + if (m.Scope == nil) != (want.Scope == nil) || (m.Scope != nil && *m.Scope != *want.Scope) { + t.Fatal("scope differs", m) + } + if m.ObservationCount != 2 { + t.Fatal("duplicate observations changed message grain", m) + } + b, _ := json.Marshal(m) + var wire map[string]any + _ = json.Unmarshal(b, &wire) + if _, ok := wire["scope"]; !ok || wire["scopeStatus"] != string(m.ScopeStatus) { + t.Fatal("wire metadata absent", string(b)) + } + } + } + ch := int32(123) + for _, id := range []*int32{nil, &ch} { + p, err := s.ListChannelMessages(ctx, id, time.Time{}, 20, []string{"YOW"}, "", 0) + if err != nil { + t.Fatal(err) + } + check(p.Items) + } + hash, err := s.ListChannelMessagesByHash(ctx, []byte{0x11}, time.Time{}, 20, []string{"YOW"}, "", 0) + if err != nil { + t.Fatal(err) + } + check(hash.Items) + catchup, err := s.ListMessagesAfterID(ctx, 0, []string{"YOW"}, "", 20) + if err != nil { + t.Fatal(err) + } + check(catchup) + router := chi.NewRouter() + router.Mount("/channels", handlers.ChannelsRouter(s)) + router.Mount("/messages", handlers.MessagesRouter(s)) + for _, path := range []string{"/channels/123/messages", "/messages", "/messages?channelHash=11", "/messages/backfill?afterId=0"} { + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest("GET", path, nil).WithContext(ctx)) + if response.Code != 200 { + t.Fatalf("%s: %d %s", path, response.Code, response.Body.String()) + } + if path == "/messages/backfill?afterId=0" { + var items []api.ChannelMessage + if err := json.Unmarshal(response.Body.Bytes(), &items); err != nil { + t.Fatal(err) + } + check(items) + } else { + var body struct { + Items []api.ChannelMessage `json:"items"` + } + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + check(body.Items) + } + } + filtered, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, []string{"YOW"}, "#yow", 0) + if err != nil || len(filtered.Items) != 1 || filtered.Items[0].Scope == nil || *filtered.Items[0].Scope != "#yow" { + t.Fatal(filtered, err) + } + catchup, err = s.ListMessagesAfterID(ctx, 0, []string{"YOW"}, "#yow", 20) + if err != nil || len(catchup) != 1 { + t.Fatal(catchup, err) + } + other, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, []string{"YVR"}, "", 0) + if err != nil || len(other.Items) != 0 { + t.Fatal("region escaped", other, err) + } + first, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 2, nil, "", 0) + if err != nil || !first.HasMore || first.NextCursor == nil { + t.Fatal(first, err) + } + second, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 2, nil, "", *first.NextCursor) + if err != nil || len(second.Items) != 2 || second.Items[0].ID >= *first.NextCursor { + t.Fatal("cursor drift", second, err) + } + // Even without the production cascade, expired packet evidence must not leak orphan messages. + if _, err := tx.Exec(ctx, "DELETE FROM packets"); err != nil { + t.Fatal(err) + } + expired, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, nil, "", 0) + if err != nil || len(expired.Items) != 0 { + t.Fatal(expired, err) + } +} diff --git a/db/channels.go b/db/channels.go index dbdbddc9..c292b766 100644 --- a/db/channels.go +++ b/db/channels.go @@ -26,12 +26,11 @@ func (s *Store) UpsertChannel(ctx context.Context, channelHash []byte, keyFinger } isHashtag := hashtag != "" row, err := s.q.UpsertChannel(ctx, sqlc.UpsertChannelParams{ - ChannelHash: channelHash, - Column2: keyFingerprint, // key_fingerprint - Name: namePtr, - Hashtag: hashtagPtr, - IsHashtag: &isHashtag, - MessageCount: nil, // message count bumped separately by InsertChannelMessage + ChannelHash: channelHash, + Column2: keyFingerprint, // key_fingerprint + Name: namePtr, + Hashtag: hashtagPtr, + IsHashtag: &isHashtag, }) if err != nil { return 0, err @@ -65,6 +64,31 @@ func (s *Store) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ingest.U return packets, nil } +// ListUndecryptedGroupTextPacketsByHash limits the boot backfill to channels that just gained a key. +func (s *Store) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ingest.UndecryptedPacket, error) { + rows, err := s.q.ListUndecryptedGroupTextPacketsByHash(ctx, hashes) + if err != nil { + return nil, err + } + packets := make([]ingest.UndecryptedPacket, 0, len(rows)) + for _, v := range rows { + packets = append(packets, ingest.UndecryptedPacket{PacketHash: v.PacketHash, RawPayload: v.RawPayload}) + } + return packets, nil +} + +// SetChannelConfigScopes replaces every configured channel's region placement; +// an empty region means Beacon-wide. +func (s *Store) SetChannelConfigScopes(ctx context.Context, fingerprints [][]byte, regions []string) error { + if err := s.q.DeleteChannelConfigScopes(ctx); err != nil { + return err + } + if len(fingerprints) == 0 { + return nil + } + return s.q.AddChannelConfigScopes(ctx, sqlc.AddChannelConfigScopesParams{Fingerprints: fingerprints, Regions: regions}) +} + func (s *Store) UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error { return s.q.UpsertChannelIATA(ctx, sqlc.UpsertChannelIATAParams{ ChannelHash: channelHash, @@ -77,19 +101,25 @@ func (s *Store) DeleteOldChannelIATAs(ctx context.Context, cutoff time.Time) err return s.q.DeleteOldChannelIATAs(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } -func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) { +func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { var cursorTS pgtype.Timestamptz if cursor > 0 { cursorTS = pgtype.Timestamptz{Time: time.UnixMilli(cursor), Valid: true} } - rows, err := s.q.ListChannels(ctx, sqlc.ListChannelsParams{ - ChannelHash: hash, - Iatas: iatas, - CursorTs: cursorTS, - PageLimit: limit + 1, - }) + var rows []sqlc.Channel + var err error + if pageCursor != nil { + rows, err = s.q.ListChannelsAfter(ctx, sqlc.ListChannelsAfterParams{ + ChannelHash: hash, Iatas: iatas, KeyKnown: keyKnown, PageLimit: limit + 1, + CursorTs: pgtype.Timestamptz{Time: pageCursor.LastSeen, Valid: true}, CursorID: pageCursor.ID, + }) + } else { + rows, err = s.q.ListChannels(ctx, sqlc.ListChannelsParams{ + ChannelHash: hash, Iatas: iatas, KeyKnown: keyKnown, CursorTs: cursorTS, PageLimit: limit + 1, + }) + } if err != nil { - return api.Page[api.ChannelSummary]{}, err + return api.ChannelPage{}, err } hasMore := len(rows) > int(limit) if hasMore { @@ -107,14 +137,17 @@ func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iata }) } var nextCursor *int64 + var nextPageCursor *string if hasMore { last := items[len(items)-1].LastSeen nextCursor = &last + row := rows[len(rows)-1] + precise := (api.ChannelCursor{LastSeen: row.LastSeen.Time, ID: row.ID}).String() + nextPageCursor = &precise } - return api.Page[api.ChannelSummary]{ - Items: items, - NextCursor: nextCursor, - HasMore: hasMore, + return api.ChannelPage{ + Page: api.Page[api.ChannelSummary]{Items: items, NextCursor: nextCursor, HasMore: hasMore}, + NextPageCursor: nextPageCursor, }, nil } @@ -133,10 +166,7 @@ func (s *Store) GetChannel(ctx context.Context, channelID int32) (*api.Channel, KeyKnown: row.KeyKnown != nil && *row.KeyKnown, }, Hashtag: row.Hashtag, - MessageCount: 0, - } - if row.MessageCount != nil { - channel.MessageCount = *row.MessageCount + MessageCount: row.MessageCount, } if row.IsHashtag != nil && *row.IsHashtag && row.KeyFingerprint != nil { fp := hex.EncodeToString(row.KeyFingerprint) @@ -145,16 +175,16 @@ func (s *Store) GetChannel(ctx context.Context, channelID int32) (*api.Channel, return &channel, nil } -func (s *Store) InsertChannelMessage(ctx context.Context, m ingest.InsertChannelMessageParams) (bool, error) { - params := sqlc.InsertChannelMessageParams{ChannelID: int32(m.ChannelID), PacketHash: m.PacketHash, SenderName: &m.SenderName, Content: &m.Content, SentAt: pgtype.Timestamptz{Time: m.SentAt, Valid: true}} - _, err := s.q.InsertChannelMessage(ctx, params) +func (s *Store) InsertChannelMessage(ctx context.Context, m ingest.InsertChannelMessageParams) (*ingest.InsertedChannelMessage, error) { + params := sqlc.InsertChannelMessageParams{ChannelID: int32(m.ChannelID), PacketHash: m.PacketHash, SenderName: &m.SenderName, Content: &m.Content, SentAt: pgtype.Timestamptz{Time: m.SentAt, Valid: true}, Column6: m.Historical} + row, err := s.q.InsertChannelMessage(ctx, params) if errors.Is(err, pgx.ErrNoRows) { - return false, nil // duplicate + return nil, nil // duplicate } if err != nil { - return false, err + return nil, err } - return true, nil + return &ingest.InsertedChannelMessage{ID: row.ID, Scope: row.ScopeName, ScopeStatus: api.RecordedChannelScopeStatus(row.ScopeName, row.TransportCodesPresent)}, nil } func (s *Store) ListChannelMessages(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) { @@ -178,7 +208,7 @@ func (s *Store) ListChannelMessages(ctx context.Context, channelID *int32, since } messages = make([]api.ChannelMessage, 0, len(rows)) for _, v := range rows { - messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount)) + messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount, v.ScopeName, v.TransportCodesPresent)) } } else { rows, err := s.q.ListChannelMessages(ctx, sqlc.ListChannelMessagesParams{ @@ -198,7 +228,7 @@ func (s *Store) ListChannelMessages(ctx context.Context, channelID *int32, since } messages = make([]api.ChannelMessage, 0, len(rows)) for _, v := range rows { - messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount)) + messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount, v.ScopeName, v.TransportCodesPresent)) } } @@ -232,7 +262,7 @@ func (s *Store) ListChannelMessagesByHash(ctx context.Context, hash []byte, sinc } messages := make([]api.ChannelMessage, 0, len(rows)) for _, v := range rows { - messages = append(messages, toChannelMessage(v.ID, hex.EncodeToString(v.PacketHash), v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount)) + messages = append(messages, toChannelMessage(v.ID, hex.EncodeToString(v.PacketHash), v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount, v.ScopeName, v.TransportCodesPresent)) } var nextCursor *int64 if hasMore && len(messages) > 0 { @@ -266,6 +296,8 @@ func (s *Store) ListMessagesAfterID(ctx context.Context, afterID int64, iatas [] v.Content, v.SentAt, v.ObservationCount, + v.ScopeName, + v.TransportCodesPresent, )) } return items, nil diff --git a/db/channels_test.go b/db/channels_test.go index 8fdea060..be0fd464 100644 --- a/db/channels_test.go +++ b/db/channels_test.go @@ -11,6 +11,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/jackc/pgx/v5/pgtype" "go.uber.org/mock/gomock" ) @@ -29,7 +30,7 @@ func TestListChannels_Empty(t *testing.T) { Return([]sqlc.Channel{}, nil) store := &Store{q: mock} - page, err := store.ListChannels(context.Background(), 10, nil, nil, 0) + page, err := store.ListChannels(context.Background(), 10, nil, nil, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -71,7 +72,7 @@ func TestListChannels_Pagination(t *testing.T) { Return(rows, nil) store := &Store{q: mock} - page, err := store.ListChannels(context.Background(), 2, nil, nil, 0) + page, err := store.ListChannels(context.Background(), 2, nil, nil, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -95,7 +96,7 @@ func TestListChannels_DBError(t *testing.T) { Return(nil, errors.New("db error")) store := &Store{q: mock} - _, err := store.ListChannels(context.Background(), 10, nil, nil, 0) + _, err := store.ListChannels(context.Background(), 10, nil, nil, nil, 0, nil) if err == nil { t.Fatal("expected error, got nil") } @@ -115,12 +116,37 @@ func TestListChannels_IATAFilter(t *testing.T) { Return([]sqlc.Channel{}, nil) store := &Store{q: mock} - _, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, 0) + _, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } } +func TestListChannels_KeyKnownFilter(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + keyKnown := false + at := time.UnixMilli(1700000000000) + + mock.EXPECT(). + ListChannels(gomock.Any(), sqlc.ListChannelsParams{KeyKnown: &keyKnown, PageLimit: 11}). + Return([]sqlc.Channel{}, nil) + mock.EXPECT(). + ListChannelsAfter(gomock.Any(), sqlc.ListChannelsAfterParams{ + KeyKnown: &keyKnown, PageLimit: 11, + CursorTs: pgtype.Timestamptz{Time: at, Valid: true}, CursorID: 7, + }). + Return([]sqlc.Channel{}, nil) + + store := &Store{q: mock} + if _, err := store.ListChannels(context.Background(), 10, nil, nil, &keyKnown, 0, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := store.ListChannels(context.Background(), 10, nil, nil, &keyKnown, 0, &api.ChannelCursor{LastSeen: at, ID: 7}); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + func TestGetChannel_Basic(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) @@ -138,7 +164,7 @@ func TestGetChannel_Basic(t *testing.T) { LastSeen: lastSeen, IsHashtag: &isHashtag, KeyKnown: &keyKnown, - MessageCount: &msgCount, + MessageCount: msgCount, }, nil) store := &Store{q: mock} diff --git a/db/clock_drift_integration_test.go b/db/clock_drift_integration_test.go new file mode 100644 index 00000000..4145a80b --- /dev/null +++ b/db/clock_drift_integration_test.go @@ -0,0 +1,191 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "os" + "slices" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +//go:embed queries/queries.sql +var clockTestQueries string + +// Fixture writes are confined to temporary tables in a rolled-back transaction. +func TestClockDriftPagePostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"nodes", "node_iatas"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO nodes (id,public_key,node_type,name,device_clock_drift_seconds,last_advert_at) +SELECT ('00000000-0000-0000-0000-'||lpad(i::text,12,'0'))::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),kind, + CASE WHEN i=2 THEN NULL ELSE 'node-'||i END,drift,CASE WHEN i=2 THEN NULL ELSE '2026-01-01 00:00:00+00'::timestamptz END +FROM (VALUES (1,2,120),(2,3,-240),(3,1,99999),(4,4,-99999),(5,2,NULL),(6,2,60),(7,3,-60),(8,2,180)) v(i,kind,drift); +INSERT INTO node_iatas (node_id,iata,last_heard) VALUES + ('00000000-0000-0000-0000-000000000001','YVR','2026-01-01 00:00:00+00'), + ('00000000-0000-0000-0000-000000000001','YYJ','2026-01-01 01:00:00+00'), + ('00000000-0000-0000-0000-000000000002','YYJ','2026-01-01 00:00:00+00');`) + if err != nil { + t.Fatal(err) + } + q := sqlc.New(tx) + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + iatas []string + threshold, limit int32 + want []int + }{ + {"global", nil, 60, 10, []int{2, 8, 1}}, + {"empty region array", []string{}, 60, 10, []int{2, 8, 1}}, + {"page limit", nil, 60, 2, []int{2, 8}}, + {"YVR", []string{"YVR"}, 60, 10, []int{1}}, + {"YYJ", []string{"YYJ"}, 60, 10, []int{2, 1}}, + {"multiple regions", []string{"YVR", "YYJ"}, 60, 10, []int{2, 1}}, + {"duplicate region", []string{"YYJ", "YYJ"}, 60, 10, []int{2, 1}}, + {"unknown region", []string{"ZZZ"}, 60, 10, nil}, + {"strict threshold", nil, 180, 10, []int{2}}, + {"boundary", nil, 240, 10, nil}, + {"positive and negative boundary", nil, 0, 10, []int{2, 8, 1, 6, 7}}, + } { + t.Run(tc.name, func(t *testing.T) { + rows, err := q.GetStatsClockDrift(ctx, sqlc.GetStatsClockDriftParams{Column1: tc.threshold, Column2: tc.iatas, Limit: tc.limit}) + if err != nil { + t.Fatal(err) + } + if len(rows) != len(tc.want) { + t.Fatalf("got %d rows, want %d", len(rows), len(tc.want)) + } + previous := int64(1 << 62) + seen := map[int]bool{} + for _, row := range rows { + var number int + if _, err := fmt.Sscanf(row.ID.String(), "00000000-0000-0000-0000-%d", &number); err != nil { + t.Fatal(err) + } + if !slices.Contains(tc.want, number) || seen[number] { + t.Fatalf("unexpected/duplicate node %d", number) + } + seen[number] = true + if row.DeviceClockDriftSeconds == nil { + t.Fatal("null drift returned") + } + wantDrift := map[int]int32{1: 120, 2: -240, 6: 60, 7: -60, 8: 180}[number] + if *row.DeviceClockDriftSeconds != wantDrift || (row.NodeType != 2 && row.NodeType != 3) { + t.Fatal("drift sign or role changed") + } + magnitude := int64(wantDrift) + if magnitude < 0 { + magnitude = -magnitude + } + if magnitude > previous { + t.Fatal("not ordered worst first") + } + previous = magnitude + if number == 2 { + if row.Name != nil || row.LastAdvertAt.Valid { + t.Fatal("nullable node fields changed") + } + } else if row.Name == nil || *row.Name != fmt.Sprintf("node-%d", number) || !row.LastAdvertAt.Valid || !row.LastAdvertAt.Time.Equal(anchor) { + t.Fatal("node fields changed") + } + var got []api.NodeIATA + if len(row.Iatas) > 0 { + if err := json.Unmarshal(row.Iatas, &got); err != nil { + t.Fatal(err) + } + } + var want []api.NodeIATA + if number == 1 { + want = []api.NodeIATA{{IATA: "YYJ", LastHeard: anchor.Add(time.Hour).UnixMilli()}, {IATA: "YVR", LastHeard: anchor.UnixMilli()}} + } + if number == 2 { + want = []api.NodeIATA{{IATA: "YYJ", LastHeard: anchor.UnixMilli()}} + } + if !slices.Equal(got, want) { + t.Fatalf("node %d memberships/order: got %v, want %v", number, got, want) + } + } + }) + } + // Use a separate large population for the work-budget measurement. + _, err = tx.Exec(ctx, ` +TRUNCATE pg_temp.node_iatas,pg_temp.nodes; +INSERT INTO nodes (id,public_key,node_type,name,device_clock_drift_seconds,last_advert_at) +SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),2,'clock-'||i,3600+i,'2026-01-01 00:00:00+00' +FROM generate_series(1,20000) i; +INSERT INTO node_iatas (node_id,iata,last_heard) +SELECT md5(i::text)::uuid,iata,'2026-01-01 00:00:00+00'::timestamptz-i*interval '1 second' +FROM generate_series(1,20000) i CROSS JOIN (VALUES ('YVR'),('YYJ')) regions(iata); +ANALYZE nodes; ANALYZE node_iatas;`) + if err != nil { + t.Fatal(err) + } + queries := strings.ReplaceAll(clockTestQueries, "\r\n", "\n") + query := strings.Split(strings.Split(queries, "-- name: GetStatsClockDrift :many\n")[1], "\n-- name:")[0] + if _, err := tx.Exec(ctx, "PREPARE clock_page AS "+query); err != nil { + t.Fatal(err) + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err := tx.Exec(ctx, "SET LOCAL plan_cache_mode = "+mode); err != nil { + t.Fatal(err) + } + var raw []byte + if err := tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE clock_page(60,NULL,50)").Scan(&raw); err != nil { + t.Fatal(err) + } + var plans []struct { + Plan map[string]any + ExecutionTime float64 `json:"Execution Time"` + } + if err := json.Unmarshal(raw, &plans); err != nil { + t.Fatal(err) + } + var memberships float64 + var walk func(map[string]any) + walk = func(p map[string]any) { + if p["Relation Name"] == "node_iatas" { + memberships += p["Actual Rows"].(float64) * p["Actual Loops"].(float64) + } + if children, ok := p["Plans"].([]any); ok { + for _, child := range children { + walk(child.(map[string]any)) + } + } + } + walk(plans[0].Plan) + t.Logf("20,000 nodes / 50-row clock page %s: %.3f ms, %.0f membership rows processed", mode, plans[0].ExecutionTime, memberships) + if memberships > 150 { + t.Errorf("clock page processed %.0f membership rows; budget 150", memberships) + } + } +} diff --git a/db/config.go b/db/config.go index ac15b6f8..3836a331 100644 --- a/db/config.go +++ b/db/config.go @@ -166,9 +166,13 @@ func (s *Store) GetRegionBySlug(ctx context.Context, slug string) (*api.Region, return &result, nil } -func (s *Store) UpsertRegionIATA(ctx context.Context, regionID int32, iata string) error { - return s.q.UpsertRegionIATA(ctx, sqlc.UpsertRegionIATAParams{ - RegionID: regionID, - Iata: iata, - }) +// SetRegionIATAs makes iatas the region's exact member list. +func (s *Store) SetRegionIATAs(ctx context.Context, regionID int32, iatas []string) error { + if iatas == nil { + iatas = []string{} + } + if err := s.q.DeleteRegionIATAsNotIn(ctx, sqlc.DeleteRegionIATAsNotInParams{RegionID: regionID, Keep: iatas}); err != nil { + return err + } + return s.q.AddRegionIATAs(ctx, sqlc.AddRegionIATAsParams{RegionID: regionID, Iatas: iatas}) } diff --git a/db/endpoint_matching_integration_test.go b/db/endpoint_matching_integration_test.go new file mode 100644 index 00000000..734c2085 --- /dev/null +++ b/db/endpoint_matching_integration_test.go @@ -0,0 +1,219 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +type endpointMatchQueries struct { + sqlc.DBTX + calls int + query string +} + +func (q *endpointMatchQueries) Query(ctx context.Context, query string, args ...any) (pgx.Rows, error) { + q.calls++ + q.query = query + return q.DBTX.Query(ctx, query, args...) +} + +func TestPacketEndpointRolesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + exec := func(sql string, args ...any) { + t.Helper() + if _, err := tx.Exec(ctx, sql, args...); err != nil { + t.Fatal(err) + } + } + for _, table := range []string{"packets", "packet_observations", "observers", "transport_scopes", "channel_messages", "nodes", "node_short_ids"} { + exec("CREATE TEMP TABLE " + table + " (LIKE public." + table + " INCLUDING ALL) ON COMMIT DROP") + } + id := func(i int) uuid.UUID { return uuid.MustParse(fmt.Sprintf("00000000-0000-0000-0000-%012d", i)) } + exec(`INSERT INTO nodes(id,public_key,name,node_type) +SELECT ('00000000-0000-0000-0000-'||lpad(i::text,12,'0'))::uuid, + decode(prefix||repeat('00',28),'hex'),name,role +FROM (VALUES (1,'aa000001','Companion',1),(2,'aa000002','Repeater',2), + (3,'ee000003','Room',3),(4,'aa000004','Sensor',4),(5,'aa000005','Unknown role',0), + (6,'bb000006','Companion only',1),(7,'aa000007','Other region',1), + (8,'cc000008','Relay',2),(9,'dd000009',NULL,1),(10,'aa000010','No membership',1)) v(i,prefix,name,role); +INSERT INTO node_short_ids(node_id,iata,prefix_4) +SELECT id,CASE WHEN name='Other region' THEN 'YYZ' ELSE 'YVR' END,substring(public_key from 1 for 4) +FROM nodes WHERE name IS DISTINCT FROM 'No membership';`) + exec("INSERT INTO observers(id,public_key) VALUES ($1,'\\x01'),($2,'\\x02')", id(101), id(102)) + queries := &endpointMatchQueries{DBTX: tx} + store := &Store{q: sqlc.New(queries)} + check := func(hop *api.ResolvedHop, confidence string, members ...int) { + t.Helper() + if hop == nil || hop.Confidence != confidence || len(hop.Nodes) != len(members) { + t.Errorf("endpoint candidate roles incorrect: confidence=%s, want %s with %d nodes", func() string { + if hop == nil { + return "nil" + } + return hop.Confidence + }(), confidence, len(members)) + return + } + want := make(map[uuid.UUID]bool) + for _, member := range members { + want[id(member)] = true + } + for _, node := range hop.Nodes { + if !want[node.ID] { + t.Errorf("unexpected endpoint node %s", node.ID) + } + delete(want, node.ID) + } + if len(want) != 0 { + t.Error("endpoint candidate missing") + } + } + for i, kind := range []int{0, 1, 2, 8} { + // These encrypted envelopes carry one-byte destination/source prefixes. + hash := []byte{byte(i + 1)} + payload := append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...) + exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +VALUES ($1,$2,0,1,$3,$4,NOW(),NOW())`, hash, kind, payload, []byte{byte(kind<<2 | 1)}) + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,path_bytes,source_broker) +VALUES ($1,$2,$3,'YVR',NOW(),1,1,1,'\xcc','fixture')`, i+1, hash, id(101)) + packet, err := store.GetPacket(ctx, hash) + if err != nil || len(packet.Observations) != 1 { + t.Fatalf("packet detail: %v", err) + } + observation := packet.Observations[0] + check(observation.ResolvedSource, "ambiguous", 1, 2, 4, 5) + check(observation.ResolvedDestination, "high", 6) + if len(observation.ResolvedPath) != 1 { + t.Fatal("relay path missing") + } + check(&observation.ResolvedPath[0], "high", 8) + } + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,source_broker) +VALUES (10,'\x01',$1,'YYZ',NOW()+interval '1 second',0,1,0,'fixture')`, id(102)) + packet, err := store.GetPacket(ctx, []byte{1}) + if err != nil || len(packet.Observations) != 2 { + t.Fatalf("regional observations: %v", err) + } + check(packet.Observations[1].ResolvedSource, "high", 7) + check(packet.Observations[1].ResolvedDestination, "none") + // Intermediate relay matching keeps its existing infrastructure-only boundary. + for width := 1; width <= 4; width++ { + hash := []byte{0xaa, 0, 0, 2}[:width] + resolved, err := store.ResolvePathHashes(ctx, "YVR", [][]byte{hash}) + if err != nil { + t.Fatal(err) + } + hops := api.BuildResolvedPath([][]byte{hash}, resolved) + check(&hops[0], "high", 2) + } + t.Log("endpoint roles and unchanged relay matching checked across four payload types") + for _, hashes := range [][][]byte{nil, {}, {{}}, {{0xaa, 0xbb}}, {{0xaa}, {0xbb, 0xcc}}} { + queries.calls = 0 + got, err := store.ResolveEndpointHashes(ctx, "YVR", hashes) + if err != nil || len(got) != 0 || queries.calls != 0 { + t.Fatal("invalid endpoint width reached the database") + } + } + queries.calls = 0 + resolved, err := store.ResolveEndpointHashes(ctx, "YVR", [][]byte{{0xee}, {0xdd}, {0xff}, {0xbb}, {0xbb}}) + if err != nil || queries.calls != 1 { + t.Fatalf("batched endpoint query: %v, calls=%d", err, queries.calls) + } + hops := api.BuildResolvedPath([][]byte{{0xee}, {0xdd}, {0xff}, {0xbb}}, resolved) + check(&hops[0], "high", 3) + check(&hops[1], "high", 9) + if hops[1].Nodes[0].Name != nil { + t.Error("unnamed node acquired a name") + } + check(&hops[2], "none") + check(&hops[3], "high", 6) + missing, err := store.ResolveEndpointHashes(ctx, "ZZZ", [][]byte{{0xaa}}) + if err != nil || len(missing) != 0 { + t.Fatal("endpoint escaped its observation region") + } + + // A large nonmatching population must not turn a short-hash lookup into a full scan. + exec(`INSERT INTO nodes(id,public_key,node_type) +SELECT ('00000000-0000-0000-0000-'||lpad(i::text,12,'0'))::uuid, + decode(lpad(to_hex(i),8,'0')||repeat('00',28),'hex'),1 FROM generate_series(10000,29999) i; +INSERT INTO node_short_ids(node_id,iata,prefix_4) +SELECT id,'YVR',substring(public_key from 1 for 4) FROM nodes WHERE substring(public_key from 1 for 1)='\x00'; +ANALYZE nodes; +ANALYZE node_short_ids;`) + if _, err := store.ResolveEndpointHashes(ctx, "YVR", [][]byte{{0xaa}}); err != nil { + t.Fatal(err) + } + exec("PREPARE endpoint_role_plan AS " + queries.query) + defer tx.Exec(context.Background(), "DEALLOCATE endpoint_role_plan") + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + exec("SET LOCAL plan_cache_mode=" + mode) + var raw []byte + if err := tx.QueryRow(ctx, `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE endpoint_role_plan('YVR',ARRAY['\xaa'::bytea])`).Scan(&raw); err != nil { + t.Fatal(err) + } + var report []struct { + Plan map[string]any + Milliseconds float64 `json:"Execution Time"` + } + if err := json.Unmarshal(raw, &report); err != nil { + t.Fatal(err) + } + var examined, nodeRows float64 + indexed := false + var walk func(map[string]any) + walk = func(plan map[string]any) { + if cond, ok := plan["Index Cond"].(string); ok && strings.Contains(cond, "prefix_1") && strings.Contains(cond, "iata") { + indexed = true + } + if plan["Relation Name"] == "node_short_ids" { + rows, _ := plan["Actual Rows"].(float64) + removed, _ := plan["Rows Removed by Filter"].(float64) + loops, _ := plan["Actual Loops"].(float64) + examined += (rows + removed) * loops + } + if plan["Relation Name"] == "nodes" { + rows, _ := plan["Actual Rows"].(float64) + removed, _ := plan["Rows Removed by Filter"].(float64) + loops, _ := plan["Actual Loops"].(float64) + nodeRows += (rows + removed) * loops + } + if children, ok := plan["Plans"].([]any); ok { + for _, child := range children { + walk(child.(map[string]any)) + } + } + } + walk(report[0].Plan) + if !indexed || examined > 10 || nodeRows > 10 { + t.Errorf("%s endpoint lookup scanned %.0f short IDs and %.0f nodes, indexed=%v", mode, examined, nodeRows, indexed) + } + t.Logf("%s: %.0f short-ID rows and %.0f node rows examined in %.3f ms", mode, examined, nodeRows, report[0].Milliseconds) + } +} diff --git a/db/endpoints.go b/db/endpoints.go new file mode 100644 index 00000000..457ff421 --- /dev/null +++ b/db/endpoints.go @@ -0,0 +1,151 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/hex" + "log/slog" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/meshcore-go/meshcore-go" +) + +// packetEndpoints is what a packet's payload says about its sender and recipient. +type packetEndpoints struct { + advert bool // source is the advert's own key, not a hash + advertKey []byte // nil for legacy adverts stored without one + source, destination []byte // one-byte hashes +} + +func parsePacketEndpoints(payloadType int16, raw, originPubkey []byte) packetEndpoints { + var ep packetEndpoints + switch payloadType { + case int16(meshcore.PayloadTypeAdvert): + ep.advert, ep.advertKey = true, originPubkey + case int16(meshcore.PayloadTypeAnonReq): + if p, err := meshcore.AnonReqFromBytes(raw); err == nil { + ep.destination = []byte{p.Destination} + } + case int16(meshcore.PayloadTypeReq): + if p, err := meshcore.RequestFromBytes(raw); err == nil { + ep.source, ep.destination = []byte{p.Source}, []byte{p.Destination} + } + case int16(meshcore.PayloadTypeResponse): + if p, err := meshcore.ResponseFromBytes(raw); err == nil { + ep.source, ep.destination = []byte{p.Source}, []byte{p.Destination} + } + case int16(meshcore.PayloadTypeTxtMsg): + if p, err := meshcore.TextMessageFromBytes(raw); err == nil { + ep.source, ep.destination = []byte{p.Source}, []byte{p.Destination} + } + case int16(meshcore.PayloadTypePath): + if p, err := meshcore.PathFromBytes(raw); err == nil { + ep.source, ep.destination = []byte{p.Source}, []byte{p.Destination} + } + } + return ep +} + +type endpointLookup struct { + iata string + ep packetEndpoints +} + +// resolveEndpoints resolves a whole page of endpoints in at most two queries. +// Results line up with lookups; nil means no such endpoint or a failed lookup. +func (s *Store) resolveEndpoints(ctx context.Context, lookups []endpointLookup) (sources, destinations []*api.ResolvedHop) { + iatas, hashes, keys := map[string]bool{}, map[byte]bool{}, map[string][]byte{} + for _, l := range lookups { + for _, h := range [][]byte{l.ep.source, l.ep.destination} { + if len(h) == 1 { + iatas[l.iata], hashes[h[0]] = true, true + } + } + if len(l.ep.advertKey) > 0 { + keys[string(l.ep.advertKey)] = l.ep.advertKey + } + } + + candidates := map[string]map[string][]api.ResolvedPathEntry{} // iata -> hash hex + hashesOK := len(hashes) > 0 + if hashesOK { + params := sqlc.ResolveEndpointHashPairsParams{} + for iata := range iatas { + params.Iatas = append(params.Iatas, iata) + } + for h := range hashes { + params.Hashes = append(params.Hashes, []byte{h}) + } + rows, err := s.q.ResolveEndpointHashPairs(ctx, params) + if err != nil { + slog.Error("store: endpoint resolution failed", "component", "db", "error", err) + hashesOK = false + } + for _, r := range rows { + if candidates[r.Iata] == nil { + candidates[r.Iata] = map[string][]api.ResolvedPathEntry{} + } + key := hex.EncodeToString(r.Hash) + candidates[r.Iata][key] = append(candidates[r.Iata][key], api.ResolvedPathEntry{ + NodeID: r.NodeID, Name: r.Name, Latitude: r.Latitude, + Longitude: r.Longitude, PublicKey: r.PublicKey, + }) + } + } + + adverts := map[string]*api.ResolvedNode{} + advertsOK := true + if len(keys) > 0 { + pubkeys := make([][]byte, 0, len(keys)) + for _, k := range keys { + pubkeys = append(pubkeys, k) + } + rows, err := s.q.GetNodesByPubkeys(ctx, pubkeys) + if err != nil { + slog.Error("store: advert source lookup failed", "component", "db", "error", err) + advertsOK = false + } + for _, r := range rows { + adverts[string(r.PublicKey)] = &api.ResolvedNode{ + ID: r.ID, Name: r.Name, PublicKey: hex.EncodeToString(r.PublicKey), + Latitude: r.Latitude, Longitude: r.Longitude, + } + } + } + + hop := func(iata string, h []byte) *api.ResolvedHop { + if len(h) != 1 || !hashesOK { + return nil + } + r := api.BuildResolvedPath([][]byte{h}, candidates[iata])[0] + return &r + } + sources = make([]*api.ResolvedHop, len(lookups)) + destinations = make([]*api.ResolvedHop, len(lookups)) + for i, l := range lookups { + if l.ep.advert { + if advertsOK { + r := api.ResolveExactNode(adverts[string(l.ep.advertKey)]) + sources[i] = &r + } + } else { + sources[i] = hop(l.iata, l.ep.source) + } + destinations[i] = hop(l.iata, l.ep.destination) + } + return sources, destinations +} + +// fillLatestEndpoints resolves the latest observer's endpoints for a page of packets. +// lookups line up with items; rows without a latest observer are skipped. +func (s *Store) fillLatestEndpoints(ctx context.Context, items []api.PacketSummary, lookups []endpointLookup) { + sources, destinations := s.resolveEndpoints(ctx, lookups) + for i := range items { + if lo := items[i].LatestObserver; lo != nil { + lo.ResolvedSource, lo.ResolvedDestination = sources[i], destinations[i] + } + } +} diff --git a/db/endpoints_test.go b/db/endpoints_test.go new file mode 100644 index 00000000..a896d2e4 --- /dev/null +++ b/db/endpoints_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "sort" + "testing" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" + "go.uber.org/mock/gomock" +) + +func TestParsePacketEndpoints(t *testing.T) { + msg := append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...) + key := []byte{1, 2, 3} + for _, tc := range []struct { + name string + kind uint8 + raw, key []byte + want packetEndpoints + }{ + {"direct message", meshcore.PayloadTypeTxtMsg, msg, nil, packetEndpoints{source: []byte{0xaa}, destination: []byte{0xbb}}}, + {"request", meshcore.PayloadTypeReq, msg, nil, packetEndpoints{source: []byte{0xaa}, destination: []byte{0xbb}}}, + {"advert", meshcore.PayloadTypeAdvert, nil, key, packetEndpoints{advert: true, advertKey: key}}, + {"channel message", meshcore.PayloadTypeGrpTxt, msg, nil, packetEndpoints{}}, + {"truncated", meshcore.PayloadTypeTxtMsg, []byte{0xbb}, nil, packetEndpoints{}}, + } { + t.Run(tc.name, func(t *testing.T) { + got := parsePacketEndpoints(int16(tc.kind), tc.raw, tc.key) + if got.advert != tc.want.advert || string(got.advertKey) != string(tc.want.advertKey) || + string(got.source) != string(tc.want.source) || string(got.destination) != string(tc.want.destination) { + t.Fatalf("got %+v, want %+v", got, tc.want) + } + }) + } +} + +func hopNames(h *api.ResolvedHop) []string { + if h == nil { + return nil + } + names := []string{h.Confidence} + for _, n := range h.Nodes { + names = append(names, *n.Name) + } + sort.Strings(names[1:]) + return names +} + +func TestResolveEndpointsBatchesAPage(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + name := func(s string) *string { return &s } + alice, bob, carol := uuid.New(), uuid.New(), uuid.New() + advertKey := []byte{0xaa, 0x11} + + mock.EXPECT().ResolveEndpointHashPairs(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, p sqlc.ResolveEndpointHashPairsParams) ([]sqlc.ResolveEndpointHashPairsRow, error) { + if len(p.Iatas) != 2 || len(p.Hashes) != 2 { + t.Errorf("want distinct IATAs and hashes, got %v %v", p.Iatas, p.Hashes) + } + return []sqlc.ResolveEndpointHashPairsRow{ + {Iata: "YYZ", Hash: []byte{0xaa}, NodeID: alice, Name: name("Alice")}, + {Iata: "YYZ", Hash: []byte{0xaa}, NodeID: bob, Name: name("Bob")}, + {Iata: "YYZ", Hash: []byte{0xbb}, NodeID: carol, Name: name("Carol")}, + {Iata: "YVR", Hash: []byte{0xaa}, NodeID: alice, Name: name("Alice")}, + {Iata: "YVR", Hash: []byte{0xbb}, NodeID: carol, Name: name("Carol")}, // cross-product extra + }, nil + }) + mock.EXPECT().GetNodesByPubkeys(gomock.Any(), [][]byte{advertKey}).Return( + []sqlc.GetNodesByPubkeysRow{{ID: alice, PublicKey: advertKey, Name: name("Alice")}}, nil) + + sources, destinations := store.resolveEndpoints(context.Background(), []endpointLookup{ + {iata: "YYZ", ep: packetEndpoints{source: []byte{0xaa}, destination: []byte{0xbb}}}, + {iata: "YVR", ep: packetEndpoints{source: []byte{0xaa}}}, + {iata: "YYZ", ep: packetEndpoints{advert: true, advertKey: advertKey}}, + {iata: "YYZ", ep: packetEndpoints{advert: true, advertKey: advertKey}}, + {iata: "YYZ", ep: packetEndpoints{advert: true}}, + {}, + }) + + for i, tc := range []struct{ source, destination []string }{ + {[]string{"ambiguous", "Alice", "Bob"}, []string{"high", "Carol"}}, + {[]string{"high", "Alice"}, nil}, + {[]string{"high", "Alice"}, nil}, + {[]string{"high", "Alice"}, nil}, + {[]string{"none"}, nil}, + {nil, nil}, + } { + if got := hopNames(sources[i]); !equalStrings(got, tc.source) { + t.Errorf("row %d source = %v, want %v", i, got, tc.source) + } + if got := hopNames(destinations[i]); !equalStrings(got, tc.destination) { + t.Errorf("row %d destination = %v, want %v", i, got, tc.destination) + } + } +} + +func TestResolveEndpointsFailedLookupLeavesHopsEmpty(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + mock.EXPECT().ResolveEndpointHashPairs(gomock.Any(), gomock.Any()).Return(nil, errors.New("boom")) + + sources, destinations := store.resolveEndpoints(context.Background(), []endpointLookup{ + {iata: "YYZ", ep: packetEndpoints{source: []byte{0xaa}, destination: []byte{0xbb}}}, + }) + if sources[0] != nil || destinations[0] != nil { + t.Fatal("a failed lookup must not report endpoints as unresolved") + } +} + +func TestResolveEndpointsSkipsQueriesWhenNothingToResolve(t *testing.T) { + ctrl := gomock.NewController(t) + store := &Store{q: mockdb.NewMockQuerier(ctrl)} // no EXPECT: any query fails the test + sources, _ := store.resolveEndpoints(context.Background(), []endpointLookup{{}, {iata: "YYZ"}}) + if len(sources) != 2 || sources[0] != nil || sources[1] != nil { + t.Fatal("rows without endpoints must stay empty") + } +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/db/meshmapper.go b/db/meshmapper.go new file mode 100644 index 00000000..fa2120de --- /dev/null +++ b/db/meshmapper.go @@ -0,0 +1,256 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetScopeCatalogue(ctx context.Context, iata, url string) (*meshmapper.Cache, error) { + row, err := s.q.GetScopeCatalogue(ctx, sqlc.GetScopeCatalogueParams{Iata: iata, Url: url}) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + cache := &meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError} + if row.Etag != nil { + cache.ETag = *row.Etag + } + return cache, nil +} + +func (s *Store) ListScopeCatalogues(ctx context.Context) ([]meshmapper.Catalogue, error) { + rows, err := s.q.ListScopeCatalogues(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.Catalogue, 0, len(rows)) + for _, row := range rows { + c := meshmapper.Catalogue{IATA: row.Iata, URL: row.Url, Cache: meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}} + if row.Etag != nil { + c.ETag = *row.Etag + } + out = append(out, c) + } + return out, nil +} + +func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, entries []scopestore.Entry) error { + params := sqlc.SaveScopeCatalogueParams{ + Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError, + CheckedAt: pgtype.Timestamptz{Time: cache.CheckedAt, Valid: !cache.CheckedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: cache.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: cache.NextAttempt, Valid: true}, + Names: []string{}, Keys: [][]byte{}, Fingerprints: [][]byte{}, + } + if !cache.CheckedAt.IsZero() { + params.Etag = &cache.ETag + } + for _, entry := range entries { + params.Names = append(params.Names, entry.Name) + params.Keys = append(params.Keys, entry.TransportKey) + params.Fingerprints = append(params.Fingerprints, entry.KeyFingerprint) + } + return s.q.SaveScopeCatalogue(ctx, params) +} + +func (s *Store) ListKnownIATAs(ctx context.Context) ([]string, error) { + rows, err := s.q.ListIATAs(ctx) + if err != nil { + return nil, err + } + iatas := make([]string, 0, len(rows)) + for _, row := range rows { + iatas = append(iatas, row.Iata) + } + return iatas, nil +} + +// ListHeardIATAs returns the IATAs observers last reported from. +func (s *Store) ListHeardIATAs(ctx context.Context) ([]string, error) { + return s.q.ListHeardIATAs(ctx) +} + +func (s *Store) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { + if keep == nil { + keep = []string{} + } + return s.q.PruneZoneBoundaries(ctx, keep) +} + +func (s *Store) ListZoneBoundaries(ctx context.Context) ([]meshmapper.Boundary, error) { + rows, err := s.q.ListZoneBoundaries(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.Boundary, 0, len(rows)) + for _, row := range rows { + b := meshmapper.Boundary{IATA: row.Iata, URL: row.Url, Feature: row.Feature, CheckedAt: row.CheckedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError} + if row.Etag != nil { + b.ETag = *row.Etag + } + out = append(out, b) + } + return out, nil +} + +func (s *Store) SaveZoneBoundary(ctx context.Context, b meshmapper.Boundary) error { + params := sqlc.SaveZoneBoundaryParams{ + Iata: b.IATA, Url: b.URL, Feature: b.Feature, LastError: b.LastError, + CheckedAt: pgtype.Timestamptz{Time: b.CheckedAt, Valid: !b.CheckedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: b.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: b.NextAttempt, Valid: true}, + } + if !b.CheckedAt.IsZero() { + params.Etag = &b.ETag + } + return s.q.SaveZoneBoundary(ctx, params) +} + +func (s *Store) ListZoneLists(ctx context.Context) ([]meshmapper.ZoneList, error) { + rows, err := s.q.ListZoneLists(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.ZoneList, 0, len(rows)) + for _, row := range rows { + l := meshmapper.ZoneList{Country: row.Country, Payload: row.Payload, FetchedAt: row.FetchedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError} + if row.Etag != nil { + l.ETag = *row.Etag + } + out = append(out, l) + } + return out, nil +} + +func (s *Store) SaveZoneList(ctx context.Context, l meshmapper.ZoneList) error { + params := sqlc.SaveZoneListParams{ + Country: l.Country, Payload: l.Payload, LastError: l.LastError, + FetchedAt: pgtype.Timestamptz{Time: l.FetchedAt, Valid: !l.FetchedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: l.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: l.NextAttempt, Valid: true}, + } + if !l.FetchedAt.IsZero() { + params.Etag = &l.ETag + } + return s.q.SaveZoneList(ctx, params) +} + +func (s *Store) ListRegionState(ctx context.Context) ([]meshmapper.RegionState, error) { + rows, err := s.q.ListRegionState(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.RegionState, 0, len(rows)) + for _, row := range rows { + out = append(out, meshmapper.RegionState{Slug: row.Slug, Name: row.Name, DisplayOrder: int(row.DisplayOrder), + Imported: row.Imported, IATAs: row.Iatas, CenterLat: row.CenterLat, CenterLng: row.CenterLng}) + } + return out, nil +} + +// SaveImportedRegion reports false when a hand-written region owns the slug. +func (s *Store) SaveImportedRegion(ctx context.Context, r meshmapper.RegionState) (bool, error) { + order := int32(r.DisplayOrder) + id, err := s.q.UpsertImportedRegion(ctx, sqlc.UpsertImportedRegionParams{Slug: r.Slug, Name: r.Name, DisplayOrder: &order, + CenterLat: r.CenterLat, CenterLng: r.CenterLng}) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + // region_iatas references iata_codes, so members Beacon hasn't heard yet are created first, as Seed does. + if err := s.q.AddIATAs(ctx, r.IATAs); err != nil { + return false, err + } + return true, s.SetRegionIATAs(ctx, id, r.IATAs) +} + +func (s *Store) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { + if keep == nil { + keep = []string{} + } + return s.q.PruneImportedRegions(ctx, keep) +} + +func (s *Store) ListChannelCatalogues(ctx context.Context) ([]meshmapper.Catalogue, error) { + rows, err := s.q.ListChannelCatalogues(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.Catalogue, 0, len(rows)) + for _, row := range rows { + c := meshmapper.Catalogue{IATA: row.Iata, URL: row.Url, Cache: meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}} + if row.Etag != nil { + c.ETag = *row.Etag + } + out = append(out, c) + } + return out, nil +} + +// SaveChannelCatalogue records a fetch; a new payload also replaces the IATA's +// channel members, which are written first so a failure never outlives its snapshot. +func (s *Store) SaveChannelCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, fingerprints [][]byte) error { + if cache.Payload != nil { + if err := s.SetChannelMembers(ctx, iata, fingerprints); err != nil { + return err + } + } + params := sqlc.SaveChannelCatalogueParams{ + Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError, + CheckedAt: pgtype.Timestamptz{Time: cache.CheckedAt, Valid: !cache.CheckedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: cache.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: cache.NextAttempt, Valid: true}, + } + if !cache.CheckedAt.IsZero() { + params.Etag = &cache.ETag + } + return s.q.SaveChannelCatalogue(ctx, params) +} + +// SetChannelMembers makes fingerprints the IATA's exact MeshMapper channel list. +func (s *Store) SetChannelMembers(ctx context.Context, iata string, fingerprints [][]byte) error { + if fingerprints == nil { + fingerprints = [][]byte{} + } + if err := s.q.DeleteChannelMembersNotIn(ctx, sqlc.DeleteChannelMembersNotInParams{Iata: iata, Keep: fingerprints}); err != nil { + return err + } + return s.q.AddChannelMembers(ctx, sqlc.AddChannelMembersParams{Iata: iata, Fingerprints: fingerprints}) +} + +func (s *Store) ClearChannelMembers(ctx context.Context) error { + return s.q.DeleteAllChannelMembers(ctx) +} + +func (s *Store) ListIATADetails(ctx context.Context) ([]meshmapper.IATADetails, error) { + rows, err := s.q.ListIATAs(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.IATADetails, 0, len(rows)) + for _, row := range rows { + d := meshmapper.IATADetails{IATA: row.Iata, Lat: row.ApproxLat, Lng: row.ApproxLng} + if row.DisplayName != nil { + d.Name = *row.DisplayName + } + out = append(out, d) + } + return out, nil +} diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go new file mode 100644 index 00000000..5b9af836 --- /dev/null +++ b/db/meshmapper_integration_test.go @@ -0,0 +1,453 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "slices" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestMeshMapperCataloguePostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"iata_codes", "transport_scopes", "meshmapper_scope_catalogues"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + manual := scopestore.FromName("manual") + manual.TransportKey[0] ^= 1 + if err := store.UpsertTransportScope(ctx, manual.Name, "Operator label", manual.TransportKey, manual.KeyFingerprint); err != nil { + t.Fatal(err) + } + if err := store.UpsertIATA(ctx, "YOW"); err != nil { + t.Fatal(err) + } + url := "https://yow.meshmapper.net/get_scopes.php" + if row, err := store.GetScopeCatalogue(ctx, "YOW", url); err != nil || row != nil { + t.Fatal(row, err) + } + now := time.Now().UTC().Truncate(time.Second) + payload := []byte(`{"generated_at":"2026-09-27T16:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":3,"scopes":[{"name":"yow","repeaters":0,"default":0,"monitored":true,"wardriving":false}]}`) + update := meshmapper.Cache{Payload: payload, ETag: `"v1"`, CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(time.Hour)} + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("manual"), scopestore.FromName("yow")}); err != nil { + t.Fatal(err) + } + manualRows, err := store.GetTransportScopes(ctx) + if err != nil || len(manualRows) != 1 || manualRows[0].TransportKey[0] != manual.TransportKey[0] { + t.Fatal("manual key changed", manualRows, err) + } + var label string + var imported bool + if err := tx.QueryRow(ctx, "SELECT display_name,imported_only FROM transport_scopes WHERE name='#manual'").Scan(&label, &imported); err != nil || label != "Operator label" || imported { + t.Fatal(label, imported, err) + } + row, err := store.GetScopeCatalogue(ctx, "YOW", url) + if err != nil || row.ETag != `"v1"` || !row.CheckedAt.Equal(now) { + t.Fatal(row, err) + } + scopes := scopestore.New() + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(store), scopes, manualRows) + if err != nil || len(scopes.Entries()) != 2 { + t.Fatal("restart did not restore imported membership", err) + } + // A conditional success preserves the document, advances checked time and clears a failure. + checked := now.Add(time.Hour) + if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{ETag: `"v1"`, CheckedAt: checked, AttemptedAt: checked, NextAttempt: checked.Add(time.Hour)}, nil); err != nil { + t.Fatal(err) + } + if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{AttemptedAt: checked.Add(time.Minute), NextAttempt: checked.Add(2 * time.Hour), LastError: "HTTP 429"}, nil); err != nil { + t.Fatal(err) + } + row, err = store.GetScopeCatalogue(ctx, "YOW", url) + if err != nil || !row.CheckedAt.Equal(checked) || row.ETag != `"v1"` || row.LastError != "HTTP 429" || len(row.Payload) == 0 { + t.Fatal("failed refresh lost durable snapshot", row, err) + } + // Empty membership deactivates matching but retains the old scope identity for historical rows. + empty := []byte(`{"generated_at":"2026-09-27T17:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":0,"scopes":[]}`) + update.Payload = empty + update.ETag = "" + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, nil); err != nil { + t.Fatal(err) + } + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(store), scopes, manualRows) + if err != nil || len(scopes.Entries()) != 1 { + t.Fatal("removed imported membership remained active", err) + } + if _, err := store.GetTransportScopeByName(ctx, "#yow"); err != nil { + t.Fatal("deleted historical identity", err) + } + row, _ = store.GetScopeCatalogue(ctx, "YOW", url) + if row.ETag != "" { + t.Fatal("200 without ETag retained obsolete validator") + } + // Reconfiguring a known imported name as manual promotes it without losing its identity. + yow := scopestore.FromName("yow") + if err := store.UpsertTransportScope(ctx, yow.Name, "Local", yow.TransportKey, yow.KeyFingerprint); err != nil { + t.Fatal(err) + } + manualRows, err = store.GetTransportScopes(ctx) + if err != nil || len(manualRows) != 2 { + t.Fatal("manual promotion failed", err) + } + // A failed cache write also rolls back the identity insertion in the same statement. + if _, err := tx.Exec(ctx, "ALTER TABLE meshmapper_scope_catalogues ADD CHECK(last_error <> 'force-failure'); SAVEPOINT rejected"); err != nil { + t.Fatal(err) + } + update.LastError = "force-failure" + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("uncommitted")}); err == nil { + t.Fatal("expected constraint failure") + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT rejected"); err != nil { + t.Fatal(err) + } + var count int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM transport_scopes WHERE name='#uncommitted'").Scan(&count); err != nil || count != 0 { + t.Fatal("partially committed catalogue", count, err) + } +} + +func TestMeshMapperZoneBoundariesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"iata_codes", "meshmapper_zone_boundaries", "meshmapper_zone_lists"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + manual := []byte(`{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,1],[0,0]]]},"properties":{"source":"file"}}`) + imported := []byte(`{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0,0],[2,0],[2,2],[0,2],[0,0]]]},"properties":{"code":"YOW"}}`) + if err := store.UpsertIATABorder(ctx, "YOW", manual); err != nil { + t.Fatal(err) + } + if err := store.UpsertIATA(ctx, "YYZ"); err != nil { + t.Fatal(err) + } + border := func(iata string) string { + t.Helper() + got, err := store.GetIATABorder(ctx, iata) + if err != nil { + t.Fatal(err) + } + return string(got) + } + if !strings.Contains(border("YOW"), `"file"`) || border("YYZ") != "" { + t.Fatal("manual border not served without an import") + } + now := time.Now().UTC().Truncate(time.Second) + good := meshmapper.Boundary{IATA: "YOW", URL: "https://yow.meshmapper.net/", Feature: imported, ETag: `"v1"`, CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)} + if err := store.SaveZoneBoundary(ctx, good); err != nil { + t.Fatal(err) + } + if !strings.Contains(border("YOW"), `"YOW"`) { + t.Fatal("imported boundary did not override the manual border") + } + if err := store.UpsertIATABorder(ctx, "YOW", manual); err != nil { + t.Fatal(err) + } + failed := meshmapper.Boundary{IATA: "YOW", URL: good.URL, AttemptedAt: now.Add(time.Hour), NextAttempt: now.Add(2 * time.Hour), LastError: "HTTP 503"} + if err := store.SaveZoneBoundary(ctx, failed); err != nil { + t.Fatal(err) + } + rows, err := store.ListZoneBoundaries(ctx) + if err != nil || len(rows) != 1 || !strings.Contains(string(rows[0].Feature), `"YOW"`) || rows[0].ETag != `"v1"` || + !rows[0].CheckedAt.Equal(now) || rows[0].LastError != "HTTP 503" || !rows[0].NextAttempt.Equal(failed.NextAttempt) { + t.Fatal("failure or reseed lost the last good boundary", rows, err) + } + if !strings.Contains(border("YOW"), `"YOW"`) { + t.Fatal("reseeding the manual border undid the override") + } + if pruned, err := store.PruneZoneBoundaries(ctx, []string{"YOW"}); err != nil || len(pruned) != 0 { + t.Fatal("configured import pruned", pruned, err) + } + if pruned, err := store.PruneZoneBoundaries(ctx, nil); err != nil || len(pruned) != 1 || pruned[0] != "YOW" { + t.Fatal(pruned, err) + } + if !strings.Contains(border("YOW"), `"file"`) { + t.Fatal("manual border did not return after pruning") + } + list := meshmapper.ZoneList{Country: "CA", Payload: []byte(`{"country":"CA","zones":[]}`), ETag: `"z1"`, + FetchedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)} + if err := store.SaveZoneList(ctx, list); err != nil { + t.Fatal(err) + } + later := now.Add(25 * time.Hour) + if err := store.SaveZoneList(ctx, meshmapper.ZoneList{Country: "CA", AttemptedAt: later, NextAttempt: later.Add(48 * time.Hour), LastError: "HTTP 429"}); err != nil { + t.Fatal(err) + } + lists, err := store.ListZoneLists(ctx) + if err != nil || len(lists) != 1 || len(lists[0].Payload) == 0 || lists[0].ETag != `"z1"` || !lists[0].FetchedAt.Equal(now) || + lists[0].LastError != "HTTP 429" || !lists[0].NextAttempt.Equal(later.Add(48*time.Hour)) { + t.Fatal("failed fetch lost the last good zone list", lists, err) + } +} + +func TestMeshMapperRegionsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + // Real tables inside the rolled-back tx: TEMP copies drop region_iatas' foreign key to iata_codes. + store := &Store{q: sqlc.New(tx)} + state := func() map[string]meshmapper.RegionState { + t.Helper() + rows, err := store.ListRegionState(ctx) + if err != nil { + t.Fatal(err) + } + out := map[string]meshmapper.RegionState{} + for _, r := range rows { + out[r.Slug] = r + } + return out + } + for _, iata := range []string{"YOW", "YUL"} { // Seed creates config region IATAs first + if err := store.UpsertIATA(ctx, iata); err != nil { + t.Fatal(err) + } + } + hand, err := store.UpsertRegion(ctx, "east", "East", "", 3, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := store.SetRegionIATAs(ctx, hand, []string{"YOW", "YUL"}); err != nil { + t.Fatal(err) + } + if err := store.SetRegionIATAs(ctx, hand, []string{"YOW"}); err != nil { + t.Fatal(err) + } + if got := state()["east"]; got.Imported || got.DisplayOrder != 3 || strings.Join(got.IATAs, ",") != "YOW" { + t.Fatal("configured members not replaced", got) + } + lat, lng := 45.5, -75.0 + group := meshmapper.RegionState{Slug: "onqc", Name: "Corridor", DisplayOrder: 4, IATAs: []string{"YOW", "YQB"}, CenterLat: &lat, CenterLng: &lng} + if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved { + t.Fatal("members Beacon hasn't heard must be created first", saved, err) + } + if got := state()["onqc"]; got.CenterLat == nil || *got.CenterLat != lat || *got.CenterLng != lng { + t.Fatal("center not saved", got) + } + details, err := store.ListIATADetails(ctx) + if err != nil || !slices.ContainsFunc(details, func(d meshmapper.IATADetails) bool { return d.IATA == "YQB" }) { + t.Fatal("member IATA not created", err) + } + group.IATAs = []string{"YQB", "YUL"} + if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved { + t.Fatal(saved, err) + } + if got := state()["onqc"]; !got.Imported || strings.Join(got.IATAs, ",") != "YQB,YUL" { + t.Fatal("imported region not updated", got) + } + if saved, err := store.SaveImportedRegion(ctx, meshmapper.RegionState{Slug: "east", Name: "Group", IATAs: []string{"YYZ"}}); err != nil || saved { + t.Fatal("imported group replaced a configured region", saved, err) + } + if got := state()["east"]; got.Name != "East" || strings.Join(got.IATAs, ",") != "YOW" { + t.Fatal("configured region changed by a clash", got) + } + if _, err := store.UpsertRegion(ctx, "onqc", "Mine", "", 1, nil, nil, nil); err != nil { + t.Fatal(err) + } + if state()["onqc"].Imported { + t.Fatal("config did not take over an imported slug") + } + if _, err := store.SaveImportedRegion(ctx, meshmapper.RegionState{Slug: "golm", Name: "Lakes", IATAs: []string{"YYZ"}}); err != nil { + t.Fatal(err) + } + if err := store.UpsertIATADetails(ctx, "YQB", "Quebec City", &lat, &lng); err != nil { + t.Fatal(err) + } + if details, _ := store.ListIATADetails(ctx); !slices.ContainsFunc(details, func(d meshmapper.IATADetails) bool { + return d.IATA == "YQB" && d.Name == "Quebec City" && d.Lat != nil && *d.Lat == lat + }) { + t.Fatal("IATA details not saved") + } + if pruned, err := store.PruneImportedRegions(ctx, nil); err != nil || strings.Join(pruned, ",") != "golm" { + t.Fatal("prune touched configured regions", pruned, err) + } +} + +func TestChannelRegionScopingPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"regions", "region_iatas", "channels", "channel_iatas", "channel_config_scopes", + "meshmapper_channel_members", "meshmapper_channel_catalogues"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + for slug, iata := range map[string]string{"east": "YOW", "west": "YYZ"} { + id, err := store.UpsertRegion(ctx, slug, slug, "", 0, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := store.SetRegionIATAs(ctx, id, []string{iata}); err != nil { + t.Fatal(err) + } + } + fp := func(b byte) []byte { return []byte{b, b, b, b, b, b, b, b} } + for name, b := range map[string]byte{"#mm-yow": 1, "#config-east": 2, "#global": 3, "#toronto": 4} { + if _, err := store.UpsertChannel(ctx, []byte{b}, fp(b), name, name[1:]); err != nil { + t.Fatal(err) + } + } + if err := store.UpsertChannelIATA(ctx, []byte{4}, "YOW", time.Now()); err != nil { + t.Fatal(err) + } + if err := store.SetChannelConfigScopes(ctx, [][]byte{fp(2), fp(3)}, []string{"east", ""}); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + for iata, b := range map[string]byte{"YOW": 1, "YYZ": 4} { + update := meshmapper.Cache{Payload: []byte(`{}`), CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)} + if err := store.SaveChannelCatalogue(ctx, iata, "https://x.meshmapper.net/get_channels.php", update, [][]byte{fp(b)}); err != nil { + t.Fatal(err) + } + } + names := func(iatas ...string) string { + t.Helper() + page, err := store.ListChannels(ctx, 50, nil, iatas, nil, 0, nil) + if err != nil { + t.Fatal(err) + } + var out []string + for _, c := range page.Items { + out = append(out, *c.Name) + } + slices.Sort(out) + return strings.Join(out, ",") + } + if got := names("YOW"); got != "#config-east,#global,#mm-yow" { + t.Fatal("YOW channels:", got) + } + if got := names("YYZ"); got != "#global,#toronto" { + t.Fatal("YYZ channels:", got) + } + if got := names(); got != "#config-east,#global,#mm-yow,#toronto" { + t.Fatal("unfiltered channels:", got) + } + // A 304 keeps membership; a new payload replaces it. + if err := store.SaveChannelCatalogue(ctx, "YOW", "https://x.meshmapper.net/get_channels.php", meshmapper.Cache{AttemptedAt: now, NextAttempt: now}, nil); err != nil { + t.Fatal(err) + } + if got := names("YOW"); got != "#config-east,#global,#mm-yow" { + t.Fatal("304 dropped membership:", got) + } + if err := store.SaveChannelCatalogue(ctx, "YOW", "https://x.meshmapper.net/get_channels.php", meshmapper.Cache{Payload: []byte(`{}`), AttemptedAt: now, NextAttempt: now}, [][]byte{}); err != nil { + t.Fatal(err) + } + if got := names("YOW"); got != "#config-east,#global" { + t.Fatal("empty list kept membership:", got) + } + cats, err := store.ListChannelCatalogues(ctx) + if err != nil || len(cats) != 2 { + t.Fatal(cats, err) + } + if err := store.ClearChannelMembers(ctx); err != nil || names("YYZ") != "#global" { + t.Fatal("members not cleared", err) + } +} diff --git a/db/migrate.go b/db/migrate.go index 48b023b2..06f2a638 100644 --- a/db/migrate.go +++ b/db/migrate.go @@ -6,18 +6,143 @@ package db import ( "context" "embed" + "errors" "fmt" "io/fs" + "log/slog" + "regexp" "sort" "strings" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" ) //go:embed migrations/*.sql var migrationFiles embed.FS +// execQuerier is the slice of pgxpool.Pool / pgx.Conn migrations need; it lets +// integration tests drive applyMigration over a plain connection. +type execQuerier interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +var ( + lineCommentRe = regexp.MustCompile(`(?m)--[^\n]*`) + concurrentIndexRe = regexp.MustCompile(`(?is)\bCREATE\s+(?:UNIQUE\s+)?INDEX\s+CONCURRENTLY\s+(?:IF\s+NOT\s+EXISTS\s+)?("?[\w.]+"?)`) + // Statements that refuse to run inside a transaction block. + nonTransactionalRe = regexp.MustCompile(`(?i)\b(?:CONCURRENTLY|VACUUM)\b`) +) + +// migrationLockKey serialises RunMigrations across instances sharing a database. +const migrationLockKey int64 = 0x6265_6163_6f6e // "beacon" + +// concurrentIndexName returns the index a CREATE INDEX CONCURRENTLY migration builds. +func concurrentIndexName(sql string) (string, bool) { + m := concurrentIndexRe.FindStringSubmatch(lineCommentRe.ReplaceAllString(sql, "")) + if m == nil { + return "", false + } + return strings.Trim(m[1], `"`), true +} + +// isDuplicateRelation reports SQLSTATE 42P07 (relation already exists). +func isDuplicateRelation(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "42P07" +} + +// applyMigration runs one file outside a transaction. An interrupted CREATE INDEX +// CONCURRENTLY leaves an invalid index that trips 42P07 on retry; drop it and rebuild once. +func applyMigration(ctx context.Context, db execQuerier, sql string) error { + _, err := db.Exec(ctx, sql) + if err == nil || !isDuplicateRelation(err) { + return err + } + name, ok := concurrentIndexName(sql) + if !ok { + return err + } + ident := pgx.Identifier(strings.Split(name, ".")).Sanitize() + var valid bool + if scanErr := db.QueryRow(ctx, + "SELECT indisvalid FROM pg_index WHERE indexrelid = to_regclass($1)", ident, + ).Scan(&valid); scanErr != nil { + if errors.Is(scanErr, pgx.ErrNoRows) { + return err + } + return fmt.Errorf("%w (checking index %s: %v)", err, name, scanErr) + } + if valid { + slog.Info(fmt.Sprintf("index %s already built, recording migration", name), "component", "db") + return nil + } + if _, dropErr := db.Exec(ctx, "DROP INDEX CONCURRENTLY IF EXISTS "+ident); dropErr != nil { + return fmt.Errorf("dropping invalid index %s: %w", name, dropErr) + } + slog.Warn(fmt.Sprintf("dropped invalid index %s, rebuilding", name), "component", "db") + _, err = db.Exec(ctx, sql) + return err +} + +// applyAndRecord applies a migration and its ledger row. Without CONCURRENTLY both go in +// one simple-protocol Exec, an implicit transaction, so a crash can't apply one without the other. +func applyAndRecord(ctx context.Context, db execQuerier, name, sql string) error { + record := "INSERT INTO schema_migrations (filename) VALUES ('" + strings.ReplaceAll(name, "'", "''") + "')" + if !nonTransactionalRe.MatchString(lineCommentRe.ReplaceAllString(sql, "")) { + _, err := db.Exec(ctx, sql+"\n;\n"+record) + return err + } + if err := applyMigration(ctx, db, sql); err != nil { + return err + } + _, err := db.Exec(ctx, record) + return err +} + +const baselineMigration = "001_baseline.sql" + +var errPreBaseline = errors.New("database schema predates Beacon 2.0.0; 2.0.0 needs a fresh database") + +// checkBaseline refuses databases built by the 1.x migration chain, which the +// 2.0.0 baseline replaced; applying it on top would collide with existing tables. +func checkBaseline(ctx context.Context, db execQuerier) error { + var ledger, hasBaseline, hasPackets bool + err := db.QueryRow(ctx, ` + SELECT EXISTS(SELECT 1 FROM schema_migrations), + EXISTS(SELECT 1 FROM schema_migrations WHERE filename = $1), + to_regclass('packets') IS NOT NULL`, baselineMigration, + ).Scan(&ledger, &hasBaseline, &hasPackets) + if err != nil { + return fmt.Errorf("failed to check migration baseline: %w", err) + } + if (ledger && !hasBaseline) || (!ledger && hasPackets) { + return errPreBaseline + } + return nil +} + func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { + conn, err := pool.Acquire(ctx) + if err != nil { + return fmt.Errorf("failed to acquire migration connection: %w", err) + } + defer conn.Release() + if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockKey); err != nil { + return fmt.Errorf("failed to take migration lock: %w", err) + } + defer func() { + // A session lock outlives Release; drop the connection if unlock fails. + if _, err := conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", migrationLockKey); err != nil { + _ = conn.Conn().Close(context.Background()) + } + }() + return runMigrations(ctx, conn) +} + +func runMigrations(ctx context.Context, pool execQuerier) error { _, err := pool.Exec(ctx, ` CREATE TABLE IF NOT EXISTS schema_migrations ( filename TEXT PRIMARY KEY, @@ -28,35 +153,8 @@ func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { return fmt.Errorf("failed to create schema_migrations: %w", err) } - // After creating schema_migrations table, check if we need to bootstrap - var count int - err = pool.QueryRow(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&count) - if err != nil { - return fmt.Errorf("failed to check migrations count: %w", err) - } - if count == 0 { - // Check if db is already initialized by looking for a known table - var exists bool - err = pool.QueryRow(ctx, ` - SELECT EXISTS( - SELECT 1 FROM information_schema.tables - WHERE table_name = 'packets' - ) - `).Scan(&exists) - if err != nil { - return fmt.Errorf("failed to check existing schema: %w", err) - } - if exists { - // Mark 001 as already applied - if _, err := pool.Exec( - ctx, - "INSERT INTO schema_migrations (filename) VALUES ($1)", - "001_initial_schema.sql", - ); err != nil { - return fmt.Errorf("failed to bootstrap migrations: %w", err) - } - fmt.Println("bootstrapped existing schema as 001_initial_schema.sql") - } + if err := checkBaseline(ctx, pool); err != nil { + return err } entries, err := fs.ReadDir(migrationFiles, "migrations") @@ -90,19 +188,11 @@ func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { return fmt.Errorf("failed to read migration %s: %w", entry.Name(), err) } - if _, err := pool.Exec(ctx, string(sql)); err != nil { + if err := applyAndRecord(ctx, pool, entry.Name(), string(sql)); err != nil { return fmt.Errorf("failed to apply migration %s: %w", entry.Name(), err) } - if _, err := pool.Exec( - ctx, - "INSERT INTO schema_migrations (filename) VALUES ($1)", - entry.Name(), - ); err != nil { - return fmt.Errorf("failed to record migration %s: %w", entry.Name(), err) - } - - fmt.Printf("applied migration: %s\n", entry.Name()) + slog.Info(fmt.Sprintf("applied migration: %s", entry.Name()), "component", "db") } return nil diff --git a/db/migrate_integration_test.go b/db/migrate_integration_test.go new file mode 100644 index 00000000..cc01ebf8 --- /dev/null +++ b/db/migrate_integration_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +// Two instances starting on an empty database must not race the baseline. +func TestRunMigrationsConcurrentStartPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + errs := make(chan error, 3) + for range 3 { + go func() { errs <- RunMigrations(ctx, pool) }() + } + for range 3 { + if err := <-errs; err != nil { + t.Error(err) + } + } + var n int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM schema_migrations").Scan(&n); err != nil || n != 1 { + t.Errorf("ledger rows = %d, %v; want 1", n, err) + } +} + +// A failed ledger insert must roll back the migration it records, or a crash between +// the two leaves a schema the baseline check refuses forever. +func TestApplyAndRecordIsAtomicPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if _, err := pool.Exec(ctx, `CREATE TABLE schema_migrations (filename TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()); +INSERT INTO schema_migrations (filename) VALUES ('900_dup.sql')`); err != nil { + t.Fatal(err) + } + if err := applyAndRecord(ctx, pool, "900_dup.sql", "CREATE TABLE atomic_probe (v INT); -- trailing comment"); err == nil { + t.Fatal("duplicate ledger row should fail") + } + var exists bool + if err := pool.QueryRow(ctx, "SELECT to_regclass('atomic_probe') IS NOT NULL").Scan(&exists); err != nil || exists { + t.Errorf("migration survived its failed ledger insert: %v %v", exists, err) + } + if err := applyAndRecord(ctx, pool, "901_it's.sql", "CREATE TABLE atomic_probe (v INT)"); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE filename = '901_it''s.sql') AND to_regclass('atomic_probe') IS NOT NULL").Scan(&exists); err != nil || !exists { + t.Errorf("migration and ledger row not both recorded: %v %v", exists, err) + } + // CONCURRENTLY can't share a transaction; it still applies and records. + if err := applyAndRecord(ctx, pool, "902_idx.sql", "CREATE INDEX CONCURRENTLY atomic_probe_idx ON atomic_probe (v)"); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE filename = '902_idx.sql') AND to_regclass('atomic_probe_idx') IS NOT NULL").Scan(&exists); err != nil || !exists { + t.Errorf("concurrent migration not recorded: %v %v", exists, err) + } +} + +// CONCURRENTLY is illegal inside a transaction, so this uses a plain connection +// and a real table rather than the usual tx + TEMP table pattern. +func TestApplyMigrationRecoversInvalidIndexPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + + table := fmt.Sprintf("migrate_recovery_test_%d", time.Now().UnixNano()) + index := table + "_idx" + if _, err := conn.Exec(ctx, "CREATE TABLE "+table+" (v INT)"); err != nil { + t.Fatal(err) + } + defer func() { _, _ = conn.Exec(context.Background(), "DROP TABLE IF EXISTS "+table+" CASCADE") }() + + build := "CREATE INDEX CONCURRENTLY " + index + " ON " + table + " (v)" + if err := applyMigration(ctx, conn, build); err != nil { + t.Fatalf("initial build: %v", err) + } + + // A valid, already-built index (process died before recording) is a success. + if err := applyMigration(ctx, conn, build); err != nil { + t.Fatalf("valid existing index must be accepted: %v", err) + } + + // Simulate an interrupted build by flipping the catalog flag. + if _, err := conn.Exec(ctx, "UPDATE pg_index SET indisvalid = false WHERE indexrelid = to_regclass($1)", index); err != nil { + t.Skipf("cannot mark index invalid (needs catalog write privilege): %v", err) + } + if err := applyMigration(ctx, conn, build); err != nil { + t.Fatalf("invalid index must be dropped and rebuilt: %v", err) + } + var valid bool + if err := conn.QueryRow(ctx, "SELECT indisvalid FROM pg_index WHERE indexrelid = to_regclass($1)", index).Scan(&valid); err != nil { + t.Fatal(err) + } + if !valid { + t.Fatal("index still invalid after recovery") + } + + // Recovery is scoped to CONCURRENTLY migrations; a plain duplicate still errors. + plain := "CREATE INDEX " + index + " ON " + table + " (v)" + if err := applyMigration(ctx, conn, plain); !isDuplicateRelation(err) { + t.Fatalf("plain duplicate index must surface 42P07, got %v", err) + } +} diff --git a/db/migrate_test.go b/db/migrate_test.go new file mode 100644 index 00000000..0c6d3e65 --- /dev/null +++ b/db/migrate_test.go @@ -0,0 +1,77 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "errors" + "fmt" + "testing" + + "github.com/jackc/pgx/v5/pgconn" +) + +func TestConcurrentIndexName(t *testing.T) { + cases := []struct { + name string + sql string + want string + ok bool + }{ + {"bare 027 style with comments", `-- RunMigrations executes each file outside an explicit transaction. Keep this +-- as one statement so other subscribers can keep writing during the build. +-- If interrupted, check pg_index.indisvalid and remove this named index before +-- retrying; do not silently accept an invalid index with IF NOT EXISTS. +CREATE INDEX CONCURRENTLY idx_known_routes_iata_last_seen +ON known_routes (iata, last_seen DESC);`, "idx_known_routes_iata_last_seen", true}, + {"if not exists", "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_a ON t (c);", "idx_a", true}, + {"unique", "CREATE UNIQUE INDEX CONCURRENTLY idx_u ON t (c);", "idx_u", true}, + {"quoted", `CREATE INDEX CONCURRENTLY "Idx_Q" ON t (c);`, "Idx_Q", true}, + {"schema qualified", "CREATE INDEX CONCURRENTLY public.idx_s ON t (c);", "public.idx_s", true}, + {"lowercase multiline", "create index\n concurrently\n idx_l\n on t (c);", "idx_l", true}, + {"plain create index", "CREATE INDEX idx_p ON t (c);", "", false}, + {"only in comment", "-- CREATE INDEX CONCURRENTLY idx_c ON t (c);\nSELECT 1;", "", false}, + {"empty", "", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := concurrentIndexName(tc.sql) + if ok != tc.ok || got != tc.want { + t.Fatalf("got (%q, %v), want (%q, %v)", got, ok, tc.want, tc.ok) + } + }) + } +} + +// The baseline must apply in the same transaction as its ledger row. +func TestBaselineIsTransactional(t *testing.T) { + sql, err := migrationFiles.ReadFile("migrations/" + baselineMigration) + if err != nil { + t.Fatal(err) + } + if nonTransactionalRe.MatchString(lineCommentRe.ReplaceAllString(string(sql), "")) { + t.Fatal("baseline contains a statement that can't run in a transaction") + } +} + +func TestIsDuplicateRelation(t *testing.T) { + dup := &pgconn.PgError{Code: "42P07"} + cases := []struct { + name string + err error + want bool + }{ + {"42P07", dup, true}, + {"wrapped 42P07", fmt.Errorf("apply: %w", dup), true}, + {"other sqlstate", &pgconn.PgError{Code: "42501"}, false}, + {"plain error", errors.New("boom"), false}, + {"nil", nil, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isDuplicateRelation(tc.err); got != tc.want { + t.Fatalf("got %v, want %v", got, tc.want) + } + }) + } +} diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql new file mode 100644 index 00000000..4126ad77 --- /dev/null +++ b/db/migrations/001_baseline.sql @@ -0,0 +1,956 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- 2.0.0 schema baseline: pg_dump of a fresh database migrated through the 1.x +-- chain (001_initial_schema..045). Add new numbered migrations after this file. + +CREATE TABLE accounts ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + deactivated_at timestamp with time zone, + CONSTRAINT accounts_name_check CHECK ((((char_length(name) >= 1) AND (char_length(name) <= 128)) AND (name = btrim(name)))) +); + +CREATE TABLE packet_observations ( + id bigint NOT NULL, + packet_hash bytea NOT NULL, + observer_id uuid NOT NULL, + iata character(3) NOT NULL, + heard_at timestamp with time zone NOT NULL, + path_length_byte smallint NOT NULL, + hash_size smallint NOT NULL, + hop_count smallint NOT NULL, + path_bytes bytea, + rssi smallint, + snr real, + propagation_time_ms integer, + radio_freq_mhz real, + spread_factor smallint, + bandwidth_khz real, + coding_rate smallint, + source_broker text, + payload_type smallint, + airtime_ms real +) +WITH (autovacuum_vacuum_scale_factor='0.05', autovacuum_vacuum_cost_limit='1000'); + +CREATE TABLE nodes ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + public_key bytea NOT NULL, + node_type smallint NOT NULL, + name text, + latitude double precision, + longitude double precision, + location_source text, + last_advert_at timestamp with time zone, + supports_multibyte_paths boolean DEFAULT false NOT NULL, + supports_multibyte_traces boolean DEFAULT false NOT NULL, + default_scope_id integer, + min_firmware_version text GENERATED ALWAYS AS ( +CASE + WHEN supports_multibyte_paths THEN '1.14.0+'::text + WHEN supports_multibyte_traces THEN '1.11.0+'::text + ELSE NULL::text +END) STORED, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + radio_freq_mhz real, + radio_sf smallint, + radio_bw_khz real, + metadata jsonb, + device_clock_drift_seconds integer +) +WITH (autovacuum_vacuum_scale_factor='0.05'); + +CREATE TABLE packets ( + packet_hash bytea NOT NULL, + payload_type smallint NOT NULL, + payload_version smallint NOT NULL, + route_type smallint NOT NULL, + transport_codes_present boolean DEFAULT false, + region_code integer, + sub_region_code integer, + scope_id integer, + origin_pubkey bytea, + raw_payload bytea NOT NULL, + raw_header bytea NOT NULL, + parsed_payload jsonb, + decrypted boolean DEFAULT false, + channel_hash bytea, + trace_tag bytea, + first_heard_at timestamp with time zone NOT NULL, + last_heard_at timestamp with time zone NOT NULL, + observation_count bigint DEFAULT 0 NOT NULL +) +WITH (autovacuum_vacuum_scale_factor='0.05', autovacuum_vacuum_cost_limit='1000'); + +CREATE TABLE observers ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + public_key bytea NOT NULL, + display_name text, + observer_type text, + software_version text, + hardware_model text, + firmware_version text, + firmware_build text, + radio_freq_mhz real, + radio_sf smallint, + radio_bw_khz real, + radio_cr smallint, + battery_level real, + uptime_seconds bigint, + status_metadata jsonb, + last_status_at timestamp with time zone, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + observation_count bigint DEFAULT 0, + metadata jsonb, + region_scope text, + last_iata character(3), + last_iata_at timestamp with time zone +); + +CREATE TABLE channel_messages ( + id bigint NOT NULL, + channel_id integer NOT NULL, + packet_hash bytea NOT NULL, + sender_name text, + sender_pubkey bytea, + content text, + sent_at timestamp with time zone NOT NULL +); + +CREATE TABLE channel_config_scopes ( + key_fingerprint bytea NOT NULL, + region_slug text +); + +CREATE TABLE channel_iatas ( + channel_hash bytea NOT NULL, + iata character(3) NOT NULL, + last_heard timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE channel_keys ( + channel_id integer NOT NULL, + key_bytes bytea NOT NULL, + key_fingerprint bytea NOT NULL, + added_at timestamp with time zone DEFAULT now() NOT NULL, + added_by text +); + +CREATE SEQUENCE channel_messages_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE channel_messages_id_seq OWNED BY channel_messages.id; + +CREATE TABLE channels ( + id integer NOT NULL, + channel_hash bytea NOT NULL, + key_fingerprint bytea, + name text, + hashtag text, + is_hashtag boolean DEFAULT false, + is_public boolean DEFAULT false, + key_known boolean DEFAULT false, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + message_count bigint DEFAULT 0 NOT NULL +); + +CREATE SEQUENCE channels_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE channels_id_seq OWNED BY channels.id; + +CREATE TABLE iata_codes ( + iata character(3) NOT NULL, + display_name text, + approx_lat double precision, + approx_lng double precision, + added_at timestamp with time zone DEFAULT now() NOT NULL, + border jsonb +); + +CREATE TABLE known_routes ( + id bigint NOT NULL, + path_key bytea NOT NULL, + node_ids uuid[] NOT NULL, + hash_prefix bytea[] NOT NULL, + iata character(3) NOT NULL, + hop_count integer NOT NULL, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + observation_count bigint DEFAULT 1 NOT NULL, + last_reconfirmed_at timestamp with time zone DEFAULT now() NOT NULL +) +WITH (autovacuum_vacuum_scale_factor='0.02', autovacuum_analyze_scale_factor='0.02', autovacuum_vacuum_cost_limit='1000'); + +ALTER TABLE known_routes ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME known_routes_new_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + +CREATE TABLE meshmapper_channel_catalogues ( + iata text NOT NULL, + url text NOT NULL, + payload jsonb, + etag text, + checked_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + +CREATE TABLE meshmapper_channel_members ( + iata text NOT NULL, + key_fingerprint bytea NOT NULL +); + +CREATE TABLE meshmapper_scope_catalogues ( + iata text NOT NULL, + url text NOT NULL, + payload jsonb, + etag text, + checked_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + +CREATE TABLE meshmapper_zone_boundaries ( + iata text NOT NULL, + url text NOT NULL, + feature jsonb, + etag text, + checked_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + +CREATE TABLE meshmapper_zone_lists ( + country text NOT NULL, + payload jsonb, + etag text, + fetched_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + +CREATE TABLE node_iatas ( + node_id uuid NOT NULL, + iata character(3) NOT NULL, + first_heard timestamp with time zone DEFAULT now() NOT NULL, + last_heard timestamp with time zone DEFAULT now() NOT NULL, + observation_count bigint DEFAULT 0 +) +WITH (autovacuum_vacuum_scale_factor='0.05'); + +CREATE MATERIALIZED VIEW mv_radio_presets AS +SELECT + concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text AS preset, + o.last_iata AS iata, + 'observer'::text AS source_type, + COUNT(*) AS count +FROM observers o +WHERE o.radio_freq_mhz IS NOT NULL + AND o.radio_sf IS NOT NULL + AND o.radio_bw_khz IS NOT NULL + AND o.last_iata IS NOT NULL +GROUP BY 1, 2 + +UNION ALL + +SELECT + concat(n.radio_freq_mhz, ',', n.radio_bw_khz, ',', n.radio_sf) AS preset, + ni.iata, + 'node' AS source_type, + COUNT(*) AS count +FROM nodes n +JOIN node_iatas ni ON ni.node_id = n.id +WHERE n.radio_freq_mhz IS NOT NULL + AND n.radio_sf IS NOT NULL + AND n.radio_bw_khz IS NOT NULL +GROUP BY 1, 2; + +CREATE TABLE node_neighbors ( + node_id uuid NOT NULL, + neighbor_id uuid NOT NULL, + iata character(3) NOT NULL, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + observation_count bigint DEFAULT 1 NOT NULL, + snr real, + region_scope text +); + +CREATE TABLE node_short_ids ( + node_id uuid NOT NULL, + iata character(3) NOT NULL, + prefix_4 bytea NOT NULL, + prefix_1 bytea GENERATED ALWAYS AS (SUBSTRING(prefix_4 FROM 1 FOR 1)) STORED, + prefix_2 bytea GENERATED ALWAYS AS (SUBSTRING(prefix_4 FROM 1 FOR 2)) STORED, + prefix_3 bytea GENERATED ALWAYS AS (SUBSTRING(prefix_4 FROM 1 FOR 3)) STORED +) +WITH (autovacuum_vacuum_scale_factor='0.05'); + +CREATE TABLE observer_brokers ( + observer_id uuid NOT NULL, + broker_name text NOT NULL, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + last_packet_at timestamp with time zone, + auth_ok boolean DEFAULT true +); + +CREATE TABLE observer_locations ( + observer_id uuid NOT NULL, + iata character(3), + latitude double precision, + longitude double precision, + reported_at timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE observer_owners ( + observer_id uuid NOT NULL, + owner_node_id uuid, + owner_pubkey bytea, + contact_name text, + contact_email text, + notes text, + source text, + added_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE observer_scopes ( + observer_id uuid NOT NULL, + scope_id integer NOT NULL, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE observer_telemetry ( + id bigint NOT NULL, + observer_id uuid NOT NULL, + reported_at timestamp with time zone NOT NULL, + battery_voltage_mv integer, + airtime_tx_secs real, + airtime_rx_secs real, + noise_floor_db real, + uptime_seconds bigint, + queue_length integer, + debug_flags integer, + receive_errors integer +); + +CREATE SEQUENCE observer_telemetry_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE observer_telemetry_id_seq OWNED BY observer_telemetry.id; + +CREATE SEQUENCE packet_observations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE packet_observations_id_seq OWNED BY packet_observations.id; + +CREATE TABLE region_iatas ( + region_id integer NOT NULL, + iata character(3) NOT NULL, + added_at timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE regions ( + id integer NOT NULL, + slug text NOT NULL, + name text NOT NULL, + description text, + display_order integer DEFAULT 0, + center_lat double precision, + center_lng double precision, + zoom_level integer DEFAULT 8, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL, + imported boolean DEFAULT false NOT NULL +); + +CREATE SEQUENCE regions_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE regions_id_seq OWNED BY regions.id; + +CREATE TABLE trace_iatas ( + trace_tag bytea NOT NULL, + iata character(3) NOT NULL, + last_heard timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE trace_tags ( + trace_tag bytea PRIMARY KEY, + first_heard_at timestamp with time zone NOT NULL, + last_heard_at timestamp with time zone NOT NULL, + packet_count bigint DEFAULT 0 NOT NULL, + trace_type text, + scope_id integer, + best_payload jsonb, + best_path_len integer DEFAULT -1 NOT NULL, + heard boolean DEFAULT false NOT NULL -- false: times are the packet's provisional ones until a hearing is recorded +); + +-- Canonical IATA set: sorted, unique, non-null, non-empty. +CREATE FUNCTION iata_set_is_canonical(s bpchar[]) RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT cardinality(s) > 0 + AND array_position(s, NULL) IS NULL + AND s = (SELECT array_agg(x ORDER BY x) FROM (SELECT DISTINCT unnest(s) AS x) d) +$$; + +-- Hourly rollups. hour = UTC date_trunc('hour', heard_at). No FKs: history outlives deletions. +CREATE TABLE analytics_hourly_iata_observations ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + observation_count bigint NOT NULL, + PRIMARY KEY (hour, iata) +); + +CREATE TABLE analytics_hourly_payload_breakdown ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + payload_type smallint NOT NULL, + count bigint NOT NULL, + PRIMARY KEY (hour, iata, payload_type) +); + +CREATE TABLE analytics_hourly_signal ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + kind integer NOT NULL, + snr_bin integer NOT NULL, + rssi_bin integer NOT NULL, + receptions bigint NOT NULL, + snr_samples bigint NOT NULL, + snr_sum double precision NOT NULL, + rssi_samples bigint NOT NULL, + rssi_sum double precision NOT NULL, + PRIMARY KEY (hour, iata, kind, snr_bin, rssi_bin) +); + +CREATE TABLE analytics_hourly_paths ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + category integer NOT NULL, + hash_bytes integer NOT NULL, + entries integer NOT NULL, + receptions bigint NOT NULL, + PRIMARY KEY (hour, iata, category, hash_bytes, entries) +); + +CREATE TABLE analytics_hourly_observer_activity ( + hour timestamp with time zone NOT NULL, + observer_id uuid NOT NULL, + payload_type smallint NOT NULL, + observations bigint NOT NULL, + airtime_ms real, + airtime_n bigint NOT NULL, + snr_sum real, + snr_n bigint NOT NULL, + snr_min real, + rssi_sum bigint, + rssi_n bigint NOT NULL, + PRIMARY KEY (hour, observer_id, payload_type) +); + +CREATE TABLE analytics_hourly_observer_identity ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + observer_id uuid NOT NULL, + observation_count bigint NOT NULL, + display_name text, + observer_type text, + PRIMARY KEY (hour, iata, observer_id) +); + +-- ADVERT hearings per origin per IATA: top nodes (observations) and advertiser last_heard. +CREATE TABLE analytics_hourly_advert_hearings ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + origin_pubkey bytea NOT NULL, + observations bigint NOT NULL, + last_heard timestamp with time zone NOT NULL, + name text, + node_type smallint, + PRIMARY KEY (hour, iata, origin_pubkey) +); + +-- Distinct ADVERT packets per origin, grouped by the set of IATAs that heard each packet that hour. +CREATE TABLE analytics_hourly_advert_sets ( + hour timestamp with time zone NOT NULL, + origin_pubkey bytea NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + advert_packets bigint NOT NULL, + flood_packets bigint NOT NULL, + direct_packets bigint NOT NULL, + PRIMARY KEY (hour, origin_pubkey, iatas) +); + +-- Distinct channel messages per sender, grouped by the set of IATAs that heard each message that hour. +CREATE TABLE analytics_hourly_talker_sets ( + hour timestamp with time zone NOT NULL, + sender_name text NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + messages bigint NOT NULL, + last_sent timestamp with time zone NOT NULL, + PRIMARY KEY (hour, sender_name, iatas) +); + +CREATE TABLE analytics_hourly_packet_sets ( + hour timestamp with time zone NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + packets bigint NOT NULL, + PRIMARY KEY (hour, iatas) +); + +CREATE TABLE analytics_hourly_scope_sets ( + hour timestamp with time zone NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + scope_id integer NOT NULL, + packets bigint NOT NULL, + PRIMARY KEY (hour, iatas, scope_id) +); + +-- Observers that heard a scoped packet, per IATA; distinct counts span any IATA filter. +CREATE TABLE analytics_hourly_scope_observers ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + scope_id integer NOT NULL, + observer_id uuid NOT NULL, + PRIMARY KEY (hour, scope_id, iata, observer_id) +); + +-- Nodes whose ADVERTs were heard in a scope, per IATA. +CREATE TABLE analytics_hourly_scope_nodes ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + scope_id integer NOT NULL, + origin_pubkey bytea NOT NULL, + PRIMARY KEY (hour, scope_id, iata, origin_pubkey) +); + +CREATE TABLE analytics_rollup_hours ( + hour timestamp with time zone PRIMARY KEY, + status text NOT NULL CHECK (status IN ('complete', 'partial', 'missing')), + rolled_at timestamp with time zone, + content_hash text +); + +-- Singletons: the rollup writes revision, cleanup writes raw_deleted_before. +CREATE TABLE analytics_state ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + revision bigint NOT NULL DEFAULT 0 +); + +INSERT INTO analytics_state DEFAULT VALUES; + +-- coverage counts retention deletions of rollup hours; cache keys combine it with revision. +CREATE TABLE analytics_raw_state ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + raw_deleted_before timestamp with time zone, + coverage bigint NOT NULL DEFAULT 0 +); + +INSERT INTO analytics_raw_state DEFAULT VALUES; + +CREATE TABLE analytics_dirty_hours ( + hour timestamp with time zone PRIMARY KEY, + enqueued_at timestamp with time zone NOT NULL DEFAULT now() +); + +CREATE TABLE transport_scopes ( + id integer NOT NULL, + name text NOT NULL, + display_name text, + transport_key bytea NOT NULL, + key_fingerprint bytea NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + imported_only boolean DEFAULT false NOT NULL +); + +CREATE SEQUENCE transport_scopes_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE transport_scopes_id_seq OWNED BY transport_scopes.id; + +ALTER TABLE ONLY channel_messages ALTER COLUMN id SET DEFAULT nextval('channel_messages_id_seq'::regclass); + +ALTER TABLE ONLY channels ALTER COLUMN id SET DEFAULT nextval('channels_id_seq'::regclass); + +ALTER TABLE ONLY observer_telemetry ALTER COLUMN id SET DEFAULT nextval('observer_telemetry_id_seq'::regclass); + +ALTER TABLE ONLY packet_observations ALTER COLUMN id SET DEFAULT nextval('packet_observations_id_seq'::regclass); + +ALTER TABLE ONLY regions ALTER COLUMN id SET DEFAULT nextval('regions_id_seq'::regclass); + +ALTER TABLE ONLY transport_scopes ALTER COLUMN id SET DEFAULT nextval('transport_scopes_id_seq'::regclass); + +ALTER TABLE ONLY accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY channel_iatas + ADD CONSTRAINT channel_iatas_pkey PRIMARY KEY (channel_hash, iata); + +ALTER TABLE ONLY channel_keys + ADD CONSTRAINT channel_keys_pkey PRIMARY KEY (channel_id); + +ALTER TABLE ONLY channel_messages + ADD CONSTRAINT channel_messages_packet_hash_key UNIQUE (packet_hash); + +ALTER TABLE ONLY channel_messages + ADD CONSTRAINT channel_messages_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY channels + ADD CONSTRAINT channels_channel_hash_key_fingerprint_key UNIQUE (channel_hash, key_fingerprint); + +ALTER TABLE ONLY channels + ADD CONSTRAINT channels_hashtag_key UNIQUE (hashtag); + +ALTER TABLE ONLY channels + ADD CONSTRAINT channels_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY iata_codes + ADD CONSTRAINT iata_codes_pkey PRIMARY KEY (iata); + +ALTER TABLE ONLY known_routes + ADD CONSTRAINT known_routes_pkey PRIMARY KEY (iata, path_key); + +ALTER TABLE ONLY meshmapper_channel_catalogues + ADD CONSTRAINT meshmapper_channel_catalogues_pkey PRIMARY KEY (iata, url); + +ALTER TABLE ONLY meshmapper_channel_members + ADD CONSTRAINT meshmapper_channel_members_pkey PRIMARY KEY (iata, key_fingerprint); + +ALTER TABLE ONLY meshmapper_scope_catalogues + ADD CONSTRAINT meshmapper_scope_catalogues_pkey PRIMARY KEY (iata, url); + +ALTER TABLE ONLY meshmapper_zone_boundaries + ADD CONSTRAINT meshmapper_zone_boundaries_pkey PRIMARY KEY (iata); + +ALTER TABLE ONLY meshmapper_zone_lists + ADD CONSTRAINT meshmapper_zone_lists_pkey PRIMARY KEY (country); + +ALTER TABLE ONLY node_iatas + ADD CONSTRAINT node_iatas_pkey PRIMARY KEY (node_id, iata); + +ALTER TABLE ONLY node_neighbors + ADD CONSTRAINT node_neighbors_pkey PRIMARY KEY (node_id, neighbor_id, iata); + +ALTER TABLE ONLY node_short_ids + ADD CONSTRAINT node_short_ids_pkey PRIMARY KEY (node_id, iata); + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_public_key_key UNIQUE (public_key); + +ALTER TABLE ONLY observer_brokers + ADD CONSTRAINT observer_brokers_pkey PRIMARY KEY (observer_id, broker_name); + +ALTER TABLE ONLY observer_locations + ADD CONSTRAINT observer_locations_pkey PRIMARY KEY (observer_id, reported_at); + +ALTER TABLE ONLY observer_owners + ADD CONSTRAINT observer_owners_pkey PRIMARY KEY (observer_id); + +ALTER TABLE ONLY observer_scopes + ADD CONSTRAINT observer_scopes_pkey PRIMARY KEY (observer_id, scope_id); + +ALTER TABLE ONLY observer_telemetry + ADD CONSTRAINT observer_telemetry_observer_id_reported_at_key UNIQUE (observer_id, reported_at); + +ALTER TABLE ONLY observer_telemetry + ADD CONSTRAINT observer_telemetry_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY observers + ADD CONSTRAINT observers_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY observers + ADD CONSTRAINT observers_public_key_key UNIQUE (public_key); + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_packet_hash_observer_id_key UNIQUE (packet_hash, observer_id); + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY packets + ADD CONSTRAINT packets_pkey PRIMARY KEY (packet_hash); + +ALTER TABLE ONLY region_iatas + ADD CONSTRAINT region_iatas_pkey PRIMARY KEY (region_id, iata); + +ALTER TABLE ONLY regions + ADD CONSTRAINT regions_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY regions + ADD CONSTRAINT regions_slug_key UNIQUE (slug); + +ALTER TABLE ONLY trace_iatas + ADD CONSTRAINT trace_iatas_pkey PRIMARY KEY (trace_tag, iata); + +ALTER TABLE ONLY transport_scopes + ADD CONSTRAINT transport_scopes_name_key UNIQUE (name); + +ALTER TABLE ONLY transport_scopes + ADD CONSTRAINT transport_scopes_pkey PRIMARY KEY (id); + +CREATE UNIQUE INDEX accounts_name_uidx ON accounts USING btree (name) WHERE (deactivated_at IS NULL); + +CREATE INDEX idx_channel_iatas_iata ON channel_iatas USING btree (iata); + +CREATE INDEX idx_channel_messages_channel ON channel_messages USING btree (channel_id, sent_at DESC); + +CREATE INDEX idx_channel_messages_sent_brin ON channel_messages USING brin (sent_at); + +CREATE INDEX idx_channel_config_scopes_fingerprint ON channel_config_scopes USING btree (key_fingerprint); + +CREATE INDEX idx_meshmapper_channel_members_fingerprint ON meshmapper_channel_members USING btree (key_fingerprint); + +CREATE INDEX idx_channels_hash ON channels USING btree (channel_hash); + +CREATE UNIQUE INDEX idx_channels_hash_no_key ON channels USING btree (channel_hash) WHERE (key_fingerprint IS NULL); + +CREATE INDEX idx_channels_hashtag ON channels USING btree (hashtag) WHERE (hashtag IS NOT NULL); + +CREATE INDEX idx_channels_last_seen_id ON channels USING btree (last_seen DESC, id DESC); + +CREATE INDEX idx_known_routes_hop_count ON known_routes USING btree (iata, hop_count); + +CREATE INDEX idx_known_routes_iata_last_seen ON known_routes USING btree (iata, last_seen DESC); + +CREATE INDEX idx_known_routes_iata_keyset ON known_routes USING btree (iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC); + +CREATE INDEX idx_known_routes_keyset ON known_routes USING btree (date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC); + +CREATE INDEX idx_known_routes_last_seen ON known_routes USING btree (last_seen DESC); + +CREATE INDEX idx_known_routes_reconfirm ON known_routes USING btree (last_reconfirmed_at); + +CREATE UNIQUE INDEX idx_mv_radio_presets ON mv_radio_presets USING btree (preset, iata, source_type); + +CREATE INDEX idx_node_iatas_iata ON node_iatas USING btree (iata, last_heard DESC); + +CREATE INDEX idx_node_neighbors_neighbor ON node_neighbors USING btree (neighbor_id, iata); + +CREATE INDEX idx_node_neighbors_node ON node_neighbors USING btree (node_id, iata); + +CREATE INDEX idx_nodes_location ON nodes USING btree (latitude, longitude) WHERE ((latitude IS NOT NULL) AND (longitude IS NOT NULL)); + +CREATE INDEX idx_nodes_min_firmware ON nodes USING btree (min_firmware_version) WHERE (min_firmware_version IS NOT NULL); + +CREATE INDEX idx_nodes_multibyte_paths ON nodes USING btree (supports_multibyte_paths) WHERE supports_multibyte_paths; + +CREATE INDEX idx_nodes_multibyte_traces ON nodes USING btree (supports_multibyte_traces) WHERE supports_multibyte_traces; + +CREATE INDEX idx_nodes_pubkey ON nodes USING btree (public_key); + +CREATE INDEX idx_nodes_type_last_seen ON nodes USING btree (node_type, last_seen DESC); + +CREATE INDEX idx_observations_heard ON packet_observations USING btree (heard_at); + +CREATE INDEX idx_observations_iata_heard ON packet_observations USING btree (iata, heard_at DESC); + +CREATE INDEX idx_observations_observer ON packet_observations USING btree (observer_id, heard_at DESC); + +CREATE INDEX idx_observations_observer_id ON packet_observations USING btree (observer_id, id); + +CREATE INDEX idx_observations_packet ON packet_observations USING btree (packet_hash); + +CREATE INDEX idx_observations_route_evidence ON packet_observations USING btree (iata, hash_size, decode(md5(path_bytes), 'hex'::text), heard_at DESC, id DESC) WHERE ((path_bytes IS NOT NULL) AND (payload_type IS NOT NULL) AND (payload_type <> 9) AND (hop_count >= 2)); + +CREATE INDEX idx_observer_locations_recent ON observer_locations USING btree (observer_id, reported_at DESC); + +CREATE INDEX idx_observer_owners_node ON observer_owners USING btree (owner_node_id) WHERE (owner_node_id IS NOT NULL); + +CREATE INDEX idx_observers_last_iata ON observers USING btree (last_iata) WHERE (last_iata IS NOT NULL); + +CREATE INDEX idx_observers_last_seen ON observers USING btree (last_seen DESC); + +CREATE INDEX idx_observers_pubkey ON observers USING btree (public_key); + +CREATE INDEX idx_observers_type ON observers USING btree (observer_type) WHERE (observer_type IS NOT NULL); + +CREATE INDEX idx_packets_channel ON packets USING btree (channel_hash, first_heard_at DESC) WHERE (channel_hash IS NOT NULL); + +CREATE INDEX idx_packets_first_heard_brin ON packets USING brin (first_heard_at); + +CREATE INDEX idx_packets_last_heard ON packets USING btree (last_heard_at DESC); + +CREATE INDEX idx_packets_origin ON packets USING btree (origin_pubkey, first_heard_at DESC) WHERE (origin_pubkey IS NOT NULL); + +CREATE INDEX idx_packets_payload_type ON packets USING btree (payload_type, first_heard_at DESC); + +CREATE INDEX idx_packets_route_type ON packets USING btree (route_type, first_heard_at DESC); + +CREATE INDEX idx_packets_scope ON packets USING btree (scope_id) WHERE (scope_id IS NOT NULL); + +CREATE INDEX idx_packets_trace_tag ON packets USING btree (trace_tag) WHERE (trace_tag IS NOT NULL); + +CREATE INDEX idx_region_iatas_iata ON region_iatas USING btree (iata); + +CREATE INDEX idx_short_ids_p1 ON node_short_ids USING btree (iata, prefix_1); + +CREATE INDEX idx_short_ids_p2 ON node_short_ids USING btree (iata, prefix_2); + +CREATE INDEX idx_short_ids_p3 ON node_short_ids USING btree (iata, prefix_3); + +CREATE INDEX idx_short_ids_p4 ON node_short_ids USING btree (iata, prefix_4); + +CREATE INDEX idx_telemetry_observer_recent ON observer_telemetry USING btree (observer_id, reported_at DESC); + +CREATE INDEX idx_telemetry_reported_brin ON observer_telemetry USING brin (reported_at); + +CREATE INDEX idx_trace_iatas_iata ON trace_iatas USING btree (iata); + +CREATE INDEX idx_trace_tags_last_heard ON trace_tags USING btree (last_heard_at DESC); + +CREATE INDEX idx_trace_tags_keyset ON trace_tags USING btree (date_trunc('milliseconds', last_heard_at, 'UTC') DESC, trace_tag DESC); + +CREATE INDEX idx_trace_tags_first_heard ON trace_tags USING btree (first_heard_at); + +CREATE INDEX idx_analytics_observer_activity_observer ON analytics_hourly_observer_activity USING btree (observer_id, hour); + +CREATE INDEX idx_analytics_packet_sets_iatas ON analytics_hourly_packet_sets USING gin (iatas); + +CREATE INDEX idx_analytics_scope_sets_iatas ON analytics_hourly_scope_sets USING gin (iatas); + +CREATE INDEX idx_analytics_scope_sets_scope ON analytics_hourly_scope_sets USING btree (scope_id, hour); + +CREATE INDEX idx_analytics_advert_sets_iatas ON analytics_hourly_advert_sets USING gin (iatas); + +CREATE INDEX idx_analytics_talker_sets_iatas ON analytics_hourly_talker_sets USING gin (iatas); + +CREATE INDEX idx_analytics_rollup_hours_missing ON analytics_rollup_hours USING btree (hour) WHERE (status = 'missing'::text); + +CREATE INDEX idx_transport_scopes_fingerprint ON transport_scopes USING btree (key_fingerprint); + +ALTER TABLE ONLY channel_iatas + ADD CONSTRAINT channel_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY channel_keys + ADD CONSTRAINT channel_keys_channel_id_fkey FOREIGN KEY (channel_id) REFERENCES channels(id) ON DELETE CASCADE; + +ALTER TABLE ONLY channel_messages + ADD CONSTRAINT channel_messages_channel_id_fkey FOREIGN KEY (channel_id) REFERENCES channels(id); + +ALTER TABLE ONLY channel_messages + ADD CONSTRAINT channel_messages_packet_hash_fkey FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; + +ALTER TABLE ONLY known_routes + ADD CONSTRAINT known_routes_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY node_iatas + ADD CONSTRAINT node_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY node_iatas + ADD CONSTRAINT node_iatas_node_id_fkey FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY node_neighbors + ADD CONSTRAINT node_neighbors_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY node_neighbors + ADD CONSTRAINT node_neighbors_neighbor_id_fkey FOREIGN KEY (neighbor_id) REFERENCES nodes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY node_neighbors + ADD CONSTRAINT node_neighbors_node_id_fkey FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY node_short_ids + ADD CONSTRAINT node_short_ids_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY node_short_ids + ADD CONSTRAINT node_short_ids_node_id_fkey FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_default_scope_id_fkey FOREIGN KEY (default_scope_id) REFERENCES transport_scopes(id); + +ALTER TABLE ONLY observer_brokers + ADD CONSTRAINT observer_brokers_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_locations + ADD CONSTRAINT observer_locations_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata); + +ALTER TABLE ONLY observer_locations + ADD CONSTRAINT observer_locations_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_owners + ADD CONSTRAINT observer_owners_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_owners + ADD CONSTRAINT observer_owners_owner_node_id_fkey FOREIGN KEY (owner_node_id) REFERENCES nodes(id); + +ALTER TABLE ONLY observer_scopes + ADD CONSTRAINT observer_scopes_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_scopes + ADD CONSTRAINT observer_scopes_scope_id_fkey FOREIGN KEY (scope_id) REFERENCES transport_scopes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_telemetry + ADD CONSTRAINT observer_telemetry_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata); + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id); + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_packet_hash_fkey FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; + +ALTER TABLE ONLY packets + ADD CONSTRAINT packets_scope_id_fkey FOREIGN KEY (scope_id) REFERENCES transport_scopes(id); + +ALTER TABLE ONLY region_iatas + ADD CONSTRAINT region_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY region_iatas + ADD CONSTRAINT region_iatas_region_id_fkey FOREIGN KEY (region_id) REFERENCES regions(id) ON DELETE CASCADE; + +ALTER TABLE ONLY trace_iatas + ADD CONSTRAINT trace_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +-- + +-- diff --git a/db/migrations/001_initial_schema.sql b/db/migrations/001_initial_schema.sql deleted file mode 100644 index 973990f9..00000000 --- a/db/migrations/001_initial_schema.sql +++ /dev/null @@ -1,439 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: agpl - --- ============================================================ --- Beacon schema migration --- ============================================================ - --- ============================================================ --- IATA CODES --- ============================================================ - -CREATE TABLE iata_codes ( - iata CHAR(3) PRIMARY KEY, - display_name TEXT, - approx_lat DOUBLE PRECISION, - approx_lng DOUBLE PRECISION, - added_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - --- ============================================================ --- REGIONS --- ============================================================ - -CREATE TABLE regions ( - id SERIAL PRIMARY KEY, - slug TEXT UNIQUE NOT NULL, - name TEXT NOT NULL, - description TEXT, - display_order INT DEFAULT 0, - center_lat DOUBLE PRECISION, - center_lng DOUBLE PRECISION, - zoom_level INT DEFAULT 8, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE TABLE region_iatas ( - region_id INT NOT NULL REFERENCES regions(id) ON DELETE CASCADE, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - added_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (region_id, iata) -); - -CREATE INDEX idx_region_iatas_iata ON region_iatas(iata); - --- ============================================================ --- TRANSPORT SCOPES --- ============================================================ - -CREATE TABLE transport_scopes ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE, -- normalized key string e.g. "#bc" - display_name TEXT, -- optional friendly name - transport_key BYTEA NOT NULL, -- 16-byte derived key - key_fingerprint BYTEA NOT NULL, -- SHA256(transport_key)[:8] for fast lookup - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_transport_scopes_fingerprint ON transport_scopes(key_fingerprint); - --- ============================================================ --- NODES (must come before observers due to observer_owners FK) --- ============================================================ - -CREATE TABLE nodes ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - public_key BYTEA UNIQUE NOT NULL, - node_type SMALLINT NOT NULL, - name TEXT, - latitude DOUBLE PRECISION, - longitude DOUBLE PRECISION, - location_source TEXT, - last_advert_at TIMESTAMPTZ, - supports_multibyte_paths BOOLEAN NOT NULL DEFAULT FALSE, - supports_multibyte_traces BOOLEAN NOT NULL DEFAULT FALSE, - default_scope_id INT REFERENCES transport_scopes(id), - min_firmware_version TEXT GENERATED ALWAYS AS ( - CASE - WHEN supports_multibyte_paths THEN '1.14.0+' - WHEN supports_multibyte_traces THEN '1.11.0+' - ELSE NULL - END - ) STORED, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - radio_freq_mhz REAL, - radio_sf SMALLINT, - radio_bw_khz REAL, - metadata JSONB -); - -CREATE INDEX idx_nodes_type_last_seen ON nodes(node_type, last_seen DESC); -CREATE INDEX idx_nodes_location ON nodes(latitude, longitude) - WHERE latitude IS NOT NULL AND longitude IS NOT NULL; -CREATE INDEX idx_nodes_pubkey ON nodes(public_key); -CREATE INDEX idx_nodes_multibyte_paths ON nodes(supports_multibyte_paths) WHERE supports_multibyte_paths; -CREATE INDEX idx_nodes_multibyte_traces ON nodes(supports_multibyte_traces) WHERE supports_multibyte_traces; -CREATE INDEX idx_nodes_min_firmware ON nodes(min_firmware_version) WHERE min_firmware_version IS NOT NULL; - --- ============================================================ --- OBSERVERS --- ============================================================ - -CREATE TABLE observers ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - public_key BYTEA UNIQUE NOT NULL, - display_name TEXT, - observer_type TEXT, - software_version TEXT, - hardware_model TEXT, - firmware_version TEXT, - firmware_build TEXT, - radio_freq_mhz REAL, - radio_sf SMALLINT, - radio_bw_khz REAL, - radio_cr SMALLINT, - battery_level REAL, - uptime_seconds BIGINT, - status_metadata JSONB, - last_status_at TIMESTAMPTZ, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - observation_count BIGINT DEFAULT 0, - metadata JSONB -); - -CREATE INDEX idx_observers_last_seen ON observers(last_seen DESC); -CREATE INDEX idx_observers_pubkey ON observers(public_key); -CREATE INDEX idx_observers_type ON observers(observer_type) WHERE observer_type IS NOT NULL; - -CREATE TABLE observer_brokers ( - observer_id UUID NOT NULL REFERENCES observers(id) ON DELETE CASCADE, - broker_name TEXT NOT NULL, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_packet_at TIMESTAMPTZ, - auth_ok BOOLEAN DEFAULT TRUE, - PRIMARY KEY (observer_id, broker_name) -); - -CREATE TABLE observer_locations ( - observer_id UUID NOT NULL REFERENCES observers(id) ON DELETE CASCADE, - iata CHAR(3) REFERENCES iata_codes(iata), - latitude DOUBLE PRECISION, - longitude DOUBLE PRECISION, - reported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (observer_id, reported_at) -); - -CREATE TABLE observer_scopes ( - observer_id UUID NOT NULL REFERENCES observers(id) ON DELETE CASCADE, - scope_id INT NOT NULL REFERENCES transport_scopes(id) ON DELETE CASCADE, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (observer_id, scope_id) -); - -CREATE INDEX idx_observer_locations_recent ON observer_locations(observer_id, reported_at DESC); - -CREATE TABLE observer_telemetry ( - id BIGSERIAL PRIMARY KEY, - observer_id UUID NOT NULL REFERENCES observers(id) ON DELETE CASCADE, - reported_at TIMESTAMPTZ NOT NULL, - battery_voltage_mv INT, - airtime_tx_pct REAL, - airtime_rx_pct REAL, - noise_floor_db REAL, - uptime_seconds BIGINT, - queue_length INT, - debug_flags INT, - receive_errors INT, - UNIQUE (observer_id, reported_at) -); - -CREATE INDEX idx_telemetry_reported_brin ON observer_telemetry USING BRIN (reported_at); -CREATE INDEX idx_telemetry_observer_recent ON observer_telemetry(observer_id, reported_at DESC); - -CREATE TABLE observer_owners ( - observer_id UUID PRIMARY KEY REFERENCES observers(id) ON DELETE CASCADE, - owner_node_id UUID REFERENCES nodes(id), - owner_pubkey BYTEA, - contact_name TEXT, - contact_email TEXT, - notes TEXT, - source TEXT, - added_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_observer_owners_node ON observer_owners(owner_node_id) WHERE owner_node_id IS NOT NULL; - --- ============================================================ --- NODE IATAS AND SHORT IDS --- ============================================================ - -CREATE TABLE node_iatas ( - node_id UUID NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - first_heard TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_heard TIMESTAMPTZ NOT NULL DEFAULT NOW(), - observation_count BIGINT DEFAULT 0, - PRIMARY KEY (node_id, iata) -); - -CREATE INDEX idx_node_iatas_iata ON node_iatas(iata, last_heard DESC); - -CREATE TABLE node_short_ids ( - node_id UUID NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - prefix_4 BYTEA NOT NULL, - prefix_1 BYTEA GENERATED ALWAYS AS (substring(prefix_4 from 1 for 1)) STORED, - prefix_2 BYTEA GENERATED ALWAYS AS (substring(prefix_4 from 1 for 2)) STORED, - prefix_3 BYTEA GENERATED ALWAYS AS (substring(prefix_4 from 1 for 3)) STORED, - PRIMARY KEY (node_id, iata) -); - -CREATE INDEX idx_short_ids_p1 ON node_short_ids(iata, prefix_1); -CREATE INDEX idx_short_ids_p2 ON node_short_ids(iata, prefix_2); -CREATE INDEX idx_short_ids_p3 ON node_short_ids(iata, prefix_3); -CREATE INDEX idx_short_ids_p4 ON node_short_ids(iata, prefix_4); - --- ============================================================ --- PACKETS --- ============================================================ - -CREATE TABLE packets ( - packet_hash BYTEA PRIMARY KEY, - payload_type SMALLINT NOT NULL, - payload_version SMALLINT NOT NULL, - route_type SMALLINT NOT NULL, - transport_codes_present BOOLEAN DEFAULT FALSE, - region_code INT, - sub_region_code INT, - scope_id INT REFERENCES transport_scopes(id), - origin_pubkey BYTEA, - raw_payload BYTEA NOT NULL, - raw_header BYTEA NOT NULL, - parsed_payload JSONB, - decrypted BOOLEAN DEFAULT FALSE, - channel_hash BYTEA, - trace_tag BYTEA, - first_heard_at TIMESTAMPTZ NOT NULL, - last_heard_at TIMESTAMPTZ NOT NULL -); - -CREATE INDEX idx_packets_first_heard_brin ON packets USING BRIN (first_heard_at); -CREATE INDEX idx_packets_payload_type ON packets(payload_type, first_heard_at DESC); -CREATE INDEX idx_packets_route_type ON packets(route_type, first_heard_at DESC); -CREATE INDEX idx_packets_origin ON packets(origin_pubkey, first_heard_at DESC) - WHERE origin_pubkey IS NOT NULL; -CREATE INDEX idx_packets_channel ON packets(channel_hash, first_heard_at DESC) - WHERE channel_hash IS NOT NULL; - CREATE INDEX idx_packets_trace_tag ON packets(trace_tag) WHERE trace_tag IS NOT NULL; - --- ============================================================ --- PACKET OBSERVATIONS --- ============================================================ - -CREATE TABLE packet_observations ( - id BIGSERIAL PRIMARY KEY, - packet_hash BYTEA NOT NULL REFERENCES packets(packet_hash) ON DELETE CASCADE, - observer_id UUID NOT NULL REFERENCES observers(id), - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata), - heard_at TIMESTAMPTZ NOT NULL, - path_length_byte SMALLINT NOT NULL, - hash_size SMALLINT NOT NULL, - hop_count SMALLINT NOT NULL, - path_bytes BYTEA, - rssi SMALLINT, - snr REAL, - propagation_time_ms INT, - radio_freq_mhz REAL, - spread_factor SMALLINT, - bandwidth_khz REAL, - coding_rate SMALLINT, - source_broker TEXT, - UNIQUE (packet_hash, observer_id, heard_at) -); - -CREATE INDEX idx_observations_heard_brin ON packet_observations USING BRIN (heard_at); -CREATE INDEX idx_observations_iata_heard ON packet_observations(iata, heard_at DESC); -CREATE INDEX idx_observations_observer ON packet_observations(observer_id, heard_at DESC); -CREATE INDEX idx_observations_packet ON packet_observations(packet_hash); - --- ============================================================ --- CHANNELS AND CHAT MESSAGES --- ============================================================ - -CREATE TABLE channels ( - id SERIAL PRIMARY KEY, - channel_hash BYTEA NOT NULL, - -- key_fingerprint is the first 8 bytes of SHA256(key_bytes). - -- Combined with channel_hash it uniquely identifies a channel, - -- since a 1-byte hash has high collision probability. - -- NULL when the key is unknown (hash-only record). - key_fingerprint BYTEA, - name TEXT, - hashtag TEXT UNIQUE, -- e.g. "meshcore"; only set for hashtag-derived channels - is_hashtag BOOLEAN DEFAULT FALSE, - is_public BOOLEAN DEFAULT FALSE, - key_known BOOLEAN DEFAULT FALSE, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - message_count BIGINT DEFAULT 0, - -- Unique on (hash, fingerprint): allows multiple channels with the same - -- hash as long as they have different keys. NULL fingerprint is allowed - -- once per hash (the unknown-key record). - UNIQUE (channel_hash, key_fingerprint) -); - -CREATE UNIQUE INDEX idx_channels_hash_no_key -ON channels (channel_hash) -WHERE key_fingerprint IS NULL; - -CREATE INDEX idx_channels_last_seen ON channels(last_seen DESC); -CREATE INDEX idx_channels_hash ON channels(channel_hash); -CREATE INDEX idx_channels_hashtag ON channels(hashtag) WHERE hashtag IS NOT NULL; - -CREATE TABLE channel_keys ( - channel_id INT PRIMARY KEY REFERENCES channels(id) ON DELETE CASCADE, - key_bytes BYTEA NOT NULL, - key_fingerprint BYTEA NOT NULL, -- first 8 bytes of SHA256(key_bytes) - added_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - added_by TEXT -); - -CREATE TABLE channel_messages ( - id BIGSERIAL PRIMARY KEY, - channel_id INT NOT NULL REFERENCES channels(id), - packet_hash BYTEA NOT NULL REFERENCES packets(packet_hash), - sender_name TEXT, - sender_pubkey BYTEA, - content TEXT, - sent_at TIMESTAMPTZ NOT NULL, - UNIQUE (packet_hash) -); - -CREATE INDEX idx_channel_messages_channel ON channel_messages(channel_id, sent_at DESC); -CREATE INDEX idx_channel_messages_sent_brin ON channel_messages USING BRIN (sent_at); - --- ============================================================ --- ROUTES --- ============================================================ - -CREATE TABLE known_routes ( - id BIGSERIAL PRIMARY KEY, - node_ids UUID[] NOT NULL, -- resolved node UUIDs in hop order - hash_prefix BYTEA[] NOT NULL, -- raw hash bytes in hop order for prefix matching - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - hop_count INT NOT NULL, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - observation_count BIGINT NOt NULL DEFAULT 1, - UNIQUE (node_ids, iata) -); - -CREATE INDEX idx_known_routes_iata ON known_routes(iata); -CREATE INDEX idx_known_routes_hop_count ON known_routes(iata, hop_count); - --- ============================================================ --- NEIGHBORS --- ============================================================ - -CREATE TABLE node_neighbors ( - node_id UUID NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, - neighbor_id UUID NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - observation_count BIGINT NOT NULL DEFAULT 1, - PRIMARY KEY (node_id, neighbor_id, iata) -); - -CREATE INDEX idx_node_neighbors_node ON node_neighbors(node_id, iata); -CREATE INDEX idx_node_neighbors_neighbor ON node_neighbors(neighbor_id, iata); - --- ============================================================ --- MATERIALIZED VIEWS --- ============================================================ - -CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS -SELECT - iata, - date_trunc('hour', heard_at)::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets, - COUNT(DISTINCT observer_id) AS active_observers -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '7 days' -GROUP BY iata, date_trunc('hour', heard_at); - -CREATE MATERIALIZED VIEW mv_top_nodes_by_iata AS -SELECT - ni.iata, - ni.node_id, - n.name, - n.node_type, - ni.observation_count, - ni.last_heard -FROM node_iatas ni -JOIN nodes n ON n.id = ni.node_id -WHERE ni.last_heard > NOW() - INTERVAL '7 days'; - -CREATE UNIQUE INDEX idx_mv_top_nodes - ON mv_top_nodes_by_iata(iata, node_id); - -CREATE UNIQUE INDEX idx_mv_hourly_iata_stats - ON mv_hourly_iata_stats(iata, hour); - -CREATE MATERIALIZED VIEW mv_radio_presets AS -SELECT - concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text AS preset, - (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) AS iata, - 'observer' AS source_type, - COUNT(*) AS count -FROM observers o -WHERE o.radio_freq_mhz IS NOT NULL - AND o.radio_sf IS NOT NULL - AND o.radio_bw_khz IS NOT NULL -GROUP BY concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text, - (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) -HAVING (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) IS NOT NULL - -UNION ALL - -SELECT - concat(n.radio_freq_mhz, ',', n.radio_bw_khz, ',', n.radio_sf) AS preset, - ni.iata, - 'node' AS source_type, - COUNT(*) AS count -FROM nodes n -JOIN node_iatas ni ON ni.node_id = n.id -WHERE n.radio_freq_mhz IS NOT NULL - AND n.radio_sf IS NOT NULL - AND n.radio_bw_khz IS NOT NULL -GROUP BY concat(n.radio_freq_mhz, ',', n.radio_bw_khz, ',', n.radio_sf), ni.iata; - -CREATE UNIQUE INDEX idx_mv_radio_presets - ON mv_radio_presets(preset, iata, source_type); diff --git a/db/migrations/002_drop_single_hop_routes.sql b/db/migrations/002_drop_single_hop_routes.sql deleted file mode 100644 index 472962e5..00000000 --- a/db/migrations/002_drop_single_hop_routes.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: agpl - --- 002_drop_single_hop_routes.sql --- routes were being stored as long as the path was --- greater than 0. -DELETE FROM known_routes WHERE hop_count < 2; diff --git a/db/migrations/003_tag_ping_traces.sql b/db/migrations/003_tag_ping_traces.sql deleted file mode 100644 index 6ebfb458..00000000 --- a/db/migrations/003_tag_ping_traces.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: agpl - --- 003_tag_ping_traces.sql -UPDATE packets -SET parsed_payload = jsonb_set(parsed_payload, '{type}', '"PING"') -WHERE trace_tag IS NOT NULL - AND jsonb_array_length(parsed_payload->'pathHashes') = 1 - AND parsed_payload->>'type' = 'TRACE'; diff --git a/db/migrations/004_clean_non_infra_neighbors.sql b/db/migrations/004_clean_non_infra_neighbors.sql deleted file mode 100644 index 6cebb97b..00000000 --- a/db/migrations/004_clean_non_infra_neighbors.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Remove neighbor records where either side is not infrastructure (repeater or room). --- These are dirty rows from historical prefix collisions resolving to non-forwarding nodes. -DELETE FROM node_neighbors nn -USING nodes n -WHERE nn.neighbor_id = n.id - AND n.node_type NOT IN (2, 3); - -DELETE FROM node_neighbors nn -USING nodes n -WHERE nn.node_id = n.id - AND n.node_type NOT IN (2, 3); diff --git a/db/migrations/005_add_neighbor_snr.sql b/db/migrations/005_add_neighbor_snr.sql deleted file mode 100644 index 4440b081..00000000 --- a/db/migrations/005_add_neighbor_snr.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Adds an optional SNR reading to node_neighbors. Most rows (advert --- path-hop-derived topology) will leave this null; it's populated only --- when the neighbor relationship comes with an actual signal reading --- (e.g. observer-detected via DISCOVER_RESP). -ALTER TABLE node_neighbors ADD COLUMN snr REAL; diff --git a/db/migrations/006_fix_observation_dedup_key.sql b/db/migrations/006_fix_observation_dedup_key.sql deleted file mode 100644 index aa3cf7a8..00000000 --- a/db/migrations/006_fix_observation_dedup_key.sql +++ /dev/null @@ -1,24 +0,0 @@ --- 006_fix_observation_dedup_key.sql --- --- The previous unique constraint on (packet_hash, observer_id, heard_at) was --- too permissive: heard_at is stamped at ingest time (time.Now()), so MQTT --- redeliveries of the same packet arrive with slightly different timestamps --- and bypass dedup, producing duplicate observations for the same observer. --- --- An observer either heard a packet or they didn't. Drop heard_at from the --- constraint so that (packet_hash, observer_id) is the dedup key. - --- Remove duplicate observations, keeping the earliest (lowest id) per (packet_hash, observer_id). -DELETE FROM packet_observations -WHERE id NOT IN ( - SELECT MIN(id) - FROM packet_observations - GROUP BY packet_hash, observer_id -); - -ALTER TABLE packet_observations - DROP CONSTRAINT packet_observations_packet_hash_observer_id_heard_at_key; - -ALTER TABLE packet_observations - ADD CONSTRAINT packet_observations_packet_hash_observer_id_key - UNIQUE (packet_hash, observer_id); diff --git a/db/migrations/007_dedupe_hashonly_channels.sql b/db/migrations/007_dedupe_hashonly_channels.sql deleted file mode 100644 index e59e1eb7..00000000 --- a/db/migrations/007_dedupe_hashonly_channels.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Cleans up the ghost "unknown key" channel rows created by the --- side_effects.go bug where UpsertChannelHashOnly was called --- unconditionally on every GRP_TXT packet, even when the key was --- known and the message decrypted successfully. --- --- Safe because hash-only rows (key_fingerprint IS NULL) never have --- channel_messages attached to them -- InsertChannelMessage is only --- ever called against a keyed channel row. We only delete a hash-only --- row when a keyed row already exists for the same channel_hash; --- hashes with no known key at all (genuinely unknown) are left as-is. - -DELETE FROM channels c -WHERE c.key_fingerprint IS NULL - AND EXISTS ( - SELECT 1 FROM channels k - WHERE k.channel_hash = c.channel_hash - AND k.key_fingerprint IS NOT NULL - ); diff --git a/db/migrations/008_packets_last_heard_index.sql b/db/migrations/008_packets_last_heard_index.sql deleted file mode 100644 index f8a4cec0..00000000 --- a/db/migrations/008_packets_last_heard_index.sql +++ /dev/null @@ -1,8 +0,0 @@ --- 008_packets_last_heard_index.sql --- --- The packet list orders by last_heard_at but there was no index on it, so --- every page seq-scanned and sorted all packets (~2.8s per page on a 1.5M --- row table; 31ms with the index). Affordable to maintain now that presence --- coalescing keeps this column from being rewritten on every observation. - -CREATE INDEX idx_packets_last_heard ON packets(last_heard_at DESC); diff --git a/db/migrations/009_clean_zeroed_observer_telemetry.sql b/db/migrations/009_clean_zeroed_observer_telemetry.sql deleted file mode 100644 index 3289dc5a..00000000 --- a/db/migrations/009_clean_zeroed_observer_telemetry.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Cleans up observer_telemetry rows written from /status messages whose --- "stats" object was missing or renamed. status.go's json.Unmarshal --- silently tolerated the missing object, leaving the stats struct at Go --- zero-values, and the row was inserted unconditionally from there -- --- uptime_secs=0, battery_mv=0, noise_floor=0, tx/rx_air_secs=0. With the --- hourly ON CONFLICT (observer_id, reported_at) DO NOTHING dedup, --- whichever message landed first per hour won, so these zero rows --- regularly became the hour's telemetry and corrupted the 24h/7d/30d --- aggregates (AVG(noise_floor_db), AVG(battery_voltage_mv), MAX-MIN --- airtime). --- --- Safe because a running observer never reports uptime_seconds == 0; --- this is the same signal now used by status.go to skip the insert going --- forward (see internal/ingest/status.go handleStatus). - -DELETE FROM observer_telemetry WHERE uptime_seconds = 0; diff --git a/db/migrations/010_add_node_clock_drift.sql b/db/migrations/010_add_node_clock_drift.sql deleted file mode 100644 index b79a56e8..00000000 --- a/db/migrations/010_add_node_clock_drift.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Stores the device-reported clock drift measured from each ADVERT: the signed --- advert body already carries the device's own wall-clock timestamp --- (advert.Timestamp, epoch seconds), which the server previously decoded and --- then discarded before it reached the nodes table. This column stores --- (device timestamp - server receive time) in seconds, computed at the same --- moment nodes.last_advert_at is set to NOW(), so last_advert_at doubles as --- the "when was this drift measured" timestamp with no separate column --- needed. NULL until a node's first successfully-verified advert is upserted. --- --- +ve = device clock ahead of the server; -ve = behind. Surfaced on the node --- API for repeaters/room servers only (nodeType 2/3) -- see internal/api/nodes.go. - -ALTER TABLE nodes ADD COLUMN device_clock_drift_seconds INTEGER; diff --git a/db/migrations/011_channel_iatas.sql b/db/migrations/011_channel_iatas.sql deleted file mode 100644 index a89885d6..00000000 --- a/db/migrations/011_channel_iatas.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Per-IATA channel activity so the IATA filter skips the ~7s EXISTS over packets. --- Keyed by raw hash (channels can share one), so no FK to channels. - -CREATE TABLE channel_iatas ( - channel_hash BYTEA NOT NULL, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - last_heard TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (channel_hash, iata) -); - -CREATE INDEX idx_channel_iatas_iata ON channel_iatas(iata); - --- Seed from retained packets; parallelism off so the join spills to disk, not /dev/shm. -SET max_parallel_workers_per_gather = 0; - -INSERT INTO channel_iatas (channel_hash, iata, last_heard) -SELECT p.channel_hash, po.iata, MAX(po.heard_at) -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -WHERE p.channel_hash IS NOT NULL -GROUP BY p.channel_hash, po.iata; - -RESET max_parallel_workers_per_gather; diff --git a/db/migrations/012_trace_iatas.sql b/db/migrations/012_trace_iatas.sql deleted file mode 100644 index ffc9840b..00000000 --- a/db/migrations/012_trace_iatas.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Per-IATA trace activity so the trace filter stops joining every trace packet --- to observations (the hash join was overrunning /dev/shm). - -CREATE TABLE trace_iatas ( - trace_tag BYTEA NOT NULL, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - last_heard TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (trace_tag, iata) -); - -CREATE INDEX idx_trace_iatas_iata ON trace_iatas(iata); - --- Seed from retained packets; parallelism off so the join spills to disk, not /dev/shm. -SET max_parallel_workers_per_gather = 0; - -INSERT INTO trace_iatas (trace_tag, iata, last_heard) -SELECT p.trace_tag, po.iata, MAX(po.heard_at) -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -WHERE p.trace_tag IS NOT NULL -GROUP BY p.trace_tag, po.iata; - -RESET max_parallel_workers_per_gather; diff --git a/db/migrations/013_packets_scope_index.sql b/db/migrations/013_packets_scope_index.sql deleted file mode 100644 index 73029cf9..00000000 --- a/db/migrations/013_packets_scope_index.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Scope stats count off scope_id, which had no index. - -CREATE INDEX idx_packets_scope ON packets(scope_id) WHERE scope_id IS NOT NULL; diff --git a/db/migrations/014_known_routes_last_seen_index.sql b/db/migrations/014_known_routes_last_seen_index.sql deleted file mode 100644 index faa58081..00000000 --- a/db/migrations/014_known_routes_last_seen_index.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Routes list orders by last_seen; index it (was seq-scanning ~150k rows per page). - -CREATE INDEX idx_known_routes_last_seen ON known_routes(last_seen DESC); diff --git a/db/migrations/015_observations_payload_type.sql b/db/migrations/015_observations_payload_type.sql deleted file mode 100644 index e3bb79bf..00000000 --- a/db/migrations/015_observations_payload_type.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Copy payload_type onto the observation (like source_broker) so the breakdown --- drops its join back to packets, which was overrunning /dev/shm and 500ing. - -ALTER TABLE packet_observations ADD COLUMN payload_type SMALLINT; - --- Backfill the last 7 days only (all the breakdown reads); parallelism off so --- the join spills to disk, not /dev/shm. -SET max_parallel_workers_per_gather = 0; - -UPDATE packet_observations po -SET payload_type = p.payload_type -FROM packets p -WHERE p.packet_hash = po.packet_hash - AND po.heard_at > NOW() - INTERVAL '7 days' - AND po.payload_type IS NULL; - -RESET max_parallel_workers_per_gather; diff --git a/db/migrations/016_mv_payload_breakdown.sql b/db/migrations/016_mv_payload_breakdown.sql deleted file mode 100644 index bce48d9d..00000000 --- a/db/migrations/016_mv_payload_breakdown.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Precomputed payload breakdown, bucketed by hour so the stats endpoint can --- serve any window (24h/7d/30d) by summing the buckets in range instead of --- scanning a month of observations per request. - -CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS -SELECT - iata, - payload_type, - date_trunc('hour', heard_at)::timestamptz AS bucket, - COUNT(*) AS count -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at); - -CREATE UNIQUE INDEX idx_mv_payload_breakdown - ON mv_payload_breakdown_by_iata(iata, payload_type, bucket); diff --git a/db/migrations/017_mv_top_observers.sql b/db/migrations/017_mv_top_observers.sql deleted file mode 100644 index 9cc0c1f2..00000000 --- a/db/migrations/017_mv_top_observers.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Precomputed observer activity, bucketed by hour so top-observers can be --- served for any window (24h/7d/30d) by summing the buckets in range. Was a --- ~2s per-request scan of a week of observations. - -CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at)::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM packet_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at); - -CREATE UNIQUE INDEX idx_mv_top_observers - ON mv_top_observers_by_iata(iata, observer_id, bucket); diff --git a/db/migrations/018_mv_top_talkers.sql b/db/migrations/018_mv_top_talkers.sql deleted file mode 100644 index 1b2132cf..00000000 --- a/db/migrations/018_mv_top_talkers.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Hourly-bucketed talker counts so top-talkers serves any window from a small --- table instead of the live channel_messages/observations join (millions of rows --- per request). Per-IATA counts; cross-IATA senders are approximate all-regions. - -CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at)::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN packet_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at); - -CREATE UNIQUE INDEX idx_mv_top_talkers - ON mv_top_talkers_by_iata(iata, sender_name, bucket); diff --git a/db/migrations/019_mv_top_advertisers.sql b/db/migrations/019_mv_top_advertisers.sql deleted file mode 100644 index 47488c1c..00000000 --- a/db/migrations/019_mv_top_advertisers.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Hourly-bucketed advert counts (distinct ADVERT packets per node = adverts sent, --- not heard) so top-advertisers serves any window from a small table, not the live --- scan. Per-IATA counts; cross-IATA adverts are approximate all-regions. - -CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at)::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at); - -CREATE UNIQUE INDEX idx_mv_top_advertisers - ON mv_top_advertisers_by_iata(iata, node_id, bucket); diff --git a/db/migrations/020_add_iata_border.sql b/db/migrations/020_add_iata_border.sql deleted file mode 100644 index bc728402..00000000 --- a/db/migrations/020_add_iata_border.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Optional GeoJSON border for the region border map feature. Stores a full --- GeoJSON Feature (type/bbox/properties/geometry), not a bare geometry, so --- properties can be added later without a schema change. Geometry is always --- Polygon or MultiPolygon; bbox is computed and stored at write time (see --- internal/config/border.go) so reads never need to recompute it. --- --- NULL for the vast majority of IATAs, matching display_name/approx_lat/ --- approx_lng's existing "optional, mostly unset" pattern. Written via the --- same config-file-driven seeding path as those columns (internal/config/ --- seed.go), not a runtime HTTP write endpoint -- this codebase has none. --- --- Deliberately excluded from GET /api/v1/iatas (border polygons are large -- --- thousands of vertices, MultiPolygon for archipelago coastlines -- and --- would bloat every list response for the ~all IATAs that don't have one). --- Served only via GET /api/v1/iatas/{iata}/border, fetched lazily one at a --- time on client-side region selection. - -ALTER TABLE iata_codes ADD COLUMN border JSONB; diff --git a/db/node_location_integration_test.go b/db/node_location_integration_test.go new file mode 100644 index 00000000..51f700c4 --- /dev/null +++ b/db/node_location_integration_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" +) + +func TestNodeLocationResetPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE nodes (LIKE public.nodes INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + lat, lon := 45.0, -75.0 + params := ingest.UpsertNodeParams{PublicKey: []byte{0x97}, NodeType: 2, Name: "location fixture", Latitude: &lat, Longitude: &lon} + id, err := store.UpsertNode(ctx, params, ingest.RadioSettings{}) + if err != nil { + t.Fatal(err) + } + f := func(v float64) *float64 { return &v } + advert := "advert" + for _, tc := range []struct { + name string + lat, lon *float64 + clear bool + wantLat *float64 + wantLon *float64 + wantSource *string + }{ + {"omission preserves location", nil, nil, false, f(45), f(-75), &advert}, + {"explicit reset clears location", nil, nil, true, nil, nil, nil}, + {"omission preserves cleared location", nil, nil, false, nil, nil, nil}, + {"new coordinates restore location", f(46), f(-76), false, f(46), f(-76), &advert}, + } { + t.Run(tc.name, func(t *testing.T) { + params.Latitude, params.Longitude, params.ClearLocation = tc.lat, tc.lon, tc.clear + updatedID, err := store.UpsertNode(ctx, params, ingest.RadioSettings{}) + if err != nil || updatedID != id { + t.Fatalf("node identity changed: %v", err) + } + assertNodeLocation(t, ctx, tx, []byte{0x97}, tc.wantLat, tc.wantLon, tc.wantSource) + }) + } + t.Run("new node with cleared location has no source", func(t *testing.T) { + fresh := ingest.UpsertNodeParams{PublicKey: []byte{0x98}, NodeType: 2, Name: "cleared fixture", ClearLocation: true} + if _, err := store.UpsertNode(ctx, fresh, ingest.RadioSettings{}); err != nil { + t.Fatal(err) + } + assertNodeLocation(t, ctx, tx, []byte{0x98}, nil, nil, nil) + }) +} + +func assertNodeLocation(t *testing.T, ctx context.Context, tx pgx.Tx, pubkey []byte, wantLat, wantLon *float64, wantSource *string) { + t.Helper() + var gotLat, gotLon *float64 + var gotSource *string + if err := tx.QueryRow(ctx, "SELECT latitude, longitude, location_source FROM nodes WHERE public_key=$1", pubkey).Scan(&gotLat, &gotLon, &gotSource); err != nil { + t.Fatal(err) + } + if !equalPtr(gotLat, wantLat) || !equalPtr(gotLon, wantLon) || !equalPtr(gotSource, wantSource) { + t.Fatalf("location = (%v,%v,%v), want (%v,%v,%v)", deref(gotLat), deref(gotLon), deref(gotSource), deref(wantLat), deref(wantLon), deref(wantSource)) + } +} + +func equalPtr[T comparable](a, b *T) bool { + return a == nil && b == nil || a != nil && b != nil && *a == *b +} + +func deref[T any](p *T) any { + if p == nil { + return nil + } + return *p +} diff --git a/db/nodes.go b/db/nodes.go index 30bea8ab..a602cb3d 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -9,7 +9,7 @@ import ( "encoding/json" "errors" "fmt" - "log" + "log/slog" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -34,6 +34,7 @@ func (s *Store) UpsertNode(ctx context.Context, n ingest.UpsertNodeParams, radio Name: &n.Name, Latitude: n.Latitude, Longitude: n.Longitude, + ClearLocation: n.ClearLocation, DeviceClockDriftSeconds: driftSeconds, } if radio.FreqMHz != 0 { @@ -61,12 +62,13 @@ func (s *Store) UpsertNodeShortID(ctx context.Context, nodeID uuid.UUID, iata st }) } -func (s *Store) UpsertNodeNeighbor(ctx context.Context, nodeID, neighborID uuid.UUID, iata string, snr *float32) error { +func (s *Store) UpsertNodeNeighbor(ctx context.Context, nodeID, neighborID uuid.UUID, iata string, snr *float32, regionScope *string) error { return s.q.UpsertNodeNeighbor(ctx, sqlc.UpsertNodeNeighborParams{ - NodeID: nodeID, - NeighborID: neighborID, - Iata: iata, - Snr: snr, + NodeID: nodeID, + NeighborID: neighborID, + Iata: iata, + Snr: snr, + RegionScope: regionScope, }) } @@ -131,7 +133,7 @@ func (s *Store) ListNodes(ctx context.Context, nodeType int16, iatas []string, s } if len(v.Iatas) > 0 { if err := json.Unmarshal(v.Iatas, &node.IATAs); err != nil { - log.Printf("store: failed to unmarshal node iatas: %v", err) + slog.Error("store: failed to unmarshal node iatas", "component", "db", "error", err) node.IATAs = []api.NodeIATA{} } } @@ -183,13 +185,13 @@ func (s *Store) GetNode(ctx context.Context, nodeID uuid.UUID) (*api.Node, error } neighbors, err := s.GetNodeNeighbors(ctx, nodeID) if err != nil { - log.Printf("store: GetNodeNeighbors failed for %s: %v", nodeID, err) + slog.Error(fmt.Sprintf("store: GetNodeNeighbors failed for %s", nodeID), "component", "db", "error", err) neighbors = []api.NodeNeighbor{} } node.Neighbors = neighbors if len(row.Iatas) > 0 { if err := json.Unmarshal(row.Iatas, &node.IATAs); err != nil { - log.Printf("store: failed to unmarshal node iatas: %v", err) + slog.Error("store: failed to unmarshal node iatas", "component", "db", "error", err) node.IATAs = []api.NodeIATA{} } } diff --git a/db/nodes_integration_test.go b/db/nodes_integration_test.go new file mode 100644 index 00000000..4a45f8a0 --- /dev/null +++ b/db/nodes_integration_test.go @@ -0,0 +1,185 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +//go:embed queries/queries.sql +var nodeTestQueries string + +// Run against a migrated PostgreSQL database with BEACON_TEST_POSTGRES_DSN. +// All fixture writes are transaction-local temporary tables, rolled back on exit. +func TestListNodesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"nodes", "node_iatas", "node_neighbors", "observers", "transport_scopes"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO transport_scopes (id, name, transport_key, key_fingerprint) VALUES (1, '#test', decode(repeat('00', 16), 'hex'), decode(repeat('00', 8), 'hex')); +INSERT INTO nodes (id, public_key, node_type, name, last_seen, default_scope_id, supports_multibyte_paths, supports_multibyte_traces) +SELECT md5(i::text)::uuid, decode(lpad(to_hex(i), 64, '0'), 'hex'), (i % 4 + 1)::smallint, + 'node-' || i, '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second', + CASE WHEN i % 2 = 0 THEN 1 END, i % 2 = 0, i % 3 = 0 +FROM generate_series(1, 20000) i; +INSERT INTO node_iatas (node_id, iata, last_heard) +SELECT md5(i::text)::uuid, 'YVR', '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second' +FROM generate_series(1, 19999) i; +INSERT INTO node_iatas (node_id, iata, last_heard) +SELECT md5(i::text)::uuid, 'YYJ', '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second' - interval '1 hour' +FROM generate_series(2, 19998, 2) i; +INSERT INTO node_neighbors (node_id, neighbor_id, iata) +SELECT md5(i::text)::uuid, md5((i+1)::text)::uuid, iata +FROM generate_series(1, 19999) i CROSS JOIN (VALUES ('YVR'), ('YYJ')) regions(iata); +INSERT INTO observers (id, public_key) +SELECT id, public_key FROM nodes WHERE name = 'node-10'; +ANALYZE nodes; ANALYZE node_iatas; ANALYZE node_neighbors; ANALYZE observers; ANALYZE transport_scopes; +`) + if err != nil { + t.Fatal(err) + } + query := sqlc.New(tx) + base := sqlc.ListNodesParams{Column1: int16(0), Column3: "any", Column4: "any", Column6: "", Limit: 51} + for _, tc := range []struct { + name string + change func(*sqlc.ListNodesParams) + want []int + }{ + {"page", func(p *sqlc.ListNodesParams) { p.Limit = 3 }, []int{1, 2, 3}}, + {"type", func(p *sqlc.ListNodesParams) { p.Column1 = 2; p.Limit = 3 }, []int{1, 5, 9}}, + {"multi IATA dedup", func(p *sqlc.ListNodesParams) { p.Column2 = []string{"YVR", "YYJ"}; p.Limit = 3 }, []int{1, 2, 3}}, + {"IATA", func(p *sqlc.ListNodesParams) { p.Column2 = []string{"YYJ"}; p.Limit = 3 }, []int{2, 4, 6}}, + {"unknown IATA", func(p *sqlc.ListNodesParams) { p.Column2 = []string{"ZZZ"} }, nil}, + {"scope", func(p *sqlc.ListNodesParams) { p.Column9 = "#test"; p.Limit = 3 }, []int{2, 4, 6}}, + {"capabilities", func(p *sqlc.ListNodesParams) { p.Column3 = "true"; p.Column4 = "false"; p.Limit = 3 }, []int{2, 4, 8}}, + {"name", func(p *sqlc.ListNodesParams) { p.Column6 = "NODE-19"; p.Limit = 3 }, []int{19, 190, 191}}, + {"cursor", func(p *sqlc.ListNodesParams) { + p.Column7 = pgtype.Timestamptz{Time: time.Date(2026, 1, 1, 0, 0, -3, 0, time.UTC), Valid: true} + p.Limit = 3 + }, []int{4, 5, 6}}, + {"public key", func(p *sqlc.ListNodesParams) { p.Column5, _ = hex.DecodeString(fmt.Sprintf("%064x", 10)) }, []int{10}}, + {"public key prefix", func(p *sqlc.ListNodesParams) { p.Column11 = strings.Repeat("0", 61) + "00A" }, []int{10}}, + {"no IATA", func(p *sqlc.ListNodesParams) { p.Column5, _ = hex.DecodeString(fmt.Sprintf("%064x", 20000)) }, []int{20000}}, + } { + for _, neighbors := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/neighbors=%t", tc.name, neighbors), func(t *testing.T) { + params := base + tc.change(¶ms) + params.Column10 = neighbors + rows, err := query.ListNodes(ctx, params) + if err != nil { + t.Fatal(err) + } + if len(rows) != len(tc.want) { + t.Fatalf("got %d rows, want %d", len(rows), len(tc.want)) + } + for i, number := range tc.want { + row := rows[i] + if row.Name == nil || *row.Name != fmt.Sprintf("node-%d", number) { + t.Fatalf("wrong row at %d: %v", i, row.Name) + } + var iatas []struct { + IATA string `json:"iata"` + } + if len(row.Iatas) > 0 { + if err := json.Unmarshal(row.Iatas, &iatas); err != nil { + t.Fatal(err) + } + } + wantIATAs := 1 + if number%2 == 0 { + wantIATAs = 2 + } + wantNeighbors := int64(1) + if number == 20000 { + wantIATAs, wantNeighbors = 0, 0 + } + if len(iatas) != wantIATAs || (len(iatas) > 0 && iatas[0].IATA != "YVR") { + t.Errorf("wrong IATAs: %s", row.Iatas) + } + if row.KnownNeighborCount != wantNeighbors { + t.Errorf("neighbor count %d, want %d", row.KnownNeighborCount, wantNeighbors) + } + if neighbors && len(row.NeighborIds) != int(wantNeighbors) { + t.Errorf("neighbor IDs %v", row.NeighborIds) + } + if !neighbors && len(row.NeighborIds) != 0 { + t.Errorf("unrequested neighbor IDs %v", row.NeighborIds) + } + if row.IsObserver != (number == 10) { + t.Error("wrong observer flag") + } + if number == 10 && row.ObserverID != row.ID { + t.Error("wrong observer ID") + } + } + }) + } + } + + // A small page must not aggregate every node's IATA membership. Assert work, + // not elapsed time: shared CI and Pi hosts have variable scheduling latency. + queries := strings.ReplaceAll(nodeTestQueries, "\r\n", "\n") + sql := strings.Split(strings.Split(queries, "-- name: ListNodes :many\n")[1], "\n-- name:")[0] + var planJSON []byte + err = tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+sql, + int16(0), []string(nil), "any", "any", []byte(nil), "", nil, int32(51), "", false, "").Scan(&planJSON) + if err != nil { + t.Fatal(err) + } + var plans []struct { + Plan map[string]any + ExecutionTime float64 `json:"Execution Time"` + } + if err := json.Unmarshal(planJSON, &plans); err != nil { + t.Fatal(err) + } + var visited float64 + var walk func(map[string]any) + walk = func(plan map[string]any) { + if plan["Relation Name"] == "node_iatas" { + visited += plan["Actual Rows"].(float64) * plan["Actual Loops"].(float64) + } + if children, ok := plan["Plans"].([]any); ok { + for _, child := range children { + walk(child.(map[string]any)) + } + } + } + walk(plans[0].Plan) + t.Logf("20,000 nodes / 51-row page: execution %.3f ms; IATA rows processed %.0f", plans[0].ExecutionTime, visited) + if visited > 200 { + t.Errorf("small node page processed %.0f IATA rows; budget 200", visited) + } +} diff --git a/db/observer_adverts_integration_test.go b/db/observer_adverts_integration_test.go new file mode 100644 index 00000000..2531e37d --- /dev/null +++ b/db/observer_adverts_integration_test.go @@ -0,0 +1,85 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/jackc/pgx/v5" +) + +func TestObserverAdvertsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + _, err = tx.Exec(ctx, ` +CREATE TEMP TABLE packets (LIKE public.packets INCLUDING ALL) ON COMMIT DROP; +CREATE TEMP TABLE packet_observations (LIKE public.packet_observations INCLUDING ALL) ON COMMIT DROP; +CREATE TEMP TABLE nodes (LIKE public.nodes INCLUDING ALL) ON COMMIT DROP; +INSERT INTO nodes(public_key,node_type,name) VALUES (decode(repeat('01',32),'hex'),1,'Known fixture node'); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,origin_pubkey,raw_payload,raw_header,first_heard_at,last_heard_at) VALUES + (decode(repeat('a1',32),'hex'),4,0,1,NULL,'\x00','\x00',now(),now()), + (decode(repeat('b2',32),'hex'),4,0,1,decode(repeat('01',32),'hex'),'\x00','\x00',now(),now()); +INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES + (1,decode(repeat('a1',32),'hex'),'00000000-0000-0000-0000-000000000106','YOW',now(),0,1,0), + (2,decode(repeat('b2',32),'hex'),'00000000-0000-0000-0000-000000000106','YOW',now(),0,1,0);`) + if err != nil { + t.Fatal(err) + } + router := handlers.ObserversRouter(&Store{q: sqlc.New(tx)}) + request := func(query string) api.Page[api.AdvertObservation] { + t.Helper() + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/00000000-0000-0000-0000-000000000106/adverts?"+query, nil).WithContext(ctx)) + if response.Code != http.StatusOK { + t.Fatalf("advert page HTTP %d", response.Code) + } + var page api.Page[api.AdvertObservation] + if err := json.Unmarshal(response.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + return page + } + page := request("limit=50") + if len(page.Items) != 2 || page.Items[0].ID != 1 || page.Items[1].ID != 2 { + t.Fatal("missing-origin advert was not preserved in the ordered page") + } + if key := page.Items[0].NodePublicKey; key == nil || *key != "" { + t.Errorf("missing origin key = %v, want an empty string", key) + } + if key := page.Items[1].NodePublicKey; key == nil || *key != strings.Repeat("01", 32) { + t.Error("known origin key changed") + } + first := request("limit=1") + if !first.HasMore || first.NextCursor == nil || *first.NextCursor != 1 { + t.Fatal("missing-origin advert broke the cursor") + } + last := request("limit=1&cursor=1") + if len(last.Items) != 1 || last.Items[0].ID != 2 || last.HasMore { + t.Fatal("continuing after the missing-origin advert lost the known advert") + } +} diff --git a/db/observer_comparison.go b/db/observer_comparison.go new file mode 100644 index 00000000..5186589f --- /dev/null +++ b/db/observer_comparison.go @@ -0,0 +1,34 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetObserverComparison(ctx context.Context, a, b uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { + row, err := s.q.GetObserverComparison(ctx, sqlc.GetObserverComparisonParams{ + ObserverA: a, ObserverB: b, + Since: pgtype.Timestamptz{Time: since, Valid: true}, + Until: pgtype.Timestamptz{Time: until, Valid: true}, + Iatas: iatas, + }) + if err != nil { + return nil, err + } + if !row.KnownA || !row.KnownB { + return nil, pgx.ErrNoRows + } + return &api.ObserverComparison{ + ObserverA: a, ObserverB: b, Since: since.UnixMilli(), Until: until.UnixMilli(), + TotalPackets: row.TotalPackets, OnlyA: row.OnlyA, OnlyB: row.OnlyB, Both: row.Both, + }, nil +} diff --git a/db/observer_comparison_integration_test.go b/db/observer_comparison_integration_test.go new file mode 100644 index 00000000..8abc290c --- /dev/null +++ b/db/observer_comparison_integration_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestObserverComparisonPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + // Temporary tables shadow the private database's tables; no retained rows + // or sequences are changed, and the fixture is rolled back on every exit. + for _, table := range []string{"observers", "packets", "packet_observations"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO observers (id, public_key) +SELECT ('00000000-0000-0000-0000-' || lpad(i::text,12,'0'))::uuid, + decode(lpad(to_hex(i),64,'0'),'hex') FROM generate_series(1,3) i; +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),4,0,route,'\x00','\x00',NOW(),NOW() +FROM (VALUES (1,1),(2,0),(3,1),(4,1),(5,0),(6,1),(7,2),(8,3),(9,1),(10,1)) v(i,route); +INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,source_broker) +SELECT id,decode(lpad(to_hex(packet),2,'0'),'hex'), + ('00000000-0000-0000-0000-' || lpad(observer::text,12,'0'))::uuid, + iata,'2026-01-01 00:00:00+00'::timestamptz+second*interval '1 second',0,1,0,broker +FROM (VALUES + (1,1,1,'YVR',1,'a'),(2,1,1,'YVR',2,'b'), + (3,2,2,'YVR',2,'a'),(4,3,1,'YVR',3,'a'),(5,3,2,'YYJ',4,'a'), + (6,4,1,'YVR',-1,'a'),(7,4,2,'YVR',5,'a'),(8,5,1,'YVR',0,'a'), + (9,6,2,'YVR',10,'a'),(10,7,1,'YVR',1,'a'),(11,7,2,'YVR',2,'a'), + (12,8,2,'YVR',1,'a'),(13,9,3,'YVR',1,'a'), + (14,10,1,'YVR',3,'a'),(15,10,2,'YVR',9,'a')) v(id,packet,observer,iata,second,broker) +ORDER BY id +ON CONFLICT (packet_hash,observer_id) DO NOTHING; +`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + a := uuid.MustParse("00000000-0000-0000-0000-000000000001") + b := uuid.MustParse("00000000-0000-0000-0000-000000000002") + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + iatas []string + from, to int + reverse bool + counts [4]int64 // union, only A, only B, both + }{ + {"global distinct floods", nil, 0, 10, false, [4]int64{6, 2, 2, 2}}, + {"empty filter", []string{}, 0, 10, false, [4]int64{6, 2, 2, 2}}, + {"YVR receptions", []string{"YVR"}, 0, 10, false, [4]int64{6, 3, 2, 1}}, + {"reversed observers", []string{"YVR"}, 0, 10, true, [4]int64{6, 2, 3, 1}}, + {"YYJ receptions", []string{"YYJ"}, 0, 10, false, [4]int64{1, 0, 1, 0}}, + {"region union", []string{"YYJ", "YVR", "YYJ"}, 0, 10, false, [4]int64{6, 2, 2, 2}}, + {"unknown IATA", []string{"ZZZ"}, 0, 10, false, [4]int64{}}, + {"inclusive start exclusive end", nil, 0, 1, false, [4]int64{1, 1, 0, 0}}, + {"next interval", nil, 1, 3, false, [4]int64{2, 1, 1, 0}}, + {"known observers empty interval", nil, 20, 30, false, [4]int64{}}, + } { + t.Run(tc.name, func(t *testing.T) { + first, second := a, b + if tc.reverse { + first, second = b, a + } + since, until := start.Add(time.Duration(tc.from)*time.Second), start.Add(time.Duration(tc.to)*time.Second) + got, err := store.GetObserverComparison(ctx, first, second, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + if got.ObserverA != first || got.ObserverB != second || got.Since != since.UnixMilli() || got.Until != until.UnixMilli() { + t.Fatalf("comparison context = %+v", got) + } + if counts := [4]int64{got.TotalPackets, got.OnlyA, got.OnlyB, got.Both}; counts != tc.counts { + t.Fatalf("counts = %v; want %v", counts, tc.counts) + } + }) + } + if _, err := store.GetObserverComparison(ctx, a, uuid.New(), start, start.Add(time.Hour), nil); !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("unknown observer: %v", err) + } +} diff --git a/db/observer_last_iata_integration_test.go b/db/observer_last_iata_integration_test.go new file mode 100644 index 00000000..ff1c4af3 --- /dev/null +++ b/db/observer_last_iata_integration_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/MeshCore-Beacon/beacon-server/internal/presence" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestObserverLastIATAPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + c := presence.New(store, time.Hour, time.Hour) + pkA, pkB, pkC := bytes.Repeat([]byte{1}, 32), bytes.Repeat([]byte{2}, 32), bytes.Repeat([]byte{3}, 32) + ids := map[string]uuid.UUID{} + heard := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + // Mirrors ingest: upsert the observer, then record the observation. + packet := func(name string, pk []byte, iata string) { + t.Helper() + id, _, err := c.UpsertObserver(ctx, pk, iata) + if err != nil { + t.Fatal(err) + } + ids[name] = id + heard = heard.Add(time.Second) + hash := []byte(heard.String()) + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ($1, 4, 0, 1, '\x00', '\x00', $2, $2)`, hash, heard); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, `INSERT INTO packet_observations (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +VALUES ($1, $2, $3, $4, 0, 1, 0)`, hash, id, iata, heard); err != nil { + t.Fatal(err) + } + } + status := func(pk []byte) { + t.Helper() + if _, _, err := c.UpsertObserver(ctx, pk, ""); err != nil { + t.Fatal(err) + } + } + + packet("A", pkA, "YVR") + packet("A", pkA, "YYZ") + status(pkA) + packet("B", pkB, "YOW") + if _, err := c.UpdateObserverStatus(ctx, ingest.UpdateObserverStatusParams{PublicKey: pkB, DisplayName: "b"}); err != nil { + t.Fatal(err) + } + packet("B", pkB, "YYJ") // write-through after the identity was dropped + status(pkC) + c.Flush(ctx) + packet("A", pkA, "YVR") // pending in the coalescer, not yet flushed + c.Flush(ctx) + + rows, err := tx.Query(ctx, `SELECT o.id, COALESCE(o.last_iata, ''), COALESCE(( + SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1), '') +FROM observers o`) + if err != nil { + t.Fatal(err) + } + n := 0 + for rows.Next() { + var id uuid.UUID + var got, want string + if err := rows.Scan(&id, &got, &want); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("observer %s last_iata = %q, latest observation = %q", id, got, want) + } + n++ + } + if rows.Err() != nil || n != 3 { + t.Fatalf("observers = %d, err %v; want 3", n, rows.Err()) + } + + for name, want := range map[string]string{"A": "YVR", "B": "YYJ"} { + if got, err := store.GetObserverLastIATA(ctx, ids[name]); err != nil || got != want { + t.Errorf("GetObserverLastIATA(%s) = %q, %v; want %q", name, got, err, want) + } + } + var idC uuid.UUID + if err := tx.QueryRow(ctx, "SELECT id FROM observers WHERE public_key = $1", pkC).Scan(&idC); err != nil { + t.Fatal(err) + } + if got, err := store.GetObserverLastIATA(ctx, idC); err != nil || got != "" { + t.Errorf("GetObserverLastIATA(no packets) = %q, %v; want empty", got, err) + } + + page, err := store.ListObservers(ctx, []string{"YYJ", "YYZ"}, "", "", "", "", "", 0, 10) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].IATA != "YYJ" { + t.Errorf("ListObservers(YYJ,YYZ) = %+v; want only B in YYJ", page.Items) + } +} diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go new file mode 100644 index 00000000..96ec8a5c --- /dev/null +++ b/db/observer_metrics_integration_test.go @@ -0,0 +1,176 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "encoding/json" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "testing" + "time" +) + +// Hourly activity reads the rollup, but the last ~2h aren't rolled yet; they come from raw rows. +func TestObserverActivityUnrolledTailPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + now := time.Now().UTC() + rolled := now.Truncate(time.Hour).Add(-5 * time.Hour) + if _, err := tx.Exec(ctx, ` + INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); + INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,last_heard_at,first_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(1,4) i;`); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, ` + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) VALUES + (int4send(1),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz+interval '10 minutes',0,1,0,4,5,-90), + (int4send(2),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz+interval '20 minutes',0,1,0,4,7,-80);`, rolled); err != nil { + t.Fatal(err) + } + rollTxHours(t, ctx, tx) + // Heard since the newest rolled hour; the rollup hasn't reached them. + if _, err := tx.Exec(ctx, ` + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) VALUES + (int4send(3),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz-interval '100 minutes',0,1,0,4,9,-70), + (int4send(4),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz-interval '5 minutes',0,1,0,NULL,0,0);`, now); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + activity, err := store.GetObserverActivity(ctx, id, 24*time.Hour, time.Hour, time.Time{}) + if err != nil { + t.Fatal(err) + } + if activity.Summary.RecordedPackets != 4 { + t.Errorf("recorded packets = %d, want 4 (2 rolled + 2 unrolled)", activity.Summary.RecordedPackets) + } + byHour := map[int64]int64{} + for _, p := range activity.Points { + byHour[p.T] = p.Observations + } + recent := now.Add(-100 * time.Minute).Truncate(time.Hour).UnixMilli() + if byHour[rolled.UnixMilli()] != 2 || byHour[recent] != 1 || byHour[now.Add(-5*time.Minute).Truncate(time.Hour).UnixMilli()] != 1 { + t.Errorf("points %+v", activity.Points) + } + for _, p := range activity.Points { + if p.T == recent && (p.SNRAvg == nil || *p.SNRAvg != 9 || p.RSSIAvg == nil || *p.RSSIAvg != -70) { + t.Errorf("tail signal: %+v", p) + } + } + types := map[int16]int64{} + for _, v := range activity.PayloadTypes { + types[v.PayloadType] = v.Count + } + if types[4] != 3 || types[-1] != 1 { + t.Errorf("payload types %+v", activity.PayloadTypes) + } +} + +// A rollup backlog longer than the usual lag must still be read from raw rows. +func TestObserverActivityRollupBacklogPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + now := time.Now().UTC() + rolled := now.Truncate(time.Hour).Add(-10 * time.Hour) + if _, err := tx.Exec(ctx, ` + INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); + INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,last_heard_at,first_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(1,3) i;`); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, ` + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type) VALUES + (int4send(1),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz+interval '10 minutes',0,1,0,4);`, rolled); err != nil { + t.Fatal(err) + } + rollTxHours(t, ctx, tx) + if _, err := tx.Exec(ctx, ` + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type) VALUES + (int4send(2),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz-interval '8 hours',0,1,0,4), + (int4send(3),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz-interval '6 hours',0,1,0,4);`, now); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + activity, err := store.GetObserverActivity(ctx, id, 24*time.Hour, time.Hour, time.Time{}) + if err != nil { + t.Fatal(err) + } + if activity.Summary.RecordedPackets != 3 { + t.Errorf("recorded packets = %d, want 3 (1 rolled + 2 in the backlog)", activity.Summary.RecordedPackets) + } + wantRolled := rolled.Add(time.Hour).UnixMilli() + if activity.RolledUntil == nil || *activity.RolledUntil != wantRolled { + t.Errorf("rolledUntil = %v, want %d", activity.RolledUntil, wantRolled) + } + if activity.RawFrom == nil || *activity.RawFrom != wantRolled { + t.Errorf("rawFrom = %v, want %d (no gap)", activity.RawFrom, wantRolled) + } +} + +func TestObserverMetricsPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, ` + INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); + INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,last_heard_at,first_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash IN (int4send(1),int4send(2)) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; + `); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + // Hourly activity is rolled before cleanup and survives the raw rows. + rollTxHours(t, ctx, tx) + if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { + t.Fatal(err) + } + until := time.Now().UTC().Truncate(time.Hour) + activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until) + if err != nil { + t.Fatal(err) + } + if _, err := json.Marshal(activity); err != nil { + t.Fatalf("non-finite sample leaked into JSON: %v", err) + } + if activity.Summary.RecordedPackets != 3 || len(activity.PayloadTypes) != 2 || activity.Summary.LatestRecordedAt != nil || activity.Summary.LastCompleteHour != 0 { + t.Fatalf("archive summary: %+v types=%+v", activity.Summary, activity.PayloadTypes) + } + if activity.WindowEnd != until.UnixMilli() || activity.WindowStart != until.Add(-7*24*time.Hour).UnixMilli() || activity.Source != "hourly" { + t.Fatalf("window: %+v", activity) + } + if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) VALUES ('\xaa',4,0,1,'\x00','\x00',NOW(),NOW()); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,'NaN'::real,-100); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW(),0,1,0) ON CONFLICT DO NOTHING;`); err != nil { + t.Fatal(err) + } + raw, err := store.GetObserverActivity(ctx, id, time.Hour, 15*time.Minute, until) + if err != nil { + t.Fatal(err) + } + if _, err := json.Marshal(raw); err != nil { + t.Fatalf("raw non-finite sample leaked into JSON: %v", err) + } + if raw.Summary.RecordedPackets != 1 || raw.Summary.LastCompleteHour != 1 || raw.Summary.LatestRecordedAt == nil || raw.PayloadTypes[0].PayloadType != -1 { + t.Fatalf("raw/duplicate: %+v", raw) + } + if err := store.UpsertObserverBroker(ctx, id, "one", false); err != nil { + t.Fatal(err) + } + var packet pgtype.Timestamptz + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if packet.Valid { + t.Fatal("status created a packet timestamp") + } + if err := store.UpsertObserverBroker(ctx, id, "one", true); err != nil { + t.Fatal(err) + } + if err := store.TouchObserverBrokers(ctx, []uuid.UUID{id}, []string{"one"}, []time.Time{until.Add(time.Hour)}, []time.Time{{}}); err != nil { + t.Fatal(err) + } + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if !packet.Valid { + t.Fatal("status erased packet timestamp") + } +} diff --git a/db/observer_retention_integration_test.go b/db/observer_retention_integration_test.go new file mode 100644 index 00000000..1992c536 --- /dev/null +++ b/db/observer_retention_integration_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/presence" + "github.com/jackc/pgx/v5" +) + +func TestObserverRetentionPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + exec := func(query string, args ...any) { + t.Helper() + if _, err := tx.Exec(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + for _, table := range []string{"observers", "packet_observations", "observer_telemetry", "observer_owners", "observer_brokers", "observer_locations", "observer_scopes"} { + exec("CREATE TEMP TABLE " + table + " (LIKE public." + table + " INCLUDING ALL) ON COMMIT DROP") + if table != "observers" { + fk := "ALTER TABLE pg_temp." + table + " ADD FOREIGN KEY (observer_id) REFERENCES pg_temp.observers(id)" + if table != "packet_observations" { + fk += " ON DELETE CASCADE" + } + exec(fk) + } + } + store := &Store{q: sqlc.New(tx)} + cutoff := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) + exec(`INSERT INTO observers(public_key,display_name,last_seen,last_status_at) +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),label,$1::timestamptz+seen*interval '1 second', + $1::timestamptz+status*interval '1 second' +FROM (VALUES (1,'expired',-1,NULL),(2,'boundary',0,NULL),(3,'recent',1,NULL), + (4,'recent status',-1,1),(5,'status boundary',-1,0), + (6,'packet history',-1,NULL),(7,'telemetry',-1,NULL),(8,'owned',-1,NULL)) v(i,label,seen,status)`, cutoff) + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT 1,'\x01',id,'YYZ',last_seen,0,1,0 FROM observers WHERE display_name='packet history'; +INSERT INTO observer_telemetry(id,observer_id,reported_at) +SELECT 1,id,last_seen FROM observers WHERE display_name='telemetry'; +INSERT INTO observer_owners(observer_id,notes) +SELECT id,'fixture ownership without a node link' FROM observers WHERE display_name='owned'; +INSERT INTO observer_brokers(observer_id,broker_name) +SELECT id,'fixture' FROM observers WHERE display_name='expired'; +INSERT INTO observer_locations(observer_id,reported_at) +SELECT id,last_seen FROM observers WHERE display_name='expired'; +INSERT INTO observer_scopes(observer_id,scope_id) +SELECT id,1 FROM observers WHERE display_name='expired';`) + ids, err := store.DeleteOldObservers(ctx, cutoff) + if err != nil || len(ids) != 1 { + t.Fatalf("expected one unreferenced expired observer deleted, got %d: %v", len(ids), err) + } + var remaining string + if err := tx.QueryRow(ctx, "SELECT string_agg(display_name,',' ORDER BY display_name) FROM observers").Scan(&remaining); err != nil { + t.Fatal(err) + } + if remaining != "boundary,owned,packet history,recent,recent status,status boundary,telemetry" { + t.Fatalf("incorrect retained observers: %s", remaining) + } + var history, metadata int + if err := tx.QueryRow(ctx, `SELECT + (SELECT count(*) FROM packet_observations)+(SELECT count(*) FROM observer_telemetry)+(SELECT count(*) FROM observer_owners), + (SELECT count(*) FROM observer_brokers)+(SELECT count(*) FROM observer_locations)+(SELECT count(*) FROM observer_scopes)`).Scan(&history, &metadata); err != nil { + t.Fatal(err) + } + if history != 3 || metadata != 0 { + t.Fatalf("history=%d, cascading metadata=%d; want 3, 0", history, metadata) + } + if ids, err := store.DeleteOldObservers(ctx, cutoff); err != nil || len(ids) != 0 { + t.Fatalf("repeated cleanup changed protected rows: %v, %v", ids, err) + } + + // All relations below are temporary; no persistent fixtures or sequences are used. + exec("TRUNCATE pg_temp.observers CASCADE") + coalescer := presence.New(store, time.Second, time.Second) + pubkey := bytes.Repeat([]byte{42}, 32) + oldID, _, err := coalescer.UpsertObserver(ctx, pubkey, "YVR") + if err != nil { + t.Fatal(err) + } + if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture", true); err != nil { + t.Fatal(err) + } + // A future cutoff simulates this cached observer having aged past retention. + ids, err = coalescer.DeleteOldObservers(ctx, time.Now().Add(time.Hour)) + if err != nil || len(ids) != 1 || ids[0] != oldID { + t.Fatalf("cached expired observer was not deleted: %v, %v", ids, err) + } + newID, _, err := coalescer.UpsertObserver(ctx, pubkey, "YVR") + if err != nil || newID == oldID { + t.Fatalf("returning observer reused a deleted ID: %v", err) + } + if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture", true); err != nil { + t.Fatal(err) + } + var brokers int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM observer_brokers WHERE observer_id=$1", newID).Scan(&brokers); err != nil || brokers != 1 { + t.Fatalf("returning observer's broker was not recreated: count=%d, err=%v", brokers, err) + } + // The persisted row looks stale while fresh activity is still coalesced. + cutoff = time.Now().Add(-24 * time.Hour) + exec("UPDATE observers SET last_seen=$1 WHERE id=$2", cutoff.Add(-time.Hour), newID) + if _, _, err := coalescer.UpsertObserver(ctx, pubkey, "YVR"); err != nil { + t.Fatal(err) + } + if ids, err := coalescer.DeleteOldObservers(ctx, cutoff); err != nil || len(ids) != 0 { + t.Fatalf("pending presence did not protect a returning observer: %v, %v", ids, err) + } + var fresh bool + if err := tx.QueryRow(ctx, "SELECT last_seen >= $1 AND observation_count=1 FROM observers WHERE id=$2", cutoff, newID).Scan(&fresh); err != nil || !fresh { + t.Fatalf("pending presence was not persisted before cleanup: %v", err) + } + + exec("TRUNCATE pg_temp.observers CASCADE") + exec(`INSERT INTO observers(public_key,last_seen) +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),$1::timestamptz-interval '1 second' +FROM generate_series(1,1005) i`, cutoff) + for _, want := range []int{1000, 5, 0} { + ids, err := store.DeleteOldObservers(ctx, cutoff) + if err != nil || len(ids) != want { + t.Fatalf("bounded cleanup removed %d observers, want %d: %v", len(ids), want, err) + } + t.Logf("bounded cleanup deleted %d observers", len(ids)) + } +} diff --git a/db/observer_window_test.go b/db/observer_window_test.go new file mode 100644 index 00000000..03f920ae --- /dev/null +++ b/db/observer_window_test.go @@ -0,0 +1,57 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "go.uber.org/mock/gomock" +) + +func TestObserverActivityLiveAndFixedWindow(t *testing.T) { + for _, interval := range []time.Duration{15 * time.Minute, time.Hour, 6 * time.Hour} { + for _, fixed := range []bool{false, true} { + t.Run(interval.String()+"/"+map[bool]string{false: "live", true: "fixed"}[fixed], func(t *testing.T) { + mock := mockdb.NewMockQuerier(gomock.NewController(t)) + id := uuid.New() + mock.EXPECT().GetObserverByID(gomock.Any(), id).Return(sqlc.Observer{}, nil) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil) + if interval < time.Hour { + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()).Return(nil, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) + } else { + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()).Return(nil, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) + } + before := time.Now().UTC() + var until time.Time + if fixed { + until = before.Add(-24*time.Hour + 123*time.Millisecond) + } + got, err := (&Store{q: mock}).GetObserverActivity(context.Background(), id, 48*time.Hour, interval, until) + if err != nil { + t.Fatal(err) + } + if fixed { + if got.WindowEnd != until.Truncate(interval).UnixMilli() { + t.Fatalf("fixed end was not aligned: %d", got.WindowEnd) + } + } else if got.WindowEnd < before.UnixMilli() || got.WindowEnd > time.Now().UnixMilli() { + t.Fatalf("live end dropped the partial bucket: %d", got.WindowEnd) + } + // Freshness is always live, even when the activity window is historical. + if got.Summary.LastCompleteHourEnd != time.UnixMilli(got.GeneratedAt).UTC().Truncate(time.Hour).UnixMilli() { + t.Fatal("freshness was anchored to the historical activity window") + } + }) + } + } +} diff --git a/db/observers.go b/db/observers.go index 5ff37060..a458560b 100644 --- a/db/observers.go +++ b/db/observers.go @@ -6,19 +6,22 @@ package db import ( "context" "encoding/hex" + "encoding/json" "fmt" - "log" + "log/slog" + "math" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/MeshCore-Beacon/beacon-server/internal/lora" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) -func (s *Store) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) { - row, err := s.q.UpsertObserver(ctx, pubkey) +func (s *Store) UpsertObserver(ctx context.Context, pubkey []byte, iata string) (uuid.UUID, string, error) { + row, err := s.q.UpsertObserver(ctx, sqlc.UpsertObserverParams{PublicKey: pubkey, Iata: iata, IataAt: ts(time.Now())}) if err != nil { return uuid.Nil, "", err } @@ -114,14 +117,14 @@ func (s *Store) GetObserver(ctx context.Context, observerID uuid.UUID) (*api.Obs RadioCR: obs.RadioCr, BatteryLevel: obs.BatteryLevel, UptimeSeconds: obs.UptimeSeconds, - StatusMetadata: obs.StatusMetadata, + StatusMetadata: json.RawMessage(obs.StatusMetadata), FirstSeen: obs.FirstSeen.Time.UnixMilli(), LastSeen: obs.LastSeen.Time.UnixMilli(), ObservationCount: *obs.ObservationCount, } scopes, err := s.GetObserverScopes(ctx, observerID) if err != nil { - log.Printf("store: GetObserverScopes failed for %s: %v", observerID, err) + slog.Error(fmt.Sprintf("store: GetObserverScopes failed for %s", observerID), "component", "db", "error", err) scopes = []string{} } observer.Scopes = scopes @@ -157,8 +160,8 @@ func (s *Store) InsertObserverTelemetry(ctx context.Context, observerID uuid.UUI ObserverID: observerID, ReportedAt: pgtype.Timestamptz{Time: reportedAt, Valid: true}, BatteryVoltageMv: batteryMV, - AirtimeTxPct: txAirSecs, - AirtimeRxPct: rxAirSecs, + AirtimeTxSecs: txAirSecs, + AirtimeRxSecs: rxAirSecs, NoiseFloorDb: &noiseFloor, UptimeSeconds: &uptimeSeconds, QueueLength: queueLen, @@ -182,8 +185,8 @@ func (s *Store) GetObserverTelemetry(ctx context.Context, observerID uuid.UUID, points = append(points, api.ObserverTelemetryPoint{ T: v.ReportedAt.Time.UnixMilli(), BatteryMV: v.BatteryVoltageMv, - AirtimeTxPct: v.AirtimeTxPct, - AirtimeRxPct: v.AirtimeRxPct, + AirtimeTxSecs: v.AirtimeTxSecs, + AirtimeRxSecs: v.AirtimeRxSecs, NoiseFloorDB: v.NoiseFloorDb, UptimeSeconds: v.UptimeSeconds, QueueLength: v.QueueLength, @@ -193,37 +196,197 @@ func (s *Store) GetObserverTelemetry(ctx context.Context, observerID uuid.UUID, return &api.ObserverTelemetry{Points: points}, nil } +// GetObserverTelemetryBucketed reads a day before since so the first bucket's counter increments have a baseline. func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]api.ObserverTelemetryPoint, error) { - var sinceTS, untilTS pgtype.Timestamptz - if !since.IsZero() { - sinceTS = pgtype.Timestamptz{Time: since, Valid: true} + from := since + if !from.IsZero() { + from = from.Add(-24 * time.Hour) } - if !until.IsZero() { - untilTS = pgtype.Timestamptz{Time: until, Valid: true} + raw, err := s.GetObserverTelemetry(ctx, observerID, from, until, 0) + if err != nil { + return nil, err + } + return api.BucketTelemetry(raw.Points, since, time.Duration(bucketHours)*time.Hour), nil +} + +// activityRawTail caps how far back hourly activity reads raw rows for hours not yet rolled. +// Cleanup holds raw rows for unrolled hours up to the same 24h (background rawHoldbackCap). +const activityRawTail = 24 * time.Hour + +// GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. +// Buckets of an hour or coarser come from the hourly rollup plus raw rows for the unrolled tail; +// anything finer reads observations directly. +// Range and Interval are left empty for the handler to fill. +func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { + obs, err := s.q.GetObserverByID(ctx, observerID) + if err != nil { + return nil, err + } + now := time.Now().UTC() + if until.IsZero() { + until = now + } else { + until = until.UTC().Truncate(interval) + } + since := until.Add(-window) + if !since.Equal(since.Truncate(interval)) { + since = since.Truncate(interval).Add(interval) + } + activity := &api.ObserverActivity{WindowStart: since.UnixMilli(), WindowEnd: until.UnixMilli(), GeneratedAt: now.UnixMilli(), Source: "raw", Summary: &api.ObserverActivitySummary{}} + if interval >= time.Hour { + activity.Source = "hourly" } - rows, err := s.q.GetObserverTelemetryBucketed(ctx, sqlc.GetObserverTelemetryBucketedParams{ + live, err := s.q.GetObserverActivityLiveSummary(ctx, sqlc.GetObserverActivityLiveSummaryParams{ + ObserverID: observerID, GeneratedAt: pgtype.Timestamptz{Time: now, Valid: true}, + HourStart: pgtype.Timestamptz{Time: now.Truncate(time.Hour).Add(-time.Hour), Valid: true}, HourEnd: pgtype.Timestamptz{Time: now.Truncate(time.Hour), Valid: true}}) + if err != nil { + return nil, err + } + activity.Summary.LastCompleteHour = live.LastCompleteHour + if live.LatestRecordedAt.Valid { + v := live.LatestRecordedAt.Time.UnixMilli() + activity.Summary.LatestRecordedAt = &v + } + activity.Summary.LastCompleteHourStart = now.Truncate(time.Hour).Add(-time.Hour).UnixMilli() + activity.Summary.LastCompleteHourEnd = now.Truncate(time.Hour).UnixMilli() + // radio is non-nil only when airtime is actually costable, so radio != null implies costed buckets + if obs.RadioSf != nil && obs.RadioBwKhz != nil && obs.RadioCr != nil && + *obs.RadioSf >= 7 && *obs.RadioSf <= 12 && *obs.RadioBwKhz > 0 && *obs.RadioCr > 0 { + activity.Radio = &api.ObserverActivityRadio{ + FreqMHz: obs.RadioFreqMhz, + SF: *obs.RadioSf, + BWKHz: *obs.RadioBwKhz, + CR: *obs.RadioCr, + PreambleSymbols: lora.PreambleSymbols(int(*obs.RadioSf)), + } + } + sinceTS := pgtype.Timestamptz{Time: since, Valid: true} + binWidth := pgtype.Interval{Microseconds: interval.Microseconds(), Valid: true} + + if interval >= time.Hour { + // Unrolled hours read raw rows from the rollup watermark, at most activityRawTail back. + watermark, err := s.q.GetLatestCompleteRollupHour(ctx) + if err != nil { + return nil, err + } + tailStart := until.Add(-activityRawTail) + if watermark.Valid { + rolledUntil := watermark.Time.Add(time.Hour) + activity.RolledUntil = new(rolledUntil.UnixMilli()) + if rolledUntil.After(tailStart) { + tailStart = rolledUntil + } + } + activity.RawFrom = new(tailStart.UnixMilli()) + tailStartTS := pgtype.Timestamptz{Time: tailStart, Valid: true} + rows, err := s.q.GetObserverActivityHourly(ctx, sqlc.GetObserverActivityHourlyParams{ + ObserverID: observerID, + Column2: sinceTS, + Column3: binWidth, + TailStart: tailStartTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, + }) + if err != nil { + return nil, err + } + activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) + for _, r := range rows { + p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations + if r.AirtimeN > 0 { + airtime := r.AirtimeMs + p.AirtimeMs = &airtime + } + if r.SnrN > 0 { + avg := r.SnrSum / float32(r.SnrN) + min := r.SnrMin + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } + } + if r.RssiN > 0 { + avg := float32(r.RssiSum) / float32(r.RssiN) + p.RSSIAvg = &avg + } + activity.Points = append(activity.Points, p) + } + typeRows, err := s.q.GetObserverActivityHourlyPayloadTypes(ctx, sqlc.GetObserverActivityHourlyPayloadTypesParams{ + ObserverID: observerID, + Column2: sinceTS, + TailStart: tailStartTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, + }) + if err != nil { + return nil, err + } + activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) + for _, v := range typeRows { + payloadType := v.PayloadType + activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), + Count: v.Count, + }) + } + return activity, nil + } + + rows, err := s.q.GetObserverActivityRaw(ctx, sqlc.GetObserverActivityRawParams{ ObserverID: observerID, Column2: sinceTS, - Column3: untilTS, - Column4: bucketHours, + Column3: binWidth, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err } - points := make([]api.ObserverTelemetryPoint, 0, len(rows)) + activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) for _, r := range rows { - points = append(points, api.ObserverTelemetryPoint{ - T: r.Bucket.Time.UnixMilli(), - BatteryMV: &r.BatteryVoltageMv, - AirtimeTxPct: &r.AirtimeTxPct, - AirtimeRxPct: &r.AirtimeRxPct, - NoiseFloorDB: &r.NoiseFloorDb, - UptimeSeconds: &r.UptimeSeconds, - QueueLength: &r.QueueLength, - ReceiveErrors: &r.ReceiveErrors, + p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations + if r.AirtimeN > 0 { + airtime := r.AirtimeMs + p.AirtimeMs = &airtime + } + if r.SnrN > 0 { + avg, min := r.SnrAvg, r.SnrMin + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } + } + if r.RssiN > 0 { + avg := r.RssiAvg + p.RSSIAvg = &avg + } + activity.Points = append(activity.Points, p) + } + typeRows, err := s.q.GetObserverActivityRawPayloadTypes(ctx, sqlc.GetObserverActivityRawPayloadTypesParams{ + ObserverID: observerID, + Column2: sinceTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, + }) + if err != nil { + return nil, err + } + activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) + for _, v := range typeRows { + payloadType := int16(-1) + if v.PayloadType != nil { + payloadType = *v.PayloadType + } + activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), + Count: v.Count, }) } - return points, nil + return activity, nil } func (s *Store) ListObserverAdverts(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) { @@ -233,7 +396,7 @@ func (s *Store) ListObserverAdverts(ctx context.Context, observerID uuid.UUID, c Limit: limit + 1, // fetch one extra to detect hasMore }) if err != nil { - log.Printf("api: ListObserverAdverts failed: %v", err) + slog.Error("api: ListObserverAdverts failed", "component", "db", "error", err) return api.Page[api.AdvertObservation]{}, err } hasMore := len(rows) > int(limit) @@ -300,10 +463,11 @@ func (s *Store) GetObserverRadio(ctx context.Context, observerID uuid.UUID) (ing return settings, nil } -func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { params := sqlc.UpsertObserverBrokerParams{ ObserverID: observerID, BrokerName: brokerName, + IsPacket: isPacket, } return s.q.UpsertObserverBroker(ctx, params) } @@ -315,6 +479,17 @@ func (s *Store) UpsertObserverScope(ctx context.Context, observerID uuid.UUID, s }) } +// UpdateObserverRegionScope records the observer's own OTA-reported region scope +// (the "self" field of a /neighbors report). Unrelated to UpsertObserverScope +// above, which links an observer to a named transport_scopes row matched by +// key fingerprint; this stores the raw OTA-configured scope string instead. +func (s *Store) UpdateObserverRegionScope(ctx context.Context, observerID uuid.UUID, regionScope string) error { + return s.q.UpdateObserverRegionScope(ctx, sqlc.UpdateObserverRegionScopeParams{ + ID: observerID, + RegionScope: ®ionScope, + }) +} + func (s *Store) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) { return s.q.GetObserverScopes(ctx, observerID) } @@ -327,3 +502,7 @@ func (s *Store) IsObserverByPubkey(ctx context.Context, pubkey []byte) bool { func (s *Store) DeleteOldTelemetry(ctx context.Context, cutoff time.Time) error { return s.q.DeleteOldTelemetry(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } + +func (s *Store) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { + return s.q.DeleteOldObservers(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) +} diff --git a/db/observers_test.go b/db/observers_test.go index f6f7572a..0943a9a3 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -5,13 +5,17 @@ package db import ( "context" + "encoding/json" "errors" + "strings" "testing" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "go.uber.org/mock/gomock" ) @@ -24,11 +28,13 @@ func TestUpsertObserver_NilDisplayName(t *testing.T) { pubkey := []byte{0x01, 0x02} mock.EXPECT(). - UpsertObserver(gomock.Any(), pubkey). + UpsertObserver(gomock.Any(), gomock.Cond(func(p sqlc.UpsertObserverParams) bool { + return string(p.PublicKey) == string(pubkey) && p.Iata == "YVR" && p.IataAt.Valid + })). Return(sqlc.Observer{ID: observerID, DisplayName: nil}, nil) store := &Store{q: mock} - id, displayName, err := store.UpsertObserver(context.Background(), pubkey) + id, displayName, err := store.UpsertObserver(context.Background(), pubkey, "YVR") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -49,11 +55,13 @@ func TestUpsertObserver_WithDisplayName(t *testing.T) { name := "test-observer" mock.EXPECT(). - UpsertObserver(gomock.Any(), pubkey). + UpsertObserver(gomock.Any(), gomock.Cond(func(p sqlc.UpsertObserverParams) bool { + return string(p.PublicKey) == string(pubkey) && p.Iata == "YVR" && p.IataAt.Valid + })). Return(sqlc.Observer{ID: observerID, DisplayName: &name}, nil) store := &Store{q: mock} - _, displayName, err := store.UpsertObserver(context.Background(), pubkey) + _, displayName, err := store.UpsertObserver(context.Background(), pubkey, "YVR") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -176,6 +184,7 @@ func TestGetObserver_OnlineStatus(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") obsCount := int64(10) + statusMetadata := []byte(`{"stats":{"noise_floor":-97}}`) mock.EXPECT(). GetObserverByID(gomock.Any(), observerID). @@ -186,6 +195,7 @@ func TestGetObserver_OnlineStatus(t *testing.T) { FirstSeen: pgtype.Timestamptz{Time: time.Now().Add(-time.Hour), Valid: true}, LastSeen: pgtype.Timestamptz{Time: time.Now().Add(-time.Minute), Valid: true}, LastStatusAt: pgtype.Timestamptz{Time: time.Now().Add(-time.Minute), Valid: true}, + StatusMetadata: statusMetadata, }, nil) mock.EXPECT(). @@ -211,6 +221,14 @@ func TestGetObserver_OnlineStatus(t *testing.T) { if observer.IATA != "YVR" { t.Errorf("expected IATA YVR, got %s", observer.IATA) } + + out, err := json.Marshal(observer) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), `"statusMetadata":{"stats":{"noise_floor":-97}}`) { + t.Fatalf("statusMetadata not an object: %s", out) + } } func TestGetObserver_OfflineStatus(t *testing.T) { @@ -380,32 +398,39 @@ func TestGetObserverTelemetryBucketed_Mapping(t *testing.T) { mock := mockdb.NewMockQuerier(ctrl) observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") - bucket := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} + since := time.Date(2026, 9, 30, 6, 0, 0, 0, time.UTC) + at := func(h int) pgtype.Timestamptz { + return pgtype.Timestamptz{Time: since.Add(time.Duration(h) * time.Hour), Valid: true} + } + f := func(v float32) *float32 { return &v } + up := func(v int64) *int64 { return &v } + bat := int32(3700) mock.EXPECT(). - GetObserverTelemetryBucketed(gomock.Any(), gomock.Any()). - Return([]sqlc.GetObserverTelemetryBucketedRow{ - { - Bucket: bucket, - BatteryVoltageMv: 3700, - NoiseFloorDb: -90.0, - UptimeSeconds: 3600, - }, - }, nil) + GetObserverTelemetry(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, p sqlc.GetObserverTelemetryParams) ([]sqlc.GetObserverTelemetryRow, error) { + if want := since.Add(-24 * time.Hour); !p.Column2.Time.Equal(want) { + t.Errorf("since = %v, want lookback to %v", p.Column2.Time, want) + } + return []sqlc.GetObserverTelemetryRow{ + {ReportedAt: at(-1), AirtimeTxSecs: f(100), UptimeSeconds: up(10000)}, + {ReportedAt: at(0), AirtimeTxSecs: f(110), UptimeSeconds: up(13600), BatteryVoltageMv: &bat}, + }, nil + }) store := &Store{q: mock} - points, err := store.GetObserverTelemetryBucketed(context.Background(), observerID, time.Time{}, time.Time{}, 1) + points, err := store.GetObserverTelemetryBucketed(context.Background(), observerID, since, time.Time{}, 6) if err != nil { t.Fatalf("unexpected error: %v", err) } if len(points) != 1 { t.Fatalf("expected 1 point, got %d", len(points)) } - if points[0].T != 1700000000000 { - t.Errorf("expected T 1700000000000, got %d", points[0].T) + if points[0].T != since.UnixMilli() { + t.Errorf("expected T %d, got %d", since.UnixMilli(), points[0].T) } - if *points[0].BatteryMV != 3700 { - t.Errorf("expected BatteryMV 3700, got %d", *points[0].BatteryMV) + if *points[0].AirtimeTxSecs != 10 || *points[0].BatteryMV != 3700 { + t.Errorf("tx %v battery %v, want 10 3700", *points[0].AirtimeTxSecs, *points[0].BatteryMV) } } @@ -550,3 +575,382 @@ func TestIsObserverByPubkey_NotFound(t *testing.T) { t.Error("expected false for missing observer") } } + +func activityObserver(sf, cr *int16, bw, freq *float32) sqlc.Observer { + return sqlc.Observer{ + ID: uuid.MustParse("00000000-0000-0000-0000-000000000001"), + RadioFreqMhz: freq, + RadioSf: sf, + RadioBwKhz: bw, + RadioCr: cr, + } +} + +func i16(v int16) *int16 { return &v } +func f32(v float32) *float32 { return &v } + +func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), f32(910.525)), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyRow{{ + Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, + Observations: 12, + AirtimeMs: 0, + AirtimeN: 0, + SnrSum: 0, + SnrN: 0, + SnrMin: 0, + RssiSum: 0, + RssiN: 0, + }}, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{ + {PayloadType: 4, Count: 9}, + {PayloadType: -1, Count: 3}, + }, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got.Points) != 1 { + t.Fatalf("expected 1 point, got %d", len(got.Points)) + } + p := got.Points[0] + if p.T != 1700000000000 { + t.Errorf("expected T 1700000000000, got %d", p.T) + } + if p.Observations != 12 { + t.Errorf("expected 12 observations, got %d", p.Observations) + } + if p.AirtimeMs != nil { + t.Errorf("expected nil AirtimeMs, got %v", *p.AirtimeMs) + } + if p.SNRAvg != nil || p.SNRMin != nil { + t.Errorf("expected nil SNR fields, got %v %v", p.SNRAvg, p.SNRMin) + } + if p.RSSIAvg != nil { + t.Errorf("expected nil RSSIAvg, got %v", *p.RSSIAvg) + } + if len(got.PayloadTypes) != 2 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 || got.PayloadTypes[1].PayloadType != -1 || got.PayloadTypes[1].Count != 3 { + t.Fatalf("expected known and unknown payloads to reconcile, got %+v", got.PayloadTypes) + } + if got.PayloadTypes[0].PayloadTypeName != api.PayloadTypeName(4) { + t.Errorf("expected payload type name %q, got %q", api.PayloadTypeName(4), got.PayloadTypes[0].PayloadTypeName) + } + if got.Radio == nil { + t.Fatal("expected radio") + } + if got.Radio.SF != 10 || got.Radio.BWKHz != 62.5 || got.Radio.CR != 5 || got.Radio.PreambleSymbols != 16 { + t.Errorf("unexpected radio %+v", *got.Radio) + } + if got.Radio.FreqMHz == nil || *got.Radio.FreqMHz != 910.525 { + t.Errorf("unexpected freq %v", got.Radio.FreqMHz) + } + if got.Range != "" || got.Interval != "" { + t.Errorf("expected range/interval left for the handler, got %q %q", got.Range, got.Interval) + } +} + +func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(7), i16(5), f32(250), nil), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyRow{{ + Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, + Observations: 8, + AirtimeMs: 123.5, + AirtimeN: 3, + SnrSum: 30, + SnrN: 4, + SnrMin: -3.5, + RssiSum: -400, + RssiN: 4, + }}, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour, time.Time{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + p := got.Points[0] + if p.AirtimeMs == nil || *p.AirtimeMs != 123.5 { + t.Errorf("expected AirtimeMs 123.5, got %v", p.AirtimeMs) + } + if p.SNRAvg == nil || *p.SNRAvg != 7.5 { + t.Errorf("expected SNRAvg 7.5, got %v", p.SNRAvg) + } + if p.SNRMin == nil || *p.SNRMin != -3.5 { + t.Errorf("expected SNRMin -3.5, got %v", p.SNRMin) + } + if p.RSSIAvg == nil || *p.RSSIAvg != -100 { + t.Errorf("expected RSSIAvg -100, got %v", p.RSSIAvg) + } + if got.PayloadTypes == nil { + t.Error("expected non-nil PayloadTypes slice") + } + if got.Radio.FreqMHz != nil { + t.Errorf("expected nil freq, got %v", *got.Radio.FreqMHz) + } + if got.Radio.PreambleSymbols != 32 { + t.Errorf("expected 32 preamble symbols at SF7, got %d", got.Radio.PreambleSymbols) + } +} + +func TestGetObserverActivity_RawPath(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawRow{{ + Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, + Observations: 5, + AirtimeMs: 50, + AirtimeN: 5, + SnrAvg: 4.25, + SnrMin: 1.5, + SnrN: 2, + RssiAvg: -95.5, + RssiN: 0, + }}, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawPayloadTypesRow{{PayloadType: i16(1), Count: 5}}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + p := got.Points[0] + if p.Observations != 5 { + t.Errorf("expected 5 observations, got %d", p.Observations) + } + if p.AirtimeMs == nil || *p.AirtimeMs != 50 { + t.Errorf("expected AirtimeMs 50, got %v", p.AirtimeMs) + } + if p.SNRAvg == nil || *p.SNRAvg != 4.25 { + t.Errorf("expected SNRAvg 4.25, got %v", p.SNRAvg) + } + if p.SNRMin == nil || *p.SNRMin != 1.5 { + t.Errorf("expected SNRMin 1.5, got %v", p.SNRMin) + } + if p.RSSIAvg != nil { + t.Errorf("expected nil RSSIAvg with rssi_n 0, got %v", *p.RSSIAvg) + } + if len(got.PayloadTypes) != 1 || got.PayloadTypes[0].Count != 5 { + t.Errorf("unexpected payload types %+v", got.PayloadTypes) + } +} + +func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawRow{{ + Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, + Observations: 7, + AirtimeMs: 0, + AirtimeN: 0, + SnrAvg: 0, + SnrMin: 0, + SnrN: 0, + RssiAvg: 0, + RssiN: 0, + }}, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + p := got.Points[0] + if p.Observations != 7 { + t.Errorf("expected 7 observations, got %d", p.Observations) + } + if p.AirtimeMs != nil { + t.Errorf("expected nil AirtimeMs with airtime_n 0, got %v", *p.AirtimeMs) + } + if p.SNRAvg != nil || p.SNRMin != nil { + t.Errorf("expected nil SNR fields with snr_n 0, got %v %v", p.SNRAvg, p.SNRMin) + } + if p.RSSIAvg != nil { + t.Errorf("expected nil RSSIAvg with rssi_n 0, got %v", *p.RSSIAvg) + } +} + +func TestGetObserverActivity_RawPathEmpty(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawRow{}, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Points == nil || len(got.Points) != 0 { + t.Errorf("expected empty non-nil points, got %#v", got.Points) + } + if got.PayloadTypes == nil || len(got.PayloadTypes) != 0 { + t.Errorf("expected empty non-nil payload types, got %#v", got.PayloadTypes) + } +} + +func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyRow{}, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Points == nil || len(got.Points) != 0 { + t.Errorf("expected empty non-nil points, got %#v", got.Points) + } +} + +func TestGetObserverActivity_UnknownObserver(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000002") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(sqlc.Observer{}, pgx.ErrNoRows) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour, time.Time{}) + if !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("expected pgx.ErrNoRows, got %v", err) + } + if got != nil { + t.Errorf("expected nil result, got %+v", got) + } +} + +func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { + cases := []struct { + name string + sf, cr *int16 + bw *float32 + }{ + {name: "sf nil", sf: nil, cr: i16(5), bw: f32(62.5)}, + {name: "sf zero", sf: i16(0), cr: i16(5), bw: f32(62.5)}, + {name: "sf below lora range", sf: i16(6), cr: i16(5), bw: f32(62.5)}, + {name: "sf above lora range", sf: i16(13), cr: i16(5), bw: f32(62.5)}, + {name: "bw zero", sf: i16(10), cr: i16(5), bw: f32(0)}, + {name: "cr nil", sf: i16(10), cr: nil, bw: f32(62.5)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(tc.sf, tc.cr, tc.bw, nil), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyRow{}, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Radio != nil { + t.Errorf("expected nil radio, got %+v", *got.Radio) + } + }) + } +} + +func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + window, interval := 24*time.Hour, 6*time.Hour + + var gotSince time.Time + var gotInterval pgtype.Interval + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, arg sqlc.GetObserverActivityHourlyParams) ([]sqlc.GetObserverActivityHourlyRow, error) { + gotSince, gotInterval = arg.Column2.Time, arg.Column3 + return nil, nil + }) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) + + before := time.Now().UTC().Truncate(interval).Add(-window) + store := &Store{q: mock} + if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval, time.Time{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + after := time.Now().UTC().Truncate(interval).Add(-window) + + if gotSince.UnixNano()%int64(interval) != 0 { + t.Errorf("since %s is not aligned to %s", gotSince, interval) + } + if gotSince.Before(before) { + t.Errorf("since %s is before the window start %s", gotSince, before) + } + if gotSince.After(after.Add(interval)) { + t.Errorf("since %s is more than one interval past the window start %s", gotSince, after) + } + if !gotInterval.Valid || gotInterval.Microseconds != interval.Microseconds() { + t.Errorf("expected interval %d us, got %+v", interval.Microseconds(), gotInterval) + } +} diff --git a/db/packet_cleanup_integration_test.go b/db/packet_cleanup_integration_test.go new file mode 100644 index 00000000..eb3815f8 --- /dev/null +++ b/db/packet_cleanup_integration_test.go @@ -0,0 +1,205 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +// Raw deletion and the rollup meet at raw_deleted_before: an hour registered but not rolled +// before its raw rows go becomes partial, while hours behind the holdback roll completely. +func TestPacketCleanupRollupHandoffPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + old := h0.Add(-3 * time.Hour).Add(10 * time.Minute) + if _, err := pool.Exec(ctx, ` +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ('\xff', 2, 0, 1, '\x00', '\x00', $1::timestamptz, $1::timestamptz); +INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +VALUES (100, '\xff', '00000000-0000-0000-0000-00000000000a', 'YVR', $1::timestamptz, 0, 1, 0)`, pgx.QueryExecModeSimpleProtocol, old); err != nil { + t.Fatal(err) + } + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0.Add(-10*time.Hour)); err != nil { + t.Fatal(err) + } + oldest, ok, err := store.OldestMissingRollupHour(ctx) + if err != nil || !ok || !oldest.Equal(old.Truncate(time.Hour)) { + t.Fatalf("oldest missing %s %v %v, want %s", oldest, ok, err, old.Truncate(time.Hour)) + } + + // Cleanup held behind h0 (as rawCutoff does when h0 is the oldest unrolled hour) leaves h0 intact. + if err := store.DeleteOldPackets(ctx, h0.Add(-35*time.Minute)); err != nil { + t.Fatal(err) + } + var deletedBefore time.Time + var remaining int + if err := pool.QueryRow(ctx, `SELECT raw_deleted_before, (SELECT count(*) FROM packet_observations) + FROM analytics_raw_state`).Scan(&deletedBefore, &remaining); err != nil { + t.Fatal(err) + } + if !deletedBefore.Equal(old) || remaining != 10 { + t.Fatalf("raw_deleted_before %s, %d observations left; want %s, 10", deletedBefore, remaining, old) + } + + if err := r.MarkPartialHours(ctx); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + var statuses string + if err := pool.QueryRow(ctx, `SELECT string_agg(status, ',' ORDER BY hour) FROM analytics_rollup_hours + WHERE hour BETWEEN $1 AND $2`, old.Truncate(time.Hour), h0.Add(time.Hour)).Scan(&statuses); err != nil { + t.Fatal(err) + } + if statuses != "partial,complete,complete,complete,complete" { + t.Errorf("statuses %q, want the deleted hour partial and the rest complete", statuses) + } + q := rollupOracles["iata observations"] + if got, want := rowsText(t, ctx, pool, q[0]), rowsText(t, ctx, pool, q[1]); got != want { + t.Errorf("held-back hours lost raw rows:\nrollup:\n%s\nraw:\n%s", got, want) + } + + if err := store.DeleteOldRollups(ctx, h0.Add(time.Hour)); err != nil { + t.Fatal(err) + } + var left int + if err := pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM analytics_rollup_hours WHERE hour < $1) + + (SELECT count(*) FROM analytics_hourly_iata_observations WHERE hour < $1)`, h0.Add(time.Hour)).Scan(&left); err != nil || left != 0 { + t.Errorf("rollup rows before the cutoff: %d, %v", left, err) + } +} + +// An in-flight observation insert holds KEY SHARE on its packet; cleanup skips it, and a +// second cleanup runner skips the first one's cohort instead of waiting. +func TestDeleteOldPacketsSkipsLockedPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YVR'); +INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x01'); +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ('\x01', 4, 0, 1, '\x00', '\x00', now() - INTERVAL '5 days', now() - INTERVAL '5 days')`); err != nil { + t.Fatal(err) + } + params := sqlc.DeleteOldPacketsParams{Cutoff: ts(time.Now().Add(-72 * time.Hour)), BatchSize: 1000} + q := sqlc.New(pool) + + ingest, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer ingest.Rollback(context.Background()) + if _, err := ingest.Exec(ctx, `INSERT INTO packet_observations (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +VALUES ('\x01', '00000000-0000-0000-0000-000000000001', 'YVR', now() - INTERVAL '5 days', 0, 1, 0)`); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("must skip in-flight reception: %d %v", n, err) + } + if err := ingest.Commit(ctx); err != nil { + t.Fatal(err) + } + + first, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer first.Rollback(context.Background()) + if n, err := sqlc.New(first).DeleteOldPackets(ctx, params); err != nil || n != 1 { + t.Fatalf("first runner: %d %v", n, err) + } + waitCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + if n, err := q.DeleteOldPackets(waitCtx, params); err != nil || n != 0 { + t.Fatalf("second runner should skip the locked cohort: %d %v", n, err) + } + if err := first.Commit(ctx); err != nil { + t.Fatal(err) + } + var left int + if err := pool.QueryRow(ctx, "SELECT (SELECT count(*) FROM packets) + (SELECT count(*) FROM packet_observations)").Scan(&left); err != nil || left != 0 { + t.Errorf("rows left after cleanup: %d %v", left, err) + } +} + +// Registration from an hour-aligned rollup cutoff never recreates hours cleanup just expired. +func TestRollupRetentionCyclePostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + cutoff := h0.Add(30 * time.Minute) // DeleteOldRollups cutoff; registration starts at the next hour + since := h0.Add(time.Hour) + for range 3 { + if err := store.DeleteOldRollups(ctx, cutoff); err != nil { + t.Fatal(err) + } + if err := r.RegisterHours(ctx, since); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + } + var stale int + var first time.Time + if err := pool.QueryRow(ctx, "SELECT count(*) FILTER (WHERE hour < $1), min(hour) FROM analytics_rollup_hours", cutoff).Scan(&stale, &first); err != nil { + t.Fatal(err) + } + if stale != 0 || !first.Equal(since) { + t.Errorf("%d expired hours re-registered, first hour %s; want 0, %s", stale, first, since) + } +} + +// Retention deletes and partial marking change what readers see, so they must move the +// revision cache keys are built from, even with no new roll. +func TestRollupCoverageMovesRevisionPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete { + t.Fatalf("roll: %d %v", outcome, err) + } + step := func(name string, change bool, do func() error) { + t.Helper() + before := revision(t, ctx, store) + if err := do(); err != nil { + t.Fatal(err) + } + if moved := revision(t, ctx, store) != before; moved != change { + t.Errorf("%s: revision moved=%v, want %v", name, moved, change) + } + } + step("no-op cleanup", false, func() error { return store.DeleteOldRollups(ctx, h0) }) + step("partial marking", true, func() error { + if _, err := pool.Exec(ctx, "UPDATE analytics_raw_state SET raw_deleted_before = $1", h0.Add(90*time.Minute)); err != nil { + return err + } + return r.MarkPartialHours(ctx) + }) + step("repeat partial marking", false, func() error { return r.MarkPartialHours(ctx) }) + step("retention delete", true, func() error { return store.DeleteOldRollups(ctx, h0.Add(time.Hour)) }) +} diff --git a/db/packet_endpoints_integration_test.go b/db/packet_endpoints_integration_test.go new file mode 100644 index 00000000..50d2c0bb --- /dev/null +++ b/db/packet_endpoints_integration_test.go @@ -0,0 +1,166 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "crypto/rand" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type endpointQueryCounter struct { + sqlc.DBTX + calls int +} + +func (q *endpointQueryCounter) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + q.calls++ + return q.DBTX.Query(ctx, sql, args...) +} +func (q *endpointQueryCounter) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { + q.calls++ + return q.DBTX.QueryRow(ctx, sql, args...) +} + +// Endpoints resolve live against today's nodes, in a fixed number of queries per page. +func TestPacketEndpointsResolveLivePostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + // Own throwaway database so CI can run it against an empty server. + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer admin.Close(context.Background()) + name := "beacon_endpoints_" + strings.ToLower(rand.Text()) + ident := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil { + t.Fatal(err) + } + defer admin.Exec(context.Background(), "DROP DATABASE "+ident+" WITH (FORCE)") + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.Database = name + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + exec := func(query string, args ...any) { + t.Helper() + if _, err := pool.Exec(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + exec("INSERT INTO iata_codes(iata) VALUES ('YYZ'),('YVR')") + + observer := uuid.MustParse("00000000-0000-0000-0000-000000000001") + alice, bob, carol := uuid.New(), uuid.New(), uuid.New() + aliceKey := "aa11" + fmt.Sprintf("%060d", 0) + observer2 := uuid.MustParse("00000000-0000-0000-0000-000000000002") + exec("INSERT INTO observers(id,public_key) VALUES ($1,'\\x01'),($2,'\\x02')", observer, observer2) + exec(`INSERT INTO nodes(id,public_key,name,node_type) VALUES + ($1,decode($4,'hex'),'Alice',1), ($2,'\xaa22','Bob',1), ($3,'\xbb33','Carol',2)`, alice, bob, carol, aliceKey) + // Alice and Bob share prefix aa in YYZ; YVR only knows Alice. + exec(`INSERT INTO node_short_ids(node_id,iata,prefix_4) VALUES + ($1,'YYZ','\xaa110000'), ($2,'YYZ','\xaa220000'), ($3,'YYZ','\xbb330000'), ($1,'YVR','\xaa110000')`, alice, bob, carol) + + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + msg := append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...) + packets := []struct { + kind int + raw []byte + origin any + iata string + }{ + {2, msg, nil, "YYZ"}, // 1: direct message, ambiguous source + {4, []byte{0}, aliceKey, "YYZ"}, // 2: advert from Alice + {4, []byte{0}, "cc" + aliceKey[2:], "YYZ"}, // 3: advert from an unknown node + {2, msg, nil, "YVR"}, // 4: same message bytes, other IATA + } + for i, p := range packets { + n := i + 1 + exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) +VALUES (decode($1,'hex'),$2,0,1,$3,'\x00',decode($4,'hex'),$5,$5)`, fmt.Sprintf("%064x", n), p.kind, p.raw, p.origin, anchor.Add(time.Duration(n)*time.Second)) + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +VALUES ($1,decode($2,'hex'),$3,$4,$5,0,1,0)`, n, fmt.Sprintf("%064x", n), observer, p.iata, anchor.Add(time.Duration(n)*time.Second)) + } + exec("UPDATE nodes SET name='Alice renamed' WHERE id=$1", alice) + + q := &endpointQueryCounter{DBTX: pool} + store := &Store{q: sqlc.New(q)} + check := func(label string, got *api.ResolvedHop, want ...string) { + t.Helper() + if names := hopNames(got); !equalStrings(names, want) { + t.Errorf("%s = %v, want %v", label, names, want) + } + } + checkPage := func(items []api.PacketSummary) { + t.Helper() + for _, item := range items { + lo := item.LatestObserver + switch item.PacketHash { + case fmt.Sprintf("%064x", 1): + check("dm source", lo.ResolvedSource, "ambiguous", "Alice renamed", "Bob") + check("dm destination", lo.ResolvedDestination, "high", "Carol") + case fmt.Sprintf("%064x", 2): + check("advert source", lo.ResolvedSource, "high", "Alice renamed") + case fmt.Sprintf("%064x", 3): + check("unknown advert", lo.ResolvedSource, "none") + case fmt.Sprintf("%064x", 4): + check("YVR source", lo.ResolvedSource, "high", "Alice renamed") + check("YVR destination", lo.ResolvedDestination, "none") + } + } + } + + // One page query plus one hash batch and one advert batch. + for _, iatas := range [][]string{nil, {"YYZ", "YVR"}} { + q.calls = 0 + page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil || len(page.Items) != 4 || q.calls != 3 { + t.Fatalf("list %v: rows=%d queries=%d err=%v", iatas, len(page.Items), q.calls, err) + } + checkPage(page.Items) + } + q.calls = 0 + backfill, err := store.ListPacketsAfterID(ctx, 0, -1, -1, nil, "", 50) + if err != nil || len(backfill) != 4 || q.calls != 3 { + t.Fatalf("backfill: rows=%d queries=%d err=%v", len(backfill), q.calls, err) + } + checkPage(backfill) + + // Detail resolves every observation in one batch, per its own IATA. + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +VALUES (5,decode($1,'hex'),$2,'YVR',$3,0,1,0)`, fmt.Sprintf("%064x", 1), observer2, anchor.Add(10*time.Second)) + q.calls = 0 + packet, err := store.GetPacket(ctx, append(make([]byte, 31), 1)) + if err != nil || len(packet.Observations) != 2 { + t.Fatalf("detail: %v", err) + } + if q.calls != 3 { + t.Errorf("detail used %d queries, want packet, observations and one endpoint batch", q.calls) + } + check("YYZ observation", packet.Observations[0].ResolvedSource, "ambiguous", "Alice renamed", "Bob") + check("YVR observation", packet.Observations[1].ResolvedSource, "high", "Alice renamed") +} diff --git a/db/packet_observation_count_integration_test.go b/db/packet_observation_count_integration_test.go new file mode 100644 index 00000000..9f5ebad6 --- /dev/null +++ b/db/packet_observation_count_integration_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" +) + +func TestPacketObservationCountPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + + hash := []byte{0xab, 0xcd} + heard := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ($1, 4, 0, 1, '\x00', '\x00', $2, $2)`, hash, heard); err != nil { + t.Fatal(err) + } + a, _, err := store.UpsertObserver(ctx, bytes.Repeat([]byte{1}, 32), "YVR") + if err != nil { + t.Fatal(err) + } + b, _, err := store.UpsertObserver(ctx, bytes.Repeat([]byte{2}, 32), "YYZ") + if err != nil { + t.Fatal(err) + } + + steps := []struct { + obs ingest.InsertObservationParams + wantInserted bool + wantCount int64 + }{ + {ingest.InsertObservationParams{PacketHash: hash, ObserverID: a, IATA: "YVR", HeardAt: heard}, true, 1}, + {ingest.InsertObservationParams{PacketHash: hash, ObserverID: b, IATA: "YYZ", HeardAt: heard}, true, 2}, + {ingest.InsertObservationParams{PacketHash: hash, ObserverID: a, IATA: "YVR", HeardAt: heard}, false, 2}, // broker duplicate + } + for i, s := range steps { + inserted, count, err := store.InsertObservation(ctx, s.obs) + if err != nil { + t.Fatal(err) + } + if inserted != s.wantInserted || count != s.wantCount { + t.Errorf("step %d: inserted=%v count=%d, want %v, %d", i, inserted, count, s.wantInserted, s.wantCount) + } + } + + var stored, raw int64 + if err := tx.QueryRow(ctx, `SELECT p.observation_count, (SELECT count(*) FROM packet_observations po WHERE po.packet_hash = p.packet_hash) +FROM packets p WHERE p.packet_hash = $1`, hash).Scan(&stored, &raw); err != nil { + t.Fatal(err) + } + if stored != raw || stored != 2 { + t.Errorf("packets.observation_count = %d, raw count = %d; want 2", stored, raw) + } + + page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 10) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].ObservationCount != 2 { + t.Errorf("ListPackets observation counts = %+v, want one packet with 2", page.Items) + } +} diff --git a/db/packet_summary_integration_test.go b/db/packet_summary_integration_test.go new file mode 100644 index 00000000..f27cf63a --- /dev/null +++ b/db/packet_summary_integration_test.go @@ -0,0 +1,139 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +type summaryQueryCounter struct { + pgx.Tx + calls int +} + +func (c *summaryQueryCounter) Query(ctx context.Context, query string, args ...any) (pgx.Rows, error) { + c.calls++ + return c.Tx.Query(ctx, query, args...) +} + +func TestPacketSummariesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"packets", "packet_observations", "observers", "transport_scopes"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,parsed_payload,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(id),2,'0'),'hex'),kind,0,1,'\x00','\x00',parsed::jsonb, + '2026-01-01'::timestamptz+id*interval '1 second','2026-01-01'::timestamptz+id*interval '1 second' +FROM (VALUES + (1,4,'{"type":"ADVERT","appData":{"name":"MD00-Repeater"}}'), + (2,4,'{"type":"ADVERT","appData":{"name":"Relay 📡"}}'), + (3,4,'{"type":"ADVERT","appData":{"name":""}}'), + (4,4,'{"type":"ADVERT","appData":{}}'), + (5,4,NULL), + (6,4,'{"type":"ADVERT","appData":{"name":null}}'), + (7,4,'{"type":"ADVERT","appData":{"name":42}}'), + (8,2,'{"appData":{"name":"not an advert"}}'), + (9,4,'{"type":"ADVERT","appData":[]}'), + (10,3,'{"type":"ACK","checksum":"01020304"}'), + (11,3,'{"type":"ACK","checksum":"AABBCCDD"}'), + (12,3,'{"type":"ACK","checksum":"00000000"}'), + (13,3,'{"type":"ACK"}'), + (14,3,'{"type":"ACK","checksum":12345678}'), + (15,3,'{"type":"ACK","checksum":"gg112233"}'), + (16,3,'{"type":"ACK","checksum":"1234"}'), + (17,3,'{"type":"ACK","checksum":"123456789"}'), + (18,3,'{"type":"TRACE","checksum":"01020304"}'), + (19,9,'{"type":"TRACE","traceTag":"efbeadde","authCode":4277009102}'), + (20,9,'{"type":"PING","traceTag":"deadBEEF"}'), + (21,9,'{"type":"OTHER","traceTag":"01020304"}'), + (22,9,'{"traceTag":"01020304"}'), + (23,9,'{"type":"TRACE","traceTag":12345678}'), + (24,9,'{"type":"TRACE","traceTag":"01020304\n"}'), + (25,9,'{"type":"TRACE","traceTag":"gg112233"}'), + (26,9,'{"type":"TRACE","traceTag":null}'), + (27,2,'{"type":"ACK","checksum":"01020304"}'), + (28,9,'{"type":"TRACE","traceTag":"00000000"}') +) v(id,kind,parsed); +`) + if err != nil { + t.Fatal(err) + } + counter := &summaryQueryCounter{Tx: tx} + store := &Store{q: sqlc.New(counter)} + check := func(items []api.PacketSummary) { + t.Helper() + if len(items) != 28 { + t.Fatalf("got %d packets, want 28", len(items)) + } + for _, item := range items { + want := map[string]string{ + "01": "MD00-Repeater", "02": "Relay 📡", + "0a": "ACK 01020304", "0b": "ACK aabbccdd", "0c": "ACK 00000000", + "13": "TRACE efbeadde", "14": "PING deadbeef", "1c": "TRACE 00000000", + }[item.PacketHash] + if want == "" { + if item.Summary != nil { + t.Fatalf("unexpected summary for %s: %q", item.PacketHash, *item.Summary) + } + } else if item.Summary == nil || *item.Summary != want { + t.Fatalf("summary for %s = %v, want %q", item.PacketHash, item.Summary, want) + } + } + if counter.calls != 1 { + t.Fatalf("list made %d queries, want 1", counter.calls) + } + counter.calls = 0 + } + page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil { + t.Fatal(err) + } + check(page.Items) // names remain available before any observation arrives + _, err = tx.Exec(ctx, ` +INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT id,decode(lpad(to_hex(id),2,'0'),'hex'),'00000000-0000-0000-0000-000000000001','YVR', + '2026-01-01'::timestamptz+id*interval '1 second',0,1,0 FROM generate_series(1,28) id; +`) + if err != nil { + t.Fatal(err) + } + for _, iatas := range [][]string{nil, {"YVR"}} { + page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil { + t.Fatal(err) + } + check(page.Items) + rows, err := store.ListPacketsAfterID(ctx, 0, -1, -1, iatas, "", 50) + if err != nil { + t.Fatal(err) + } + check(rows) + } +} diff --git a/db/packets.go b/db/packets.go index 68bd34cc..620acceb 100644 --- a/db/packets.go +++ b/db/packets.go @@ -8,16 +8,14 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" - "errors" "fmt" - "log" + "log/slog" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/google/uuid" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/meshcore-go/meshcore-go" ) @@ -58,6 +56,32 @@ func (s *Store) SetPacketDecrypted(ctx context.Context, hash []byte) error { return s.q.SetPacketDecrypted(ctx, hash) } +// buildLatestObserverPath fills in PacketLatestObserver's optional path fields from the +// nullable path_length_byte/hash_size/hop_count/path_bytes columns joined in alongside the +// latest (or matching) observation. hashSize/hopCount nil means no observation joined at all +// (only possible via ListPackets/ListPacketsByIATAs' LEFT JOIN LATERAL -- ListPacketsAfterID's +// inner join always has them, callers there can pass &v.Field directly). +func buildLatestObserverPath(pathLengthByte, hashSize, hopCount *int16, pathBytes []byte) (*api.PacketPathLength, *string) { + if hashSize == nil || hopCount == nil { + return nil, nil + } + raw := "" + if pathLengthByte != nil { + raw = fmt.Sprintf("%02x", *pathLengthByte) + } + pathLength := &api.PacketPathLength{ + Raw: raw, + HashSize: *hashSize, + HopCount: *hopCount, + } + var pathBytesHex *string + if pathBytes != nil { + s := hex.EncodeToString(pathBytes) + pathBytesHex = &s + } + return pathLength, pathBytesHex +} + func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int16, iatas []string, scopes []string, since, until time.Time, cursor int64, limit int32) (api.Page[api.PacketSummary], error) { if len(iatas) > 0 { return s.listPacketsByIATAs(ctx, payloadTypes, routeTypes, iatas, scopes, since, until, cursor, limit) @@ -91,7 +115,9 @@ func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int1 rows = rows[:limit] } items := make([]api.PacketSummary, 0, len(rows)) + lookups := make([]endpointLookup, 0, len(rows)) for _, v := range rows { + var lookup endpointLookup item := api.PacketSummary{ PacketHash: hex.EncodeToString(v.PacketHash), PayloadType: v.PayloadType, @@ -103,15 +129,24 @@ func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int1 LastHeardAt: v.LastHeardAt.Time.UnixMilli(), ObservationCount: int32(v.ObservationCount), } + if v.Summary != "" { + item.Summary = &v.Summary + } if v.LatestObserverID != (uuid.UUID{}) { item.LatestObserver = &api.PacketLatestObserver{ ID: v.LatestObserverID, DisplayName: v.LatestObserverName, IATA: v.LatestObserverIata, } + lookup = endpointLookup{iata: v.LatestObserverIata, ep: parsePacketEndpoints(v.PayloadType, v.RawPayload, v.OriginPubkey)} + item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( + &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, + ) } items = append(items, item) + lookups = append(lookups, lookup) } + s.fillLatestEndpoints(ctx, items, lookups) var nextCursor *int64 if hasMore && len(items) > 0 { last := items[len(items)-1].LastHeardAt @@ -158,8 +193,18 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes if hasMore { rows = rows[:limit] } + // Observers duplicate a packet across the scan, so the page can collapse + // short while the site still has history below scan_floor. Stopping here + // would strand every older packet behind a hasMore=false. + var scanFloor pgtype.Timestamptz + if len(rows) > 0 && rows[0].ScanSaturated { + hasMore = true + scanFloor = rows[0].ScanFloor + } items := make([]api.PacketSummary, 0, len(rows)) + lookups := make([]endpointLookup, 0, len(rows)) for _, v := range rows { + var lookup endpointLookup item := api.PacketSummary{ PacketHash: hex.EncodeToString(v.PacketHash), PayloadType: v.PayloadType, @@ -171,20 +216,35 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes LastHeardAt: v.LastHeardAt.Time.UnixMilli(), ObservationCount: int32(v.ObservationCount), } + if v.Summary != "" { + item.Summary = &v.Summary + } if v.LatestObserverID != (uuid.UUID{}) { item.LatestObserver = &api.PacketLatestObserver{ ID: v.LatestObserverID, DisplayName: v.LatestObserverName, IATA: v.LatestObserverIata, } + lookup = endpointLookup{iata: v.LatestObserverIata, ep: parsePacketEndpoints(v.PayloadType, v.RawPayload, v.OriginPubkey)} + item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( + &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, + ) } items = append(items, item) + lookups = append(lookups, lookup) } + s.fillLatestEndpoints(ctx, items, lookups) // Cursor follows site-local recency, not the packet's global last_heard_at. var nextCursor *int64 if hasMore && len(rows) > 0 { - last := rows[len(rows)-1].SiteHeardAt.Time.UnixMilli() - nextCursor = &last + last := rows[len(rows)-1].SiteHeardAt.Time + // A saturated scan never read below its floor. Paging past it would + // skip that band; resuming at it only repeats packets already shown. + if scanFloor.Valid && scanFloor.Time.After(last) { + last = scanFloor.Time + } + ms := last.UnixMilli() + nextCursor = &ms } return api.Page[api.PacketSummary]{ Items: items, @@ -206,7 +266,9 @@ func (s *Store) ListPacketsAfterID(ctx context.Context, afterObservationID int64 return nil, err } items := make([]api.PacketSummary, 0, len(rows)) + lookups := make([]endpointLookup, 0, len(rows)) for _, v := range rows { + var lookup endpointLookup item := api.PacketSummary{ PacketHash: hex.EncodeToString(v.PacketHash), PayloadType: v.PayloadType, @@ -218,15 +280,26 @@ func (s *Store) ListPacketsAfterID(ctx context.Context, afterObservationID int64 LastHeardAt: v.LastHeardAt.Time.UnixMilli(), ObservationCount: int32(v.ObservationCount), } + if v.Summary != "" { + item.Summary = &v.Summary + } if v.LatestObserverID != (uuid.UUID{}) { item.LatestObserver = &api.PacketLatestObserver{ ID: v.LatestObserverID, DisplayName: v.LatestObserverName, IATA: v.LatestObserverIata, } + lookup = endpointLookup{iata: v.LatestObserverIata, ep: parsePacketEndpoints(v.PayloadType, v.RawPayload, v.OriginPubkey)} + // Inner join here (unlike ListPackets/listPacketsByIATAs' LEFT JOIN LATERAL), so + // these are never nil when an observer was joined at all. + item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( + &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, + ) } items = append(items, item) + lookups = append(lookups, lookup) } + s.fillLatestEndpoints(ctx, items, lookups) return items, nil } @@ -292,7 +365,10 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, ObservationCount: int32(len(obsRows)), Observations: make([]api.PacketObservationDetail, 0, len(obsRows)), } - minHeardAt := obsRows[0].HeardAt.Time + var minHeardAt time.Time + if len(obsRows) > 0 { + minHeardAt = obsRows[0].HeardAt.Time + } if len(obsRows) > 1 { maxHeardAt := obsRows[0].HeardAt.Time for _, v := range obsRows[1:] { @@ -337,49 +413,14 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, } } } - // 1-byte source/destination hashes for the payload types that carry a resolvable - // endpoint (REQUEST, RESPONSE, TEXT_MESSAGE, PATH, ANON_REQ's destination). Constant - // for this packet hash, so parsed once; resolution itself still happens per-observation - // below since candidates depend on the observation's IATA, same as the intermediate - // hop resolution already does. - var sourceHashByte, destHashByte []byte - switch row.PayloadType { - case int16(meshcore.PayloadTypeAnonReq): - if anonReq, err := meshcore.AnonReqFromBytes(row.RawPayload); err == nil { - destHashByte = []byte{anonReq.Destination} - } - case int16(meshcore.PayloadTypeReq): - if req, err := meshcore.RequestFromBytes(row.RawPayload); err == nil { - sourceHashByte = []byte{req.Source} - destHashByte = []byte{req.Destination} - } - case int16(meshcore.PayloadTypeResponse): - if resp, err := meshcore.ResponseFromBytes(row.RawPayload); err == nil { - sourceHashByte = []byte{resp.Source} - destHashByte = []byte{resp.Destination} - } - case int16(meshcore.PayloadTypeTxtMsg): - if txt, err := meshcore.TextMessageFromBytes(row.RawPayload); err == nil { - sourceHashByte = []byte{txt.Source} - destHashByte = []byte{txt.Destination} - } - case int16(meshcore.PayloadTypePath): - if path, err := meshcore.PathFromBytes(row.RawPayload); err == nil { - sourceHashByte = []byte{path.Source} - destHashByte = []byte{path.Destination} - } + // Endpoints resolve against each observation's IATA, all in one batch. + ep := parsePacketEndpoints(row.PayloadType, row.RawPayload, row.OriginPubkey) + lookups := make([]endpointLookup, len(obsRows)) + for i, v := range obsRows { + lookups[i] = endpointLookup{iata: v.Iata, ep: ep} } - // ADVERT's source is an exact pubkey match, not ambiguous like the above -- and unlike - // them it doesn't depend on IATA, so resolve it once here rather than per observation. - var resolvedAdvertSource *api.ResolvedNode - if row.PayloadType == int16(meshcore.PayloadTypeAdvert) && row.OriginPubkey != nil { - if nodeID, err := s.GetNodeByPubkey(ctx, row.OriginPubkey); err == nil { - if nodes, err := s.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { - resolvedAdvertSource = nodes[nodeID] - } - } - } - for _, v := range obsRows { + sources, destinations := s.resolveEndpoints(ctx, lookups) + for i, v := range obsRows { obs := api.PacketObservationDetail{ ID: v.ID, ObserverID: v.ObserverID, @@ -391,9 +432,11 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, HashSize: v.HashSize, HopCount: v.HopCount, }, - RSSI: v.Rssi, - SNR: v.Snr, - SourceBroker: *v.SourceBroker, + RSSI: v.Rssi, + SNR: v.Snr, + } + if v.SourceBroker != nil { + obs.SourceBroker = *v.SourceBroker } prop := int32(v.HeardAt.Time.Sub(minHeardAt).Milliseconds()) obs.PropagationTimeMs = &prop @@ -402,7 +445,7 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, if len(traceRawHashes) > 0 { resolved, err := s.ResolvePathHashes(ctx, v.Iata, traceRawHashes) if err != nil { - log.Printf("store: path resolution failed for observation %d: %v", v.ID, err) + slog.Error(fmt.Sprintf("store: path resolution failed for observation %d", v.ID), "component", "db", "error", err) } else { resolvedPath = api.BuildResolvedPath(traceRawHashes, resolved) } @@ -415,31 +458,13 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, } resolved, err := s.ResolvePathHashes(ctx, v.Iata, hashes) if err != nil { - log.Printf("store: path resolution failed for observation %d: %v", v.ID, err) + slog.Error(fmt.Sprintf("store: path resolution failed for observation %d", v.ID), "component", "db", "error", err) } else { resolvedPath = api.BuildResolvedPath(hashes, resolved) } } obs.ResolvedPath = resolvedPath - if row.PayloadType == int16(meshcore.PayloadTypeAdvert) { - hop := api.ResolveExactNode(resolvedAdvertSource) - obs.ResolvedSource = &hop - } else if len(sourceHashByte) == 1 { - if r, err := s.ResolvePathHashes(ctx, v.Iata, [][]byte{sourceHashByte}); err != nil { - log.Printf("store: source resolution failed for observation %d: %v", v.ID, err) - } else { - hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] - obs.ResolvedSource = &hop - } - } - if len(destHashByte) == 1 { - if r, err := s.ResolvePathHashes(ctx, v.Iata, [][]byte{destHashByte}); err != nil { - log.Printf("store: destination resolution failed for observation %d: %v", v.ID, err) - } else { - hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] - obs.ResolvedDestination = &hop - } - } + obs.ResolvedSource, obs.ResolvedDestination = sources[i], destinations[i] if row.PayloadType == int16(meshcore.PayloadTypeTrace) && len(traceRawHashes) > 0 { // Swap in the trace's own path hashes so PathData's hop-block split (driven by // pathBytes + hashSize) lines up 1:1 with resolvedPath -- the raw SNR bytes @@ -487,7 +512,7 @@ func (s *Store) UpsertIATA(ctx context.Context, iata string) error { return s.q.UpsertIATA(ctx, iata) } -func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservationParams) (bool, error) { +func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservationParams) (bool, int64, error) { params := sqlc.InsertObservationParams{ PacketHash: o.PacketHash, ObserverID: o.ObserverID, @@ -506,15 +531,13 @@ func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservatio CodingRate: &o.CodingRate, SourceBroker: &o.SourceBroker, PayloadType: &o.PayloadType, + AirtimeMs: o.AirtimeMs, } row, err := s.q.InsertObservation(ctx, params) - if errors.Is(err, pgx.ErrNoRows) { - return false, nil // conflict, not an error - } if err != nil { - return false, err + return false, 0, err } - return row.ID != 0, nil + return row.Inserted, row.ObservationCount, nil } func (s *Store) ListNodeObservations(ctx context.Context, nodeID uuid.UUID, cursor int64, limit int32) (api.Page[api.PacketObservationSummary], error) { @@ -556,10 +579,14 @@ func (s *Store) ListNodeObservations(ctx context.Context, nodeID uuid.UUID, curs }, nil } -func (s *Store) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) { - return s.q.GetPacketObservationCount(ctx, packetHash) -} +// Small because each packet cascades to its observations. +const packetDeleteBatch = 1000 func (s *Store) DeleteOldPackets(ctx context.Context, cutoff time.Time) error { - return s.q.DeleteOldPackets(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) + return deleteInBatches(ctx, packetDeleteBatch, func(ctx context.Context, n int32) (int64, error) { + return s.q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{ + Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, + BatchSize: n, + }) + }) } diff --git a/db/packets_integration_test.go b/db/packets_integration_test.go new file mode 100644 index 00000000..2d019ee8 --- /dev/null +++ b/db/packets_integration_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +// The packet and its observation are separate ingest writes. A list request +// between them must return the packet without inventing an observer or failing. +func TestListPacketsBeforeObservationPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"packets", "packet_observations", "observers", "transport_scopes"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ('\x01', 4, 0, 1, '\x00', '\x00', NOW(), NOW())`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil { + t.Fatalf("packet without observation must remain readable: %v", err) + } + if len(page.Items) != 1 || page.Items[0].PacketHash != "01" { + t.Fatalf("packet missing: %+v", page) + } + if page.Items[0].LatestObserver != nil || page.Items[0].ObservationCount != 0 { + t.Fatalf("invented observation: %+v", page.Items[0]) + } + _, err = tx.Exec(ctx, ` +INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x02'); +INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes) +VALUES (1, '\x01', '00000000-0000-0000-0000-000000000001', 'YVR', NOW(), 2, 1, 2, '\x1122'); +UPDATE packets SET observation_count = 1 WHERE packet_hash = '\x01';`) + if err != nil { + t.Fatal(err) + } + for _, iatas := range [][]string{nil, {"YVR"}} { + page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].ObservationCount != 1 { + t.Fatalf("observed packet missing for %v: %+v", iatas, page) + } + observer := page.Items[0].LatestObserver + if observer == nil || observer.IATA != "YVR" || observer.PathLength == nil || observer.PathBytes == nil { + t.Fatalf("observation metadata missing: %+v", observer) + } + if observer.PathLength.Raw != "02" || observer.PathLength.HashSize != 1 || observer.PathLength.HopCount != 2 || *observer.PathBytes != "1122" { + t.Fatalf("observation path changed: %+v", observer) + } + } +} diff --git a/db/packets_test.go b/db/packets_test.go index ba228c59..c5f06271 100644 --- a/db/packets_test.go +++ b/db/packets_test.go @@ -12,7 +12,6 @@ import ( mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/google/uuid" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "go.uber.org/mock/gomock" ) @@ -157,6 +156,59 @@ func TestListPackets_LatestObserverSet(t *testing.T) { } } +func TestListPackets_LatestObserverPathFields(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + heardAt := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + pathLengthByte := int16(0x42) + hashSize := int16(1) + hopCount := int16(2) + pathBytes := []byte{0xa1, 0xb2} + + mock.EXPECT(). + ListPackets(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsRow{ + { + PacketHash: []byte{0xde, 0xad}, + FirstHeardAt: heardAt, + LastHeardAt: heardAt, + LatestObserverID: observerID, + LatestObserverPathLengthByte: pathLengthByte, + LatestObserverHashSize: hashSize, + LatestObserverHopCount: hopCount, + LatestObserverPathBytes: pathBytes, + }, + }, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 10) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + obs := page.Items[0].LatestObserver + if obs == nil { + t.Fatal("expected LatestObserver to be set") + } + if obs.PathLength == nil { + t.Fatal("expected PathLength to be set") + } + if obs.PathLength.HashSize != 1 || obs.PathLength.HopCount != 2 { + t.Errorf("expected hashSize=1 hopCount=2, got hashSize=%d hopCount=%d", obs.PathLength.HashSize, obs.PathLength.HopCount) + } + if obs.PathLength.Raw != "42" { + t.Errorf("expected raw 42, got %s", obs.PathLength.Raw) + } + if obs.PathBytes == nil || *obs.PathBytes != "a1b2" { + t.Errorf("expected pathBytes a1b2, got %v", obs.PathBytes) + } + // Legacy rows without a captured snapshot still omit resolved endpoints on lists. + if obs.ResolvedPath != nil || obs.ResolvedSource != nil || obs.ResolvedDestination != nil { + t.Error("expected no resolved path/source/destination on the list endpoint") + } +} + func TestInsertObservation_Success(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) @@ -165,10 +217,10 @@ func TestInsertObservation_Success(t *testing.T) { mock.EXPECT(). InsertObservation(gomock.Any(), gomock.Any()). - Return(sqlc.PacketObservation{ID: 1}, nil) + Return(sqlc.InsertObservationRow{Inserted: true, ObservationCount: 4}, nil) store := &Store{q: mock} - inserted, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ + inserted, count, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ PacketHash: []byte{0xde, 0xad}, ObserverID: observerID, IATA: "YVR", @@ -177,8 +229,8 @@ func TestInsertObservation_Success(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if !inserted { - t.Error("expected inserted true") + if !inserted || count != 4 { + t.Errorf("got inserted=%v count=%d, want true, 4", inserted, count) } } @@ -188,18 +240,18 @@ func TestInsertObservation_Conflict(t *testing.T) { mock.EXPECT(). InsertObservation(gomock.Any(), gomock.Any()). - Return(sqlc.PacketObservation{}, pgx.ErrNoRows) + Return(sqlc.InsertObservationRow{Inserted: false, ObservationCount: 2}, nil) store := &Store{q: mock} - inserted, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ + inserted, count, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ PacketHash: []byte{0xde, 0xad}, HeardAt: time.Now(), }) if err != nil { t.Fatalf("expected nil error on conflict, got %v", err) } - if inserted { - t.Error("expected inserted false on conflict") + if inserted || count != 2 { + t.Errorf("got inserted=%v count=%d, want false, 2", inserted, count) } } @@ -352,6 +404,45 @@ func TestListPacketsAfterID_PassesIATAsAsArray(t *testing.T) { } } +func TestListPacketsAfterID_LatestObserverPathFields(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + heardAt := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT(). + ListPacketsAfterID(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsAfterIDRow{ + { + PacketHash: []byte{0xde, 0xad}, + FirstHeardAt: heardAt, + LastHeardAt: heardAt, + LatestObserverID: observerID, + LatestObserverPathLengthByte: 0x42, + LatestObserverHashSize: 1, + LatestObserverHopCount: 2, + LatestObserverPathBytes: []byte{0xa1, 0xb2}, + }, + }, nil) + + store := &Store{q: mock} + items, err := store.ListPacketsAfterID(context.Background(), 0, -1, -1, nil, "", 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + obs := items[0].LatestObserver + if obs == nil || obs.PathLength == nil { + t.Fatal("expected LatestObserver and PathLength to be set") + } + if obs.PathLength.HashSize != 1 || obs.PathLength.HopCount != 2 { + t.Errorf("expected hashSize=1 hopCount=2, got hashSize=%d hopCount=%d", obs.PathLength.HashSize, obs.PathLength.HopCount) + } + if obs.PathBytes == nil || *obs.PathBytes != "a1b2" { + t.Errorf("expected pathBytes a1b2, got %v", obs.PathBytes) + } +} + func TestListNodeObservations_Pagination(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) @@ -435,6 +526,107 @@ func TestListPackets_IATAFilterRoutesToObservationIndex(t *testing.T) { } } +// A site whose packets are heard by more observers than scan_depth allows +// for collapses to a short page while history remains below the floor. +// The page must still report more, or the client stops paging for good. +func TestListPackets_SaturatedShortPageKeepsPaging(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + oldest := time.Date(2026, 8, 7, 12, 57, 25, 0, time.UTC) + floor := time.Date(2026, 8, 7, 12, 50, 0, 0, time.UTC) + + // Two rows for a limit of 5: the scan filled up but collapsed to a short page. + mock.EXPECT(). + ListPacketsByIATAs(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsByIATAsRow{ + { + PacketHash: []byte{0x01}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest.Add(time.Minute), Valid: true}, + ScanSaturated: true, + ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, + }, + { + PacketHash: []byte{0x02}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, + ScanSaturated: true, + ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, + }, + }, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW"}, nil, time.Time{}, time.Time{}, 0, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(page.Items) != 2 { + t.Fatalf("got %d items, want 2", len(page.Items)) + } + if !page.HasMore { + t.Error("hasMore = false on a saturated short page, want true") + } + // Oldest returned item sits above the floor, so it is the safe cursor. + if page.NextCursor == nil || *page.NextCursor != oldest.UnixMilli() { + t.Errorf("next cursor = %v, want %d (oldest item)", page.NextCursor, oldest.UnixMilli()) + } +} + +// The floor is newer than the oldest item, so paging past it would skip the +// band the saturated site never read. +func TestListPackets_CursorClampsToScanFloor(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + floor := time.Date(2026, 8, 7, 12, 50, 0, 0, time.UTC) + oldest := floor.Add(-30 * time.Minute) + + mock.EXPECT(). + ListPacketsByIATAs(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsByIATAsRow{{ + PacketHash: []byte{0x01}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, + ScanSaturated: true, + ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, + }}, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW", "YYZ"}, nil, time.Time{}, time.Time{}, 0, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if page.NextCursor == nil || *page.NextCursor != floor.UnixMilli() { + t.Errorf("next cursor = %v, want %d (clamped to floor)", page.NextCursor, floor.UnixMilli()) + } +} + +// An unsaturated scan read the site dry, so paging has to stop. +func TestListPackets_UnsaturatedShortPageEndsPaging(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + oldest := time.Date(2026, 8, 7, 12, 57, 25, 0, time.UTC) + + mock.EXPECT(). + ListPacketsByIATAs(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsByIATAsRow{{ + PacketHash: []byte{0x01}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, + ScanSaturated: false, + }}, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW"}, nil, time.Time{}, time.Time{}, 0, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if page.HasMore { + t.Error("hasMore = true on an exhausted site, want false") + } + if page.NextCursor != nil { + t.Errorf("next cursor = %v, want nil", page.NextCursor) + } +} + func TestListPackets_UnfilteredKeepsGlobalQuery(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) @@ -449,3 +641,23 @@ func TestListPackets_UnfilteredKeepsGlobalQuery(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestDeleteOldPackets_LoopsUntilShortBatch(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + cutoff := time.Date(2026, 9, 18, 0, 0, 0, 0, time.UTC) + + byCutoff := gomock.Cond(func(p sqlc.DeleteOldPacketsParams) bool { + return p.Cutoff.Time.Equal(cutoff) && p.BatchSize == packetDeleteBatch + }) + gomock.InOrder( + mock.EXPECT().DeleteOldPackets(gomock.Any(), byCutoff).Return(int64(packetDeleteBatch), nil), + mock.EXPECT().DeleteOldPackets(gomock.Any(), byCutoff).Return(int64(packetDeleteBatch), nil), + mock.EXPECT().DeleteOldPackets(gomock.Any(), byCutoff).Return(int64(7), nil), + ) + + if err := store.DeleteOldPackets(context.Background(), cutoff); err != nil { + t.Fatal(err) + } +} diff --git a/db/paths.go b/db/paths.go new file mode 100644 index 00000000..2189f926 --- /dev/null +++ b/db/paths.go @@ -0,0 +1,75 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + rows, err := s.q.GetPathStats(ctx, sqlc.GetPathStatsParams{Since: pgtype.Timestamptz{Time: since, Valid: true}, Until: pgtype.Timestamptz{Time: until, Valid: true}, Iatas: iatas}) + if err != nil { + return nil, err + } + stats := &api.PathStats{Since: since.UnixMilli(), Until: until.UnixMilli(), HashWidths: []api.PathHashWidth{{Bytes: 1}, {Bytes: 2}, {Bytes: 3}}, PathLengths: []api.PathLengthBin{}, Hourly: []api.PathHour{}} + var lengths [64]int64 + for _, row := range rows { + if row.Category == 0 && (row.HashBytes < 1 || int(row.HashBytes) > len(stats.HashWidths) || row.Entries < 0 || int(row.Entries) >= len(lengths)) { + return nil, fmt.Errorf("invalid path statistics bucket") + } + if row.IsHourly { + hour := row.Hour.Time.UnixMilli() + if len(stats.Hourly) == 0 || stats.Hourly[len(stats.Hourly)-1].Hour != hour { + stats.Hourly = append(stats.Hourly, api.PathHour{Hour: hour}) + } + point := &stats.Hourly[len(stats.Hourly)-1] + point.Receptions += row.Receptions + point.MaxEntries = max(point.MaxEntries, row.MaxEntries) + switch row.Category { + case 0: + switch row.HashBytes { + case 1: + point.OneByte += row.Receptions + case 2: + point.TwoByte += row.Receptions + case 3: + point.ThreeByte += row.Receptions + } + case 1: + point.Empty += row.Receptions + case 2: + point.Trace += row.Receptions + case 3: + point.Unclassified += row.Receptions + } + continue + } + stats.Receptions += row.Receptions + switch row.Category { + case 0: + stats.Hashed += row.Receptions + stats.HashWidths[row.HashBytes-1].Receptions += row.Receptions + lengths[row.Entries] += row.Receptions + case 1: + stats.Empty += row.Receptions + lengths[0] += row.Receptions + case 2: + stats.Trace += row.Receptions + case 3: + stats.Unclassified += row.Receptions + } + } + for entries, count := range lengths { + if count > 0 { + stats.PathLengths = append(stats.PathLengths, api.PathLengthBin{Entries: int32(entries), Receptions: count}) + } + } + return stats, nil +} diff --git a/db/paths_integration_test.go b/db/paths_integration_test.go new file mode 100644 index 00000000..fbec87f4 --- /dev/null +++ b/db/paths_integration_test.go @@ -0,0 +1,192 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "slices" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/jackc/pgx/v5" + meshcore "github.com/meshcore-go/meshcore-go" +) + +func TestPathsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + _, err = tx.Exec(ctx, ` +SET LOCAL TIME ZONE 'America/Vancouver'; +CREATE TEMP TABLE fixture(heard_at timestamptz NOT NULL,iata char(3) NOT NULL,payload_type smallint,path_length_byte smallint NOT NULL,hash_size smallint NOT NULL,hop_count smallint NOT NULL,path_bytes bytea); +INSERT INTO fixture +SELECT CASE id WHEN 1 THEN '2026-01-01 00:30+00'::timestamptz WHEN 17 THEN '2026-01-01 04:00+00'::timestamptz WHEN 18 THEN '2026-01-01 04:30+00'::timestamptz WHEN 20 THEN '2026-01-01 03:00+00'::timestamptz + ELSE '2026-01-01 00:30+00'::timestamptz+(id/10)*interval '2 hour'+(id%10)*interval '1 minute' END, + iata,payload,raw,width,hops,CASE WHEN size IS NULL THEN NULL ELSE decode(repeat('ab',size),'hex') END +FROM (VALUES + (1,'YVR',4,2,1,2,2),(2,'YVR',4,67,2,3,6),(3,'YYJ',5,130,3,2,6), + (4,'YVR',4,64,2,0,NULL),(5,'YVR',9,3,1,3,3),(6,'YYJ',NULL,2,1,2,2), + (7,'YVR',4,3,1,3,2),(8,'YVR',4,194,4,2,8),(9,'YVR',4,96,2,32,64), + (10,'YVR',4,97,2,33,66),(11,'YVR',4,64,1,0,0),(12,'YVR',4,0,1,1,1), + (13,'YYZ',5,0,1,0,0),(14,'YVR',16,1,1,1,1),(15,'YVR',4,63,1,63,63), + (16,'YVR',4,149,3,21,63),(17,'YVR',4,1,1,1,1),(18,'YVR',4,1,1,1,1), + (19,'YYJ',9,3,1,3,3),(20,'YVR',4,64,2,0,0) +) v(id,iata,payload,raw,width,hops,size); +ALTER TABLE fixture ADD COLUMN n serial; +INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT int4send(n),4,0,1,'\x00','\x00',heard_at,heard_at FROM fixture; +INSERT INTO packet_observations (packet_hash,observer_id,heard_at,iata,payload_type,path_length_byte,hash_size,hop_count,path_bytes) +SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_type,path_length_byte,hash_size,hop_count,path_bytes FROM fixture;`) + if err != nil { + t.Fatal(err) + } + since := time.Now().UTC().Truncate(24 * time.Hour).Add(-48 * time.Hour) + if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { + t.Fatal(err) + } + rollTxHours(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + until := since.Add(4 * time.Hour) + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err := tx.Exec(ctx, "SET LOCAL plan_cache_mode="+mode); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + iatas []string + counts [5]int64 + }{ + {"global", nil, [5]int64{18, 6, 3, 2, 7}}, {"empty filter", []string{}, [5]int64{18, 6, 3, 2, 7}}, + {"YVR", []string{"YVR"}, [5]int64{14, 5, 2, 1, 6}}, {"overlap", []string{"YVR", "YYJ", "YVR"}, [5]int64{17, 6, 2, 2, 7}}, + {"unknown", []string{"ZZZ"}, [5]int64{}}, {"empty region", []string{""}, [5]int64{}}, + } { + t.Run(mode+"/"+tc.name, func(t *testing.T) { + got, err := store.GetPathStats(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + if [5]int64{got.Receptions, got.Hashed, got.Empty, got.Trace, got.Unclassified} != tc.counts { + t.Fatalf("counts: %+v", got) + } + var widths, lengths, hours int64 + for _, b := range got.HashWidths { + widths += b.Receptions + } + for _, b := range got.PathLengths { + lengths += b.Receptions + } + for i, h := range got.Hourly { + hours += h.Receptions + if h.Receptions != h.OneByte+h.TwoByte+h.ThreeByte+h.Empty+h.Trace+h.Unclassified || h.Hour%3600000 != 0 || (i > 0 && h.Hour <= got.Hourly[i-1].Hour) { + t.Fatalf("hour: %+v", h) + } + } + if widths != got.Hashed || lengths != got.Hashed+got.Empty || hours != got.Receptions { + t.Fatal("partitions do not reconcile") + } + if tc.name == "global" && (got.HashWidths[0].Receptions != 2 || got.HashWidths[1].Receptions != 2 || got.HashWidths[2].Receptions != 2 || got.PathLengths[0].Entries != 0 || got.PathLengths[0].Receptions != 3 || len(got.Hourly) != 3) { + t.Fatalf("empty/trace widths or missing hour: %+v", got) + } + }) + } + } + // The longest routed path per hour; hours with only empty, TRACE or invalid paths report 0. + for _, tc := range []struct { + iatas []string + want []int32 + }{ + {nil, []int32{32, 63, 0}}, + {[]string{"YVR", "YYJ"}, []int32{32, 63, 0}}, + {[]string{"YYJ"}, []int32{2, 0}}, + {[]string{"YYZ"}, []int32{0}}, + } { + got, err := store.GetPathStats(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + var max []int32 + for _, h := range got.Hourly { + max = append(max, h.MaxEntries) + } + if !slices.Equal(max, tc.want) { + t.Errorf("maxEntries for %v = %v, want %v", tc.iatas, max, tc.want) + } + } + w := httptest.NewRecorder() + handlers.StatsRouter(store, handlers.StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) + var response api.PathStats + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 14 || response.Hashed != 5 { + t.Fatalf("HTTP %d: %s (%v)", w.Code, w.Body.String(), err) + } + if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT int4send(1000+i),4,0,1,'\x00','\x00',$1,$1 FROM generate_series(0,255)i`, since); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `INSERT INTO packet_observations (packet_hash,observer_id,heard_at,iata,payload_type,path_length_byte,hash_size,hop_count,path_bytes) +SELECT int4send(1000+i),'00000000-0000-0000-0000-000000000001',$1::timestamptz+interval '1 hour','YVR',4,i,(i>>6)+1,i&63,decode(repeat('ab',((i>>6)+1)*(i&63)),'hex') FROM generate_series(0,255)i;`, since); err != nil { + t.Fatal(err) + } + stale, err := store.GetPathStats(ctx, since, until, nil) + if err != nil || stale.Receptions != 18 { + t.Fatalf("snapshot lost: %+v %v", stale, err) + } + rollTxHours(t, ctx, tx) + got, err := store.GetPathStats(ctx, since, until, nil) + if err != nil { + t.Fatal(err) + } + var hashed, empty, unknown int64 + var widths [3]int64 + for raw := 0; raw < 256; raw++ { + if !meshcore.IsValidPathLen(uint8(raw)) { + unknown++ + } else if raw&63 == 0 { + empty++ + } else { + hashed++ + widths[raw>>6]++ + } + } + if got.Hashed != hashed || got.Empty != empty || got.Unclassified != unknown || got.Receptions != 256 { + t.Fatalf("decoder agreement: %+v", got) + } + for i, count := range widths { + if got.HashWidths[i].Receptions != count { + t.Fatal("decoder width disagreement") + } + } + if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { + t.Fatal(err) + } + rollTxHours(t, ctx, tx) + got, err = store.GetPathStats(ctx, since, until, nil) + if err != nil || got.Receptions != 0 || got.Hourly == nil || got.PathLengths == nil || len(got.HashWidths) != 3 { + t.Fatalf("empty: %+v %v", got, err) + } +} diff --git a/db/paths_test.go b/db/paths_test.go new file mode 100644 index 00000000..3b13606d --- /dev/null +++ b/db/paths_test.go @@ -0,0 +1,60 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5/pgtype" +) + +type pathBucketQuerier struct { + sqlc.Querier + row sqlc.GetPathStatsRow +} + +func (q pathBucketQuerier) GetPathStats(context.Context, sqlc.GetPathStatsParams) ([]sqlc.GetPathStatsRow, error) { + return []sqlc.GetPathStatsRow{q.row}, nil +} + +func TestPathStatsRejectsInvalidDatabaseBucket(t *testing.T) { + for _, row := range []sqlc.GetPathStatsRow{ + {HashBytes: 0}, {HashBytes: 4}, {HashBytes: 1, Entries: -1}, {HashBytes: 1, Entries: 64}, + {HashBytes: 4, IsHourly: true}, + } { + store := &Store{q: pathBucketQuerier{row: row}} + if _, err := store.GetPathStats(context.Background(), time.Time{}, time.Time{}, nil); err == nil { + t.Fatalf("invalid bucket accepted: %+v", row) + } + } +} + +type pathRowsQuerier struct { + sqlc.Querier + rows []sqlc.GetPathStatsRow +} + +func (q pathRowsQuerier) GetPathStats(context.Context, sqlc.GetPathStatsParams) ([]sqlc.GetPathStatsRow, error) { + return q.rows, nil +} + +// An hour's maxEntries is the largest across its category/width rows, not the last one seen. +func TestPathStatsHourlyMaxEntries(t *testing.T) { + hour := pgtype.Timestamptz{Time: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), Valid: true} + store := &Store{q: pathRowsQuerier{rows: []sqlc.GetPathStatsRow{ + {IsHourly: true, Hour: hour, Category: 0, HashBytes: 1, Receptions: 3, MaxEntries: 9}, + {IsHourly: true, Hour: hour, Category: 0, HashBytes: 2, Receptions: 1, MaxEntries: 5}, + {IsHourly: true, Hour: hour, Category: 1, HashBytes: 0, Receptions: 2, MaxEntries: 0}, + }}} + got, err := store.GetPathStats(context.Background(), time.Time{}, time.Time{}, nil) + if err != nil { + t.Fatal(err) + } + if len(got.Hourly) != 1 || got.Hourly[0].MaxEntries != 9 { + t.Errorf("hourly %+v, want one hour with maxEntries 9", got.Hourly) + } +} diff --git a/db/presence.go b/db/presence.go index ea2d6fea..9120d540 100644 --- a/db/presence.go +++ b/db/presence.go @@ -12,19 +12,22 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error { +func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, iatas []string, iataAts []time.Time) error { return s.q.TouchObservers(ctx, sqlc.TouchObserversParams{ Column1: ids, Column2: toTimestamptzs(seen), Column3: counts, + Column4: iatas, + Column5: nullableTimes(iataAts), }) } -func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error { +func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error { return s.q.TouchObserverBrokers(ctx, sqlc.TouchObserverBrokersParams{ Column1: ids, Column2: brokers, Column3: toTimestamptzs(seen), + Column4: nullableTimes(packets), }) } @@ -42,3 +45,11 @@ func toTimestamptzs(ts []time.Time) []pgtype.Timestamptz { } return out } + +func nullableTimes(ts []time.Time) []pgtype.Timestamptz { + out := make([]pgtype.Timestamptz, len(ts)) + for i, t := range ts { + out[i] = pgtype.Timestamptz{Time: t, Valid: !t.IsZero()} + } + return out +} diff --git a/db/queries/observer_comparison.sql b/db/queries/observer_comparison.sql new file mode 100644 index 00000000..dab05da1 --- /dev/null +++ b/db/queries/observer_comparison.sql @@ -0,0 +1,23 @@ +-- name: GetObserverComparison :one +-- Count packet identities, not reception rows: a broker repeat or another +-- reception by the same observer must not inflate the comparison. +WITH heard AS ( + SELECT po.packet_hash, + bool_or(po.observer_id = @observer_a::uuid) AS seen_a, + bool_or(po.observer_id = @observer_b::uuid) AS seen_b + FROM packet_observations po + JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.observer_id IN (@observer_a::uuid, @observer_b::uuid) + AND po.heard_at >= @since::timestamptz + AND po.heard_at < @until::timestamptz + AND p.route_type IN (0, 1) + AND (COALESCE(cardinality(@iatas::text[]), 0) = 0 OR po.iata = ANY(@iatas::text[])) + GROUP BY po.packet_hash +) +SELECT count(*) AS total_packets, + count(*) FILTER (WHERE seen_a AND NOT seen_b) AS only_a, + count(*) FILTER (WHERE seen_b AND NOT seen_a) AS only_b, + count(*) FILTER (WHERE seen_a AND seen_b) AS both, + EXISTS (SELECT 1 FROM observers WHERE id = @observer_a::uuid) AS known_a, + EXISTS (SELECT 1 FROM observers WHERE id = @observer_b::uuid) AS known_b +FROM heard; diff --git a/db/queries/paths.sql b/db/queries/paths.sql new file mode 100644 index 00000000..effe72f3 --- /dev/null +++ b/db/queries/paths.sql @@ -0,0 +1,21 @@ +-- name: GetPathStats :many +-- All HTTP aggregates read the hourly rollup, never observations. +WITH readings AS ( + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz + AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz + AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) +) +SELECT (grouping(hour) = 0)::boolean AS is_hourly, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + category, hash_bytes, COALESCE(entries, 0)::integer AS entries, + sum(receptions)::bigint AS receptions, + -- Non-routed categories store entries = 0, so an hour of only empty paths gives 0. + COALESCE(max(entries) FILTER (WHERE receptions > 0), 0)::integer AS max_entries +FROM readings +GROUP BY GROUPING SETS ((category, hash_bytes, entries), (hour, category, hash_bytes)) +ORDER BY is_hourly, hour, category, hash_bytes, entries; + diff --git a/db/queries/queries.sql b/db/queries/queries.sql index b2253b3f..2a2c470b 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -10,12 +10,20 @@ INSERT INTO iata_codes (iata) VALUES ($1) ON CONFLICT (iata) DO NOTHING; +-- name: AddIATAs :exec +INSERT INTO iata_codes (iata) +SELECT unnest(@iatas::bpchar[]) +ON CONFLICT (iata) DO NOTHING; + -- name: GetIATA :one SELECT * FROM iata_codes WHERE iata = $1; -- name: ListIATAs :many SELECT * FROM iata_codes ORDER BY iata; +-- name: ListHeardIATAs :many +SELECT DISTINCT last_iata::text AS iata FROM observers WHERE last_iata IS NOT NULL ORDER BY 1; + -- name: UpsertIATADetails :exec INSERT INTO iata_codes (iata, display_name, approx_lat, approx_lng) VALUES ($1, $2, $3, $4) @@ -25,10 +33,13 @@ ON CONFLICT (iata) DO UPDATE SET approx_lng = EXCLUDED.approx_lng; -- name: GetIATABorder :one --- border is NULL when the IATA exists but has no border configured; a --- missing row (unknown IATA) is sql.ErrNoRows, same not-found distinction --- GetIATA already makes. -SELECT border FROM iata_codes WHERE iata = $1; +-- An imported MeshMapper boundary overrides the configured one. border is NULL +-- when neither exists; a missing row (unknown IATA) is sql.ErrNoRows, same +-- not-found distinction GetIATA already makes. +SELECT COALESCE(z.feature, i.border)::jsonb AS border +FROM iata_codes i +LEFT JOIN meshmapper_zone_boundaries z ON z.iata = i.iata +WHERE i.iata = $1; -- name: UpsertIATABorder :exec -- Written by the config-file-driven seeder (internal/config/seed.go), not a @@ -49,10 +60,11 @@ VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO UPDATE SET display_name = EXCLUDED.display_name, transport_key = EXCLUDED.transport_key, - key_fingerprint = EXCLUDED.key_fingerprint; + key_fingerprint = EXCLUDED.key_fingerprint, + imported_only = FALSE; -- name: GetTransportScopes :many -SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name; +SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name; -- name: GetTransportScopeByName :one SELECT id FROM transport_scopes WHERE name = $1; @@ -60,48 +72,40 @@ SELECT id FROM transport_scopes WHERE name = $1; -- name: GetScopeNames :many SELECT name FROM transport_scopes ORDER BY name; --- name: GetScopesByIATAs :many -SELECT - ts.name, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count -FROM transport_scopes ts -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])) -GROUP BY ts.name -ORDER BY ts.name; - -- name: GetScopeByName :one +-- Packets and IATAs cover every retained rollup hour; IATAs are those that heard the +-- scope's own packets. Observer and node counts are current memberships. SELECT ts.name, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count, - array_remove(array_agg(DISTINCT po.iata ORDER BY po.iata), NULL)::text[] AS iatas + COALESCE((SELECT SUM(s.packets) FROM analytics_hourly_scope_sets s WHERE s.scope_id = ts.id), 0)::bigint AS packet_count, + (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id)::bigint AS observer_count, + (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id)::bigint AS node_count, + cardinality(i.iatas)::bigint AS iata_count, + i.iatas FROM transport_scopes ts -LEFT JOIN packets p ON p.scope_id = ts.id -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -WHERE ts.name = $1 -GROUP BY ts.name; +CROSS JOIN LATERAL ( + SELECT COALESCE(array_agg(DISTINCT x.iata::text ORDER BY x.iata::text), '{}')::text[] AS iatas + FROM analytics_hourly_scope_sets s CROSS JOIN LATERAL unnest(s.iatas) AS x(iata) + WHERE s.scope_id = ts.id +) i +WHERE ts.name = $1; -- ============================================================ -- OBSERVERS -- ============================================================ -- name: UpsertObserver :one -INSERT INTO observers (public_key, observer_type, last_seen) -VALUES ($1, 'unknown', NOW()) +-- Empty iata (status/neighbors) leaves last_iata alone; otherwise the newest iata_at wins. +INSERT INTO observers (public_key, observer_type, last_seen, last_iata, last_iata_at) +VALUES ($1, 'unknown', NOW(), NULLIF(sqlc.arg(iata)::text, ''), + CASE WHEN sqlc.arg(iata)::text <> '' THEN sqlc.arg(iata_at)::timestamptz END) ON CONFLICT (public_key) DO UPDATE SET last_seen = NOW(), - observation_count = observers.observation_count + 1 + observation_count = observers.observation_count + 1, + last_iata = CASE WHEN EXCLUDED.last_iata IS NOT NULL + AND (observers.last_iata_at IS NULL OR EXCLUDED.last_iata_at > observers.last_iata_at) + THEN EXCLUDED.last_iata ELSE observers.last_iata END, + last_iata_at = GREATEST(observers.last_iata_at, EXCLUDED.last_iata_at) RETURNING *; -- name: UpdateObserverStatus :one @@ -129,11 +133,16 @@ RETURNING id; -- from regressing a newer write-through (e.g. a status update). UPDATE observers o SET last_seen = GREATEST(o.last_seen, v.seen), - observation_count = COALESCE(o.observation_count, 0) + v.delta + observation_count = COALESCE(o.observation_count, 0) + v.delta, + last_iata = CASE WHEN v.iata <> '' AND (o.last_iata_at IS NULL OR v.iata_at > o.last_iata_at) + THEN v.iata ELSE o.last_iata END, + last_iata_at = GREATEST(o.last_iata_at, v.iata_at) FROM ( SELECT unnest($1::uuid[]) AS id, unnest($2::timestamptz[]) AS seen, - unnest($3::int[]) AS delta + unnest($3::int[]) AS delta, + unnest($4::text[]) AS iata, + unnest($5::timestamptz[]) AS iata_at ) v WHERE o.id = v.id; @@ -177,23 +186,13 @@ COALESCE(CASE WHEN GREATEST(COALESCE(o.last_status_at, o.last_seen), o.last_seen) > NOW() - INTERVAL '5 minutes' THEN 'online' ELSE 'offline' END, 'offline')::text AS status, -COALESCE(( - SELECT po.iata - FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC - LIMIT 1 -), '')::text AS iata +COALESCE(o.last_iata, '')::text AS iata FROM observers o LEFT JOIN observer_brokers ob ON ob.observer_id = o.id LEFT JOIN observer_scopes os ON os.observer_id = o.id LEFT JOIN transport_scopes ts ON ts.id = os.scope_id WHERE - (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ( - SELECT po.iata FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC LIMIT 1 - ) = ANY($1::bpchar[])) + (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR o.last_iata = ANY($1::bpchar[])) AND ($2 = '' OR o.observer_type = $2) AND ($3 = '' OR ob.broker_name = $3) AND ($4 = '' OR CASE @@ -212,10 +211,7 @@ ORDER BY o.last_seen DESC LIMIT $7; -- name: GetObserverLastIATA :one -SELECT iata FROM packet_observations -WHERE observer_id = $1 -ORDER BY heard_at DESC -LIMIT 1; +SELECT COALESCE(last_iata, '')::text FROM observers WHERE id = $1; -- name: GetObserverRadio :one SELECT radio_freq_mhz, radio_bw_khz, radio_sf, radio_cr @@ -226,14 +222,14 @@ WHERE id = $1; -- Inserts a telemetry snapshot for an observer. The reported_at timestamp should -- be truncated to the configured resolution before calling to ensure deduplication. INSERT INTO observer_telemetry ( - observer_id, reported_at, battery_voltage_mv, airtime_tx_pct, - airtime_rx_pct, noise_floor_db, uptime_seconds, queue_length, + observer_id, reported_at, battery_voltage_mv, airtime_tx_secs, + airtime_rx_secs, noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (observer_id, reported_at) DO NOTHING; -- name: GetObserverTelemetry :many -SELECT id, reported_at, battery_voltage_mv, airtime_tx_pct, airtime_rx_pct, +SELECT id, reported_at, battery_voltage_mv, airtime_tx_secs, airtime_rx_secs, noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors FROM observer_telemetry WHERE observer_id = $1 @@ -242,27 +238,81 @@ WHERE observer_id = $1 AND ($4 = 0 OR id > $4) ORDER BY reported_at ASC; --- name: GetObserverTelemetryBucketed :many +-- name: GetObserverActivityRaw :many +-- Sub-hour activity buckets straight off idx_observations_observer; no join to packets. +-- Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as +-- NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. SELECT - (date_trunc('day', reported_at) + - (EXTRACT(HOUR FROM reported_at)::int / $4::int) * ($4::int * interval '1 hour'))::timestamptz AS bucket, - AVG(battery_voltage_mv)::int AS battery_voltage_mv, - GREATEST(MAX(airtime_tx_pct) - MIN(airtime_tx_pct), 0)::real AS airtime_tx_pct, - GREATEST(MAX(airtime_rx_pct) - MIN(airtime_rx_pct), 0)::real AS airtime_rx_pct, - AVG(noise_floor_db)::real AS noise_floor_db, - MAX(uptime_seconds)::bigint AS uptime_seconds, - AVG(queue_length)::int AS queue_length, - GREATEST(MAX(receive_errors) - MIN(receive_errors), 0)::int AS receive_errors -FROM observer_telemetry -WHERE observer_id = $1 - AND ($2::timestamptz IS NULL OR reported_at >= $2) - AND ($3::timestamptz IS NULL OR reported_at <= $3) + date_bin($3::interval, heard_at, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + COALESCE(AVG(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS snr_avg, + COALESCE(MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS snr_min, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz GROUP BY bucket -ORDER BY bucket ASC; +ORDER BY bucket; + +-- name: GetObserverActivityRawPayloadTypes :many +SELECT payload_type, COUNT(*)::bigint AS count +FROM packet_observations +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz +GROUP BY payload_type +ORDER BY count DESC; + +-- name: GetObserverActivityHourly :many +-- Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. +-- Hours from @tail_start on aren't rolled yet and read raw rows. +WITH src AS ( + SELECT a.hour AS t, a.observations, a.airtime_ms, a.airtime_n, a.snr_sum, a.snr_n, a.snr_min, a.rssi_sum, a.rssi_n + FROM analytics_hourly_observer_activity a + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST(@until::timestamptz, @tail_start::timestamptz) + UNION ALL + SELECT o.heard_at, 1::bigint, o.airtime_ms, (o.airtime_ms IS NOT NULL)::int::bigint, + s.snr, (s.snr IS NOT NULL)::int::bigint, s.snr, s.rssi::bigint, (s.rssi IS NOT NULL)::int::bigint + FROM packet_observations o CROSS JOIN + LATERAL (SELECT CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.snr END AS snr, + CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.rssi END AS rssi) s + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, @tail_start::timestamptz) AND o.heard_at < @until::timestamptz +) +SELECT + date_bin($3::interval, t, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + SUM(observations)::bigint AS observations, + COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, + SUM(airtime_n)::bigint AS airtime_n, + COALESCE(SUM(snr_sum), 0)::real AS snr_sum, + SUM(snr_n)::bigint AS snr_n, + COALESCE(MIN(snr_min), 0)::real AS snr_min, + COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, + SUM(rssi_n)::bigint AS rssi_n +FROM src +GROUP BY 1 +ORDER BY 1; + +-- name: GetObserverActivityHourlyPayloadTypes :many +-- Same rollup/raw-tail split as GetObserverActivityHourly. +WITH src AS ( + SELECT a.payload_type, a.observations AS n + FROM analytics_hourly_observer_activity a + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST(@until::timestamptz, @tail_start::timestamptz) + UNION ALL + SELECT COALESCE(o.payload_type, -1)::smallint, 1::bigint + FROM packet_observations o + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, @tail_start::timestamptz) AND o.heard_at < @until::timestamptz +) +SELECT payload_type, SUM(n)::bigint AS count +FROM src +GROUP BY payload_type +ORDER BY count DESC; -- name: ListObserverAdverts :many -- Returns advert packets (payload_type=4) heard by a specific observer. -- Pass cursor=0 to start from the beginning, or the last seen id for pagination. +-- Keep missing-origin adverts; the generated key field expects a string, not NULL. SELECT po.id, encode(po.packet_hash, 'hex') AS packet_hash_hex, @@ -273,7 +323,7 @@ SELECT po.snr, po.hop_count, n.name AS node_name, - encode(p.origin_pubkey, 'hex') AS node_public_key + COALESCE(encode(p.origin_pubkey, 'hex'), '')::text AS node_public_key FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash LEFT JOIN nodes n ON n.public_key = p.origin_pubkey @@ -287,25 +337,45 @@ LIMIT $3; -- Deletes telemetry rows older than the given cutoff. Called by the cleanup goroutine. DELETE FROM observer_telemetry WHERE reported_at < $1; +-- name: DeleteOldObservers :many +-- Opt-in age-out: preserve retained history and manually recorded ownership. +-- Bound deletions per cleanup tick and skip observers being updated by ingest. +WITH expired AS ( + SELECT o.id + FROM observers o + WHERE o.last_seen < $1 + AND (o.last_status_at IS NULL OR o.last_status_at < $1) + AND NOT EXISTS (SELECT 1 FROM packet_observations po WHERE po.observer_id = o.id) + AND NOT EXISTS (SELECT 1 FROM observer_telemetry ot WHERE ot.observer_id = o.id) + AND NOT EXISTS (SELECT 1 FROM observer_owners oo WHERE oo.observer_id = o.id) + ORDER BY o.last_seen, o.id + LIMIT 1000 + FOR UPDATE OF o SKIP LOCKED +) +DELETE FROM observers o USING expired e +WHERE o.id = e.id +RETURNING o.id; + -- ============================================================ -- OBSERVER BROKERS -- ============================================================ -- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN @is_packet::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW(); + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at); -- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name; @@ -314,6 +384,9 @@ WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name; -- ============================================================ -- name: UpsertPacket :one +-- A new TRACE packet adds itself to its tag summary in the same statement, so the summary +-- can't miss a stored packet. 'TRACE' > 'PING'; the longest path keeps the best payload. +WITH up AS ( INSERT INTO packets ( packet_hash, payload_type, @@ -336,8 +409,25 @@ INSERT INTO packets ( ) ON CONFLICT (packet_hash) DO UPDATE SET last_heard_at = NOW() -RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, (xmax = 0) -AS inserted; +RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, trace_tag, scope_id, (xmax = 0) +AS inserted +), trace AS ( + INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, packet_count, trace_type, scope_id, best_payload, best_path_len) + SELECT u.trace_tag, u.first_heard_at, u.last_heard_at, 1, u.parsed_payload->>'type', u.scope_id, u.parsed_payload, + CASE WHEN jsonb_typeof(u.parsed_payload->'pathHashes') = 'array' + THEN jsonb_array_length(u.parsed_payload->'pathHashes') ELSE 0 END + FROM up u + WHERE u.inserted AND u.trace_tag IS NOT NULL + ON CONFLICT (trace_tag) DO UPDATE SET + packet_count = trace_tags.packet_count + 1, + trace_type = GREATEST(trace_tags.trace_type, EXCLUDED.trace_type), + scope_id = COALESCE(trace_tags.scope_id, EXCLUDED.scope_id), + best_payload = CASE WHEN EXCLUDED.best_path_len > trace_tags.best_path_len + THEN EXCLUDED.best_payload ELSE trace_tags.best_payload END, + best_path_len = GREATEST(trace_tags.best_path_len, EXCLUDED.best_path_len) +) +SELECT packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, inserted +FROM up; -- name: SetPacketDecrypted :exec UPDATE packets SET decrypted = true WHERE packet_hash = $1; @@ -362,22 +452,27 @@ LEFT JOIN channel_messages cm ON cm.packet_hash = p.packet_hash WHERE p.packet_hash = $1; -- name: GetPacketsByTraceTag :many --- Returns all packets for a given trace tag with observations. +-- Return distinct observation IATAs in first-heard order for path resolution, +-- without fetching full observations separately for every trace packet. SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, p.route_type, p.first_heard_at, p.last_heard_at, p.parsed_payload, p.scope_id, - ts.name AS scope_name + ts.name AS scope_name, + ARRAY( + SELECT po.iata + FROM packet_observations po + WHERE po.packet_hash = p.packet_hash + GROUP BY po.iata + ORDER BY MIN(po.heard_at), po.iata + )::bpchar[] AS iatas FROM packets p LEFT JOIN transport_scopes ts ON ts.id = p.scope_id WHERE p.trace_tag = decode($1, 'hex') ORDER BY p.first_heard_at ASC; --- name: GetPacketObservationCount :one -SELECT COUNT(*) FROM packet_observations WHERE packet_hash = $1; - -- name: ListPackets :many -- Returns packets with the latest observation rolled in for display. -- Pass cursor=0 to start from the beginning. IATA-filtered requests are @@ -385,18 +480,37 @@ SELECT COUNT(*) FROM packet_observations WHERE packet_hash = $1; SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, + -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, + p.raw_payload, + p.origin_pubkey FROM packets p LEFT JOIN LATERAL ( - SELECT observer_id, iata + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -420,25 +534,14 @@ LIMIT $6; -- to fill a page for a quiet site; walking the site's own observation log is -- proportional to the page size instead. Results are ordered by when the -- requested sites heard the packet (site-local recency) and the cursor --- follows that ordering. scan_depth is a multiple of the page size to --- absorb per-observer duplicates; if duplication exceeds it across a --- page, pagination ends early (hasMore=false) rather than returning a --- short page, even though deeper matches exist. -SELECT - p.packet_hash, - p.payload_type, - p.route_type, - p.first_heard_at, - p.last_heard_at, - p.scope_id, - ts.name AS scope_name, - sh.site_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, - po.observer_id AS latest_observer_id, - o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata -FROM ( - SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at +-- follows that ordering. scan_depth caps how deep each site's observation +-- log is walked. A packet repeats once per observer that heard it, so a +-- page can collapse to fewer distinct packets than were asked for without +-- the site being exhausted. scan_saturated reports whether any site hit +-- that cap and scan_floor the oldest heard_at they all cover, so a short +-- page can keep paging instead of reading as the end of the data. +WITH scanned AS ( + SELECT req.iata AS req_iata, hits.packet_hash, hits.heard_at FROM unnest(@iatas::bpchar[]) AS req(iata) CROSS JOIN LATERAL ( SELECT po3.packet_hash, po3.heard_at @@ -456,18 +559,71 @@ FROM ( ORDER BY po3.heard_at DESC LIMIT @scan_depth ) hits - GROUP BY hits.packet_hash +), +-- A site that filled scan_depth still has unread history below its floor. +-- The newest such floor is the point above which every site is covered. +saturation AS ( + SELECT + COUNT(*) > 0 AS scan_saturated, + MAX(floor_ts)::timestamptz AS scan_floor + FROM ( + SELECT MIN(heard_at) AS floor_ts + FROM scanned + GROUP BY req_iata + HAVING COUNT(*) >= @scan_depth + ) filled +), +page AS ( + SELECT scanned.packet_hash, MAX(scanned.heard_at)::timestamptz AS site_heard_at + FROM scanned + GROUP BY scanned.packet_hash HAVING (@cursor_ts::timestamptz IS NULL OR NOT EXISTS ( SELECT 1 FROM packet_observations px - WHERE px.packet_hash = hits.packet_hash + WHERE px.packet_hash = scanned.packet_hash AND px.iata = ANY(@iatas::bpchar[]) AND px.heard_at >= @cursor_ts)) ORDER BY site_heard_at DESC LIMIT @page_limit -) sh +) +SELECT + p.packet_hash, + p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, + p.last_heard_at, + p.scope_id, + ts.name AS scope_name, + sh.site_heard_at, + sat.scan_saturated, + sat.scan_floor, + p.observation_count AS observation_count, + -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. + po.observer_id AS latest_observer_id, + o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, + p.raw_payload, + p.origin_pubkey +FROM page sh +CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( - SELECT observer_id, iata + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -483,13 +639,31 @@ ORDER BY sh.site_heard_at DESC; SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, + p.raw_payload, + p.origin_pubkey, ts.name AS scope_name FROM packets p JOIN packet_observations po ON po.packet_hash = p.packet_hash @@ -503,11 +677,24 @@ WHERE po.id > $1 ORDER BY po.id ASC LIMIT $6; - --- name: DeleteOldPackets :exec --- Deletes packets and their observations older than the given cutoff. --- packet_observations cascade-delete via FK. -DELETE FROM packets WHERE last_heard_at < $1; +-- name: DeleteOldPackets :one +-- Deletes one bounded cohort (observations cascade). SKIP LOCKED leaves packets an in-flight +-- observation insert holds. raw_deleted_before tells the rollup which hours lost raw rows. +WITH victims AS MATERIALIZED ( + SELECT vp.packet_hash, vp.last_heard_at FROM packets vp + WHERE vp.last_heard_at < @cutoff::timestamptz + ORDER BY vp.last_heard_at, vp.packet_hash + LIMIT @batch_size::integer + FOR UPDATE OF vp SKIP LOCKED +), deleted AS ( + DELETE FROM packets dp USING victims v WHERE dp.packet_hash = v.packet_hash + RETURNING dp.last_heard_at +), mark AS ( + UPDATE analytics_raw_state + SET raw_deleted_before = GREATEST(raw_deleted_before, (SELECT max(last_heard_at) FROM deleted)) + WHERE EXISTS (SELECT 1 FROM deleted) +) +SELECT count(*) FROM deleted; -- name: DeleteOldNodes :exec -- Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- @@ -521,6 +708,21 @@ DELETE FROM nodes WHERE last_seen < $1 AND id NOT IN (SELECT owner_node_id FROM observer_owners WHERE owner_node_id IS NOT NULL); +-- name: DeleteOldRoutes :execrows +-- One batch of routes past retention, or past grace with too few observations. +-- GREATEST keeps the scan on idx_known_routes_last_seen. +WITH expired AS ( + SELECT r.iata, r.path_key + FROM known_routes r + WHERE r.last_seen < GREATEST(@retention_cutoff::timestamptz, @grace_cutoff::timestamptz) + AND (r.last_seen < @retention_cutoff OR + (r.observation_count < @min_observations AND r.last_seen < @grace_cutoff)) + LIMIT @batch_size + FOR UPDATE OF r SKIP LOCKED +) +DELETE FROM known_routes kr USING expired e +WHERE kr.iata = e.iata AND kr.path_key = e.path_key; + -- name: DeleteOldChannelIATAs :exec -- Keeps the channel IATA filter in step with packet retention. DELETE FROM channel_iatas WHERE last_heard < $1; @@ -534,6 +736,8 @@ DELETE FROM trace_iatas WHERE last_heard < $1; -- ============================================================ -- name: InsertObservation :one +-- Bumps packets.observation_count only for a new row; a duplicate returns the current count. +WITH ins AS ( INSERT INTO packet_observations ( packet_hash, observer_id, @@ -551,12 +755,21 @@ INSERT INTO packet_observations ( bandwidth_khz, coding_rate, source_broker, - payload_type + payload_type, + airtime_ms ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING -RETURNING *; +RETURNING packet_hash +), bump AS ( + UPDATE packets p SET observation_count = p.observation_count + 1 + FROM ins WHERE p.packet_hash = ins.packet_hash + RETURNING p.observation_count +) +SELECT EXISTS (SELECT 1 FROM ins) AS inserted, + COALESCE((SELECT b.observation_count FROM bump b), + (SELECT pk.observation_count FROM packets pk WHERE pk.packet_hash = $1), 0)::bigint AS observation_count; -- name: ListObservationsForPacket :many SELECT po.*, o.display_name AS observer_name @@ -571,13 +784,13 @@ ORDER BY po.heard_at ASC; -- name: UpsertNode :one INSERT INTO nodes (public_key, node_type, name, latitude, longitude, location_source, last_advert_at, last_seen, radio_freq_mhz, radio_sf, radio_bw_khz, device_clock_drift_seconds) -VALUES ($1, $2, $3, $4, $5, 'advert', NOW(), NOW(), $6, $7, $8, $9) +VALUES (@public_key, @node_type, @name, @latitude, @longitude, CASE WHEN @latitude::double precision IS NOT NULL THEN 'advert' END, NOW(), NOW(), @radio_freq_mhz, @radio_sf, @radio_bw_khz, @device_clock_drift_seconds) ON CONFLICT (public_key) DO UPDATE SET node_type = EXCLUDED.node_type, name = COALESCE(EXCLUDED.name, nodes.name), - latitude = COALESCE(EXCLUDED.latitude, nodes.latitude), - longitude = COALESCE(EXCLUDED.longitude, nodes.longitude), - location_source = CASE WHEN EXCLUDED.latitude IS NOT NULL THEN 'advert' ELSE nodes.location_source END, + latitude = CASE WHEN @clear_location::bool THEN NULL ELSE COALESCE(EXCLUDED.latitude, nodes.latitude) END, + longitude = CASE WHEN @clear_location::bool THEN NULL ELSE COALESCE(EXCLUDED.longitude, nodes.longitude) END, + location_source = CASE WHEN @clear_location::bool THEN NULL WHEN EXCLUDED.latitude IS NOT NULL THEN 'advert' ELSE nodes.location_source END, last_advert_at = NOW(), last_seen = NOW(), radio_freq_mhz = EXCLUDED.radio_freq_mhz, @@ -616,21 +829,17 @@ WHERE id = ANY($1::uuid[]); -- name: GetNodeByPubkey :one SELECT id FROM nodes WHERE public_key = $1; +-- name: GetNodesByPubkeys :many +SELECT id, public_key, name, latitude, longitude +FROM nodes +WHERE public_key = ANY(@pubkeys::bytea[]); + -- name: ListNodes :many +-- Limit the filtered node page before enriching IATA membership and neighbours. +WITH page AS ( SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, - n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, - ts.name AS default_scope_name, - json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) FILTER (WHERE ni.iata IS NOT NULL) AS iatas, - EXISTS (SELECT 1 FROM observers o WHERE o.public_key = n.public_key) AS is_observer, - (SELECT o.id FROM observers o WHERE o.public_key = n.public_key LIMIT 1) AS observer_id, - (SELECT COUNT(DISTINCT nn.neighbor_id) FROM node_neighbors nn WHERE nn.node_id = n.id)::bigint AS known_neighbor_count, - -- CASE short-circuits: the array_agg subquery only runs when $10 is true, - -- so requests that don't ask for neighbor IDs don't pay for it. -(CASE WHEN $10::bool THEN - (SELECT COALESCE(array_agg(DISTINCT nn.neighbor_id), '{}'::uuid[]) FROM node_neighbors nn WHERE nn.node_id = n.id) - ELSE NULL END)::uuid[] AS neighbor_ids + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, ts.name AS default_scope_name FROM nodes n -LEFT JOIN node_iatas ni ON ni.node_id = n.id LEFT JOIN transport_scopes ts ON ts.id = n.default_scope_id WHERE ($1 = 0 OR n.node_type = $1) @@ -650,9 +859,21 @@ WHERE AND ($7::timestamptz IS NULL OR n.last_seen < $7) AND ($9::text = '' OR ts.name = $9::text) AND ($11::text = '' OR encode(n.public_key, 'hex') ILIKE $11 || '%') -GROUP BY n.id, ts.name ORDER BY n.last_seen DESC -LIMIT $8; +LIMIT $8 +) +SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, n.default_scope_name, + (SELECT json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) + FROM node_iatas ni WHERE ni.node_id = n.id) AS iatas, + EXISTS (SELECT 1 FROM observers o WHERE o.public_key = n.public_key) AS is_observer, + (SELECT o.id FROM observers o WHERE o.public_key = n.public_key LIMIT 1) AS observer_id, + (SELECT COUNT(DISTINCT nn.neighbor_id) FROM node_neighbors nn WHERE nn.node_id = n.id)::bigint AS known_neighbor_count, + (CASE WHEN $10::bool THEN + (SELECT COALESCE(array_agg(DISTINCT nn.neighbor_id), '{}'::uuid[]) FROM node_neighbors nn WHERE nn.node_id = n.id) + ELSE NULL END)::uuid[] AS neighbor_ids +FROM page n +ORDER BY n.last_seen DESC; -- name: ListNodeObservations :many SELECT po.id, encode(po.packet_hash, 'hex') AS packet_hash_hex, @@ -692,8 +913,7 @@ INSERT INTO channels (channel_hash, key_fingerprint, name, hashtag, is_hashtag, VALUES ($1, $2::bytea, $3, $4, $5, ($2 IS NOT NULL), NOW()) ON CONFLICT (channel_hash, key_fingerprint) DO UPDATE SET last_seen = NOW(), - name = COALESCE(EXCLUDED.name, channels.name), - message_count = CASE WHEN $6 THEN channels.message_count + 1 ELSE channels.message_count END + name = COALESCE(EXCLUDED.name, channels.name) RETURNING *; -- name: UpsertChannelHashOnly :one @@ -711,6 +931,19 @@ RETURNING id; SELECT packet_hash, raw_payload FROM packets WHERE payload_type = 5 AND decrypted IS NOT TRUE; +-- name: ListUndecryptedGroupTextPacketsByHash :many +-- Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key. +SELECT packet_hash, raw_payload FROM packets +WHERE payload_type = 5 AND decrypted IS NOT TRUE AND channel_hash = ANY(@hashes::bytea[]); + +-- name: DeleteChannelConfigScopes :exec +DELETE FROM channel_config_scopes; + +-- name: AddChannelConfigScopes :exec +-- An empty region places the channel Beacon-wide. +INSERT INTO channel_config_scopes (key_fingerprint, region_slug) +SELECT unnest(@fingerprints::bytea[]), NULLIF(unnest(@regions::text[]), ''); + -- name: UpsertChannelIATA :exec -- Refreshes at most hourly so repeat hears don't churn the row. INSERT INTO channel_iatas (channel_hash, iata, last_heard) @@ -719,49 +952,104 @@ ON CONFLICT (channel_hash, iata) DO UPDATE SET last_heard = EXCLUDED.last_heard WHERE EXCLUDED.last_heard > channel_iatas.last_heard + INTERVAL '1 hour'; --- name: UpsertTraceIATA :exec --- Refreshes at most hourly so repeat hears don't churn the row. -INSERT INTO trace_iatas (trace_tag, iata, last_heard) -VALUES ($1, $2, $3) -ON CONFLICT (trace_tag, iata) DO UPDATE SET - last_heard = EXCLUDED.last_heard -WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour'; +-- name: RecordTrace :exec +-- One hearing of a TRACE packet: trace_iatas refreshes at most hourly and the tag's heard +-- window widens (the first hearing replaces the packet's provisional times). Packet counts +-- and payloads come from UpsertPacket. +WITH iata AS ( + INSERT INTO trace_iatas (trace_tag, iata, last_heard) + VALUES (@trace_tag, @iata, @heard_at) + ON CONFLICT (trace_tag, iata) DO UPDATE SET + last_heard = EXCLUDED.last_heard + WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour' +) +INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, heard) +VALUES (@trace_tag, @heard_at, @heard_at, true) +ON CONFLICT (trace_tag) DO UPDATE SET + first_heard_at = CASE WHEN trace_tags.heard THEN LEAST(trace_tags.first_heard_at, EXCLUDED.first_heard_at) + ELSE EXCLUDED.first_heard_at END, + last_heard_at = CASE WHEN trace_tags.heard THEN GREATEST(trace_tags.last_heard_at, EXCLUDED.last_heard_at) + ELSE EXCLUDED.last_heard_at END, + heard = true; + +-- name: DeleteOldTraceTags :exec +DELETE FROM trace_tags WHERE last_heard_at < $1; -- name: ListChannels :many --- Channels ordered by last seen, optionally filtered by hash and/or IATAs --- (membership via channel_iatas). NULL hash / empty array skip those filters. +-- Channels ordered by last seen, optionally filtered by hash and/or IATAs. +-- A channel belongs to an IATA when MeshMapper lists it there or config scopes it +-- to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters. -- Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). SELECT c.* FROM channels c WHERE (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) - AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci - WHERE ci.iata = ANY(@iatas::bpchar[]) - )) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY(@iatas::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY(@iatas::bpchar[]))))) + AND (sqlc.narg(key_known)::boolean IS NULL OR COALESCE(c.key_known, false) = sqlc.narg(key_known)) AND (@cursor_ts::timestamptz IS NULL OR c.last_seen < @cursor_ts) -ORDER BY c.last_seen DESC +ORDER BY c.last_seen DESC, c.id DESC +LIMIT @page_limit; + +-- name: ListChannelsAfter :many +-- Keep the non-null tuple boundary separate from the legacy optional cursor so +-- generic prepared plans can seek directly into the composite ordered index. +SELECT c.* FROM channels c +WHERE (c.last_seen, c.id) < (@cursor_ts::timestamptz, @cursor_id::integer) + AND (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY(@iatas::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY(@iatas::bpchar[]))))) + AND (sqlc.narg(key_known)::boolean IS NULL OR COALESCE(c.key_known, false) = sqlc.narg(key_known)) +ORDER BY c.last_seen DESC, c.id DESC LIMIT @page_limit; -- name: GetChannelByID :one SELECT * FROM channels WHERE id = $1; - -- ============================================================ -- CHANNEL MESSAGES -- ============================================================ -- name: InsertChannelMessage :one -INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) -VALUES ($1, $2, $3, $4, $5) -ON CONFLICT (packet_hash) DO NOTHING -RETURNING id; +-- Read the immutable first-packet scope in the same statement as insertion. +-- A later reception's transport code must not give live and historical messages different tags. +-- message_count is a lifetime count, bumped only for a new message. A historical (backfilled) +-- message queues its observation hours for a re-roll in the same statement, so the two can't diverge. +WITH inserted AS ( + INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (packet_hash) DO NOTHING + RETURNING id, packet_hash, channel_id +), bump AS ( + UPDATE channels c SET message_count = c.message_count + 1 + FROM inserted WHERE c.id = inserted.channel_id +), dirty AS ( + INSERT INTO analytics_dirty_hours (hour) + SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC') + FROM packet_observations po JOIN inserted i ON i.packet_hash = po.packet_hash + WHERE $6::boolean + ON CONFLICT (hour) DO UPDATE SET enqueued_at = now() +) +SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present +FROM inserted +JOIN packets p ON p.packet_hash = inserted.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id; -- name: ListChannelMessages :many -- Returns messages for a channel identified by integer ID. -- Pass a zero/null timestamp for since to return all messages up to limit. -- Pass empty string for iata to skip IATA filtering. -- Pass cursor=0 to start from the beginning. -SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -779,8 +1067,8 @@ LIMIT $6; -- Returns all messages across all channels with optional time, IATA, scope and cursor filters. -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. -SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -798,8 +1086,8 @@ LIMIT $5; -- May return messages from multiple channels if the hash collides across different keys. -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. -SELECT DISTINCT ON (cm.id) cm.*, c.channel_hash, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +SELECT DISTINCT ON (cm.id) cm.*, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, + p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -816,8 +1104,8 @@ LIMIT $6; -- name: ListMessagesAfterID :many -- Returns messages after the given message ID, ordered oldest first. -- Used for WS reconnect backfill. -SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -833,38 +1121,41 @@ LIMIT $4; -- STATS -- ============================================================ --- name: GetStatsOverview :one -SELECT - COUNT(DISTINCT po.packet_hash) AS total_packets, - COUNT(*) AS total_observations, - COUNT(DISTINCT po.observer_id) AS active_observers, - COUNT(DISTINCT po.iata) AS active_iatas -FROM packet_observations po -WHERE po.heard_at > NOW() - INTERVAL '24 hours' - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])); - -- name: GetHourlyStats :many -SELECT iata, hour, observation_count, unique_packets, active_observers -FROM mv_hourly_iata_stats -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND hour >= NOW() - $2::interval +SELECT iata, hour, observation_count +FROM analytics_hourly_iata_observations +WHERE (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + AND hour >= @since::timestamptz ORDER BY iata, hour; -- name: GetTopNodes :many -SELECT * FROM mv_top_nodes_by_iata -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) -ORDER BY observation_count DESC -LIMIT $2; +-- Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; +-- node_id is NULL once the node row is gone. iata is a representative one. +WITH ranked AS ( + SELECT origin_pubkey, SUM(observations)::bigint AS observation_count, MAX(last_heard) AS last_heard, + MAX(iata) AS iata, MAX(name) AS name, MAX(node_type) AS node_type + FROM analytics_hourly_advert_hearings + WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + GROUP BY origin_pubkey + ORDER BY observation_count DESC, origin_pubkey + LIMIT @row_limit +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, r.name, '')::text AS name, COALESCE(n.node_type, r.node_type, 0)::smallint AS node_type, + r.iata::bpchar AS iata, r.observation_count, r.last_heard::timestamptz AS last_heard +FROM ranked r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.observation_count DESC, r.origin_pubkey; -- name: GetStatsPayloadBreakdown :many --- Payload-type counts for the IATA within the window, summed from the --- precomputed hourly buckets. +-- Payload-type observation counts since the given hour, from the hourly rollup. SELECT payload_type, SUM(count)::bigint AS count -FROM mv_payload_breakdown_by_iata -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND bucket >= NOW() - $2::interval +FROM analytics_hourly_payload_breakdown +WHERE (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + AND hour >= @since::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -880,49 +1171,93 @@ GROUP BY n.node_type ORDER BY count DESC; -- name: GetStatsTopObservers :many --- Top N observers for the IATA within the window, summed from the precomputed --- hourly buckets. Counts sum across matched IATAs; iata is a representative one. -SELECT - observer_id AS id, - display_name, - observer_type, - COALESCE(SUM(observation_count), 0)::bigint AS observation_count, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_observers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY observer_id, display_name, observer_type -ORDER BY observation_count DESC -LIMIT $3; +-- Top N observers since the given hour. Counts sum across matched IATAs; iata is a +-- representative one. Live names win over the rolled snapshot. +WITH ranked AS ( + SELECT observer_id, SUM(observation_count)::bigint AS observation_count, MAX(iata) AS iata, + MAX(display_name) AS display_name, MAX(observer_type) AS observer_type + FROM analytics_hourly_observer_identity + WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + GROUP BY observer_id + ORDER BY observation_count DESC, observer_id + LIMIT @row_limit +) +SELECT r.observer_id AS id, COALESCE(o.display_name, r.display_name, '')::text AS display_name, + COALESCE(o.observer_type, r.observer_type, '')::text AS observer_type, + r.observation_count, r.iata::bpchar AS iata +FROM ranked r +LEFT JOIN observers o ON o.id = r.observer_id +ORDER BY r.observation_count DESC, r.observer_id; -- name: GetStatsTopAdvertisers :many --- Top N advertisers in the window, summed from the hourly buckets. +-- Top N advertisers since the given hour. Each ADVERT packet counts once per hour heard, +-- however many of the requested IATAs heard it (IATA-set rollup). Live names win. +WITH ranked AS ( + SELECT origin_pubkey, SUM(advert_packets)::bigint AS advert_count, + SUM(flood_packets)::bigint AS flood_advert_count, SUM(direct_packets)::bigint AS direct_advert_count + FROM analytics_hourly_advert_sets + WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iatas && @iatas::bpchar[]) + GROUP BY origin_pubkey + ORDER BY advert_count DESC, origin_pubkey + LIMIT @row_limit +), heard AS ( + SELECT h.origin_pubkey, MAX(h.last_heard) AS last_heard, MAX(h.iata) AS iata, + MAX(h.name) AS name, MAX(h.node_type) AS node_type + FROM analytics_hourly_advert_hearings h + WHERE h.hour >= @since::timestamptz AND h.origin_pubkey IN (SELECT origin_pubkey FROM ranked) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR h.iata = ANY(@iatas::bpchar[])) + GROUP BY h.origin_pubkey +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, h.name, '')::text AS name, COALESCE(n.node_type, h.node_type, 0)::smallint AS node_type, + r.advert_count, r.flood_advert_count, r.direct_advert_count, + h.last_heard::timestamptz AS last_heard, COALESCE(h.iata, '')::bpchar AS iata +FROM ranked r +LEFT JOIN heard h ON h.origin_pubkey = r.origin_pubkey +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.advert_count DESC, r.origin_pubkey; + +-- name: GetStatsClockDrift :many +-- Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds +-- the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's +-- latest measured drift, not an aggregate over a period. +-- Select the filtered page before aggregating its IATA memberships. +WITH page AS ( SELECT - node_id AS id, - name, - node_type, - COALESCE(SUM(advert_count), 0)::bigint AS advert_count, - MAX(last_heard)::timestamptz AS last_heard, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_advertisers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY node_id, name, node_type -ORDER BY advert_count DESC -LIMIT $3; + n.id, + n.name, + n.node_type, + n.device_clock_drift_seconds, + n.last_advert_at +FROM nodes n +WHERE n.node_type IN (2, 3) + AND n.device_clock_drift_seconds IS NOT NULL + AND ABS(n.device_clock_drift_seconds) > $1::int + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) +ORDER BY ABS(n.device_clock_drift_seconds) DESC +LIMIT $3 +) +SELECT n.id, n.name, n.node_type, n.device_clock_drift_seconds, n.last_advert_at, + (SELECT json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) + FROM node_iatas ni WHERE ni.node_id = n.id) AS iatas +FROM page n +ORDER BY ABS(n.device_clock_drift_seconds) DESC; -- name: GetStatsTopTalkers :many --- Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. +-- Top N senders since the given hour. Each message counts once per hour heard, however many +-- of the requested IATAs heard it (IATA-set rollup). SELECT sender_name, - COALESCE(SUM(message_count), 0)::bigint AS message_count, + SUM(messages)::bigint AS message_count, MAX(last_sent)::timestamptz AS last_sent -FROM mv_top_talkers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) +FROM analytics_hourly_talker_sets +WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iatas && @iatas::bpchar[]) GROUP BY sender_name -ORDER BY message_count DESC -LIMIT $3; +ORDER BY message_count DESC, sender_name +LIMIT @row_limit; -- name: GetRadioPresets :many SELECT preset, iata, source_type, count @@ -931,15 +1266,73 @@ WHERE ($1::text = '' OR preset = $1::text) AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) ORDER BY preset, iata, source_type; +-- name: GetScopeStatsHourly :many +-- GetScopeStats packet counts split by hour (same window and IATA-set filter), with the +-- distinct observers and advertising nodes active in each scope that hour. Empty hours omitted. +WITH pk AS ( + SELECT s.scope_id, s.hour, SUM(s.packets)::bigint AS n + FROM analytics_hourly_scope_sets s + WHERE s.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR s.iatas && @iatas::bpchar[]) + GROUP BY s.scope_id, s.hour +), ob AS ( + SELECT o.scope_id, o.hour, COUNT(DISTINCT o.observer_id)::bigint AS n + FROM analytics_hourly_scope_observers o + WHERE o.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR o.iata = ANY(@iatas::bpchar[])) + GROUP BY o.scope_id, o.hour +), nd AS ( + SELECT d.scope_id, d.hour, COUNT(DISTINCT d.origin_pubkey)::bigint AS n + FROM analytics_hourly_scope_nodes d + WHERE d.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR d.iata = ANY(@iatas::bpchar[])) + GROUP BY d.scope_id, d.hour +), k AS ( + SELECT scope_id, hour FROM pk UNION SELECT scope_id, hour FROM ob UNION SELECT scope_id, hour FROM nd +) +SELECT ts.name, k.hour, + COALESCE(pk.n, 0)::bigint AS packets, + COALESCE(ob.n, 0)::bigint AS observers, + COALESCE(nd.n, 0)::bigint AS nodes +FROM k +JOIN transport_scopes ts ON ts.id = k.scope_id +LEFT JOIN pk ON pk.scope_id = k.scope_id AND pk.hour = k.hour +LEFT JOIN ob ON ob.scope_id = k.scope_id AND ob.hour = k.hour +LEFT JOIN nd ON nd.scope_id = k.scope_id AND nd.hour = k.hour +WHERE COALESCE(pk.n, 0) > 0 OR COALESCE(ob.n, 0) > 0 OR COALESCE(nd.n, 0) > 0 +ORDER BY ts.name, k.hour; + -- name: GetScopeStats :many --- Count each table on its own; the old cross-join blew up to millions of rows --- before COUNT(DISTINCT) (~10s). +-- Packets since the given hour come from the IATA-set rollup (counted once per hour heard). +-- Observer and node counts are current memberships; observers filter by their latest IATA. +WITH packet_counts AS ( + SELECT s.scope_id, SUM(s.packets)::bigint AS packet_count + FROM analytics_hourly_scope_sets s + WHERE s.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR s.iatas && @iatas::bpchar[]) + GROUP BY s.scope_id +), observer_counts AS ( + SELECT os.scope_id, COUNT(*)::bigint AS observer_count + FROM observer_scopes os JOIN observers o ON o.id = os.observer_id + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR o.last_iata = ANY(@iatas::bpchar[]) + GROUP BY os.scope_id +), node_counts AS ( + SELECT n.default_scope_id AS scope_id, COUNT(*)::bigint AS node_count + FROM nodes n + WHERE n.default_scope_id IS NOT NULL + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY(@iatas::bpchar[]))) + GROUP BY n.default_scope_id +) SELECT ts.name, - (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count, - (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count, - (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count + COALESCE(pc.packet_count, 0)::bigint AS packet_count, + COALESCE(oc.observer_count, 0)::bigint AS observer_count, + COALESCE(nc.node_count, 0)::bigint AS node_count FROM transport_scopes ts +LEFT JOIN packet_counts pc ON pc.scope_id = ts.id +LEFT JOIN observer_counts oc ON oc.scope_id = ts.id +LEFT JOIN node_counts nc ON nc.scope_id = ts.id ORDER BY ts.name; -- ============================================================ @@ -976,81 +1369,121 @@ ON CONFLICT (slug) DO UPDATE SET center_lat = EXCLUDED.center_lat, center_lng = EXCLUDED.center_lng, zoom_level = EXCLUDED.zoom_level, + imported = FALSE, -- config owns the slug from now on updated_at = NOW() RETURNING id; --- name: UpsertRegionIATA :exec +-- name: DeleteRegionIATAsNotIn :exec +DELETE FROM region_iatas WHERE region_id = @region_id AND NOT (iata = ANY(@keep::bpchar[])); + +-- name: AddRegionIATAs :exec INSERT INTO region_iatas (region_id, iata) -VALUES ($1, $2) +SELECT @region_id, unnest(@iatas::bpchar[]) ON CONFLICT (region_id, iata) DO NOTHING; +-- name: ListRegionState :many +-- Every region with its members, for reconciling imported MeshMapper groups. +SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, r.center_lat, r.center_lng, + COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas +FROM regions r +LEFT JOIN region_iatas ri ON ri.region_id = r.id +GROUP BY r.id +ORDER BY r.slug; + +-- name: UpsertImportedRegion :one +-- A hand-written region owns its slug: the WHERE turns a clash into no row. +INSERT INTO regions (slug, name, display_order, center_lat, center_lng, zoom_level, imported, updated_at) +VALUES (@slug, @name, @display_order, sqlc.narg(center_lat), sqlc.narg(center_lng), NULL, TRUE, NOW()) +ON CONFLICT (slug) DO UPDATE SET + name = EXCLUDED.name, + display_order = EXCLUDED.display_order, + center_lat = EXCLUDED.center_lat, + center_lng = EXCLUDED.center_lng, + updated_at = NOW() +WHERE regions.imported +RETURNING id; + +-- name: PruneImportedRegions :many +DELETE FROM regions WHERE imported AND NOT (slug = ANY(@keep::text[])) RETURNING slug; + -- ============================================================ -- TRACES -- ============================================================ -- name: ListTraceTags :many --- Returns distinct trace tags with summary info, ordered by most recent first. --- IATA membership comes from trace_iatas (joining observations here spilled the --- hash join). Per-tag details filled in only for the returned page. -WITH tags AS ( - SELECT - p.trace_tag, - MIN(p.first_heard_at) AS first_heard_at, - MAX(p.last_heard_at) AS last_heard_at, - COUNT(*) AS packet_count, - MAX(p.parsed_payload->>'type') AS trace_type - FROM packets p - WHERE p.trace_tag IS NOT NULL - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR p.trace_tag IN ( - SELECT ti.trace_tag FROM trace_iatas ti WHERE ti.iata = ANY($1::bpchar[]))) - AND ($2::text = '' OR p.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) - AND ($3::timestamptz IS NULL OR p.first_heard_at >= $3) - AND ($4::timestamptz IS NULL OR p.first_heard_at <= $4) - AND ($5::timestamptz IS NULL OR p.last_heard_at < $5) - AND ($7::text = '' OR p.parsed_payload->>'type' = $7) - GROUP BY p.trace_tag - ORDER BY MAX(p.last_heard_at) DESC - LIMIT $6 -) +-- Tags, most recent first. Filters match the tag summary (see RecordTrace). SELECT encode(t.trace_tag, 'hex') AS trace_tag, - t.first_heard_at::timestamptz AS first_heard_at, - t.last_heard_at::timestamptz AS last_heard_at, + t.first_heard_at, + t.last_heard_at, t.packet_count, (SELECT COUNT(*) FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, - t.trace_type::text AS trace_type, - (SELECT p3.parsed_payload - FROM packets p3 - WHERE p3.trace_tag = t.trace_tag - ORDER BY jsonb_array_length(p3.parsed_payload->'pathHashes') DESC - LIMIT 1) AS best_payload -FROM tags t -ORDER BY t.last_heard_at DESC; + COALESCE(t.trace_type, '')::text AS trace_type, + t.best_payload +FROM trace_tags t +WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( + SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) + AND ($2::text = '' OR t.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) + AND ($3::timestamptz IS NULL OR t.first_heard_at >= $3) + AND ($4::timestamptz IS NULL OR t.first_heard_at <= $4) + -- Keyset on the millisecond clients see, tie-broken by tag; matches idx_trace_tags_keyset. + AND ($5::timestamptz IS NULL + OR date_trunc('milliseconds', t.last_heard_at, 'UTC') < $5 + OR ($8::bytea IS NOT NULL AND date_trunc('milliseconds', t.last_heard_at, 'UTC') = $5 AND t.trace_tag < $8)) + AND ($7::text = '' OR t.trace_type = $7) +ORDER BY date_trunc('milliseconds', t.last_heard_at, 'UTC') DESC, t.trace_tag DESC +LIMIT $6; -- ============================================================ -- ROUTES -- ============================================================ -- name: UpsertKnownRoute :exec --- Inserts or updates a known route (all hops resolved to high confidence). --- node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and --- their hash bytes. last_seen is bumped on conflict. -INSERT INTO known_routes (node_ids, hash_prefix, iata, hop_count) -VALUES ($1, $2, $3, $4) -ON CONFLICT (node_ids, iata) DO UPDATE SET +-- Route identity is path_key, an md5 of node_ids computed by the caller. +-- On conflict, observation_count and last_seen are bumped and hash_prefix follows the +-- latest hearing, so evidence matches the hash width the route uses now. +INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) +VALUES ($1, $2, $3, $4, $5) +ON CONFLICT (iata, path_key) DO UPDATE SET + hash_prefix = EXCLUDED.hash_prefix, last_seen = NOW(), observation_count = known_routes.observation_count + 1; -- name: ListKnownRoutes :many +-- Only one branch runs. Keep the IATA range ordered by the composite index: +-- generic plans can otherwise prefer scanning the global timestamp index. +-- The text equality preserves exact input matching, including trailing spaces. +-- Pages by (last_seen to the ms, id) so routes sharing the cursor's millisecond +-- aren't skipped; cursor id 0 keeps the plain timestamp cursor. +SELECT r.id, r.node_ids, r.hash_prefix, r.iata, r.hop_count, r.first_seen, r.last_seen, r.observation_count +FROM (( SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes -WHERE ($1 = '' OR iata = $1) +WHERE $1 = '' AND ($2 = 0 OR hop_count = $2) - AND ($3::timestamptz IS NULL OR last_seen < $3) -ORDER BY last_seen DESC + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC +LIMIT $4 +) +UNION ALL +( +SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count +FROM known_routes +WHERE $1 <> '' AND iata::text = $1 + AND iata >= $1::bpchar AND iata <= $1::bpchar + AND ($2 = 0 OR hop_count = $2) + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC +LIMIT $4 +)) r +ORDER BY date_trunc('milliseconds', r.last_seen, 'UTC') DESC, r.id DESC LIMIT $4; -- name: SearchKnownRoutes :many @@ -1079,15 +1512,24 @@ ORDER BY hop_count ASC, last_seen DESC; -- Records or updates a neighbor relationship between two nodes observed in the same IATA. -- node_id is the advertising node, neighbor_id is the first-hop forwarder. -- snr is optional; pass NULL when no signal reading is available (the --- common case). On conflict, snr is only overwritten when a new non-null --- value is supplied, so a later no-SNR observation doesn't erase an --- earlier real reading. -INSERT INTO node_neighbors (node_id, neighbor_id, iata, observation_count, snr) -VALUES ($1, $2, $3, 1, $4) +-- common case). regionScope is optional too; pass NULL whenever the OTA +-- scope query for this neighbor didn't succeed (status != "responded"), +-- so a failed/timed-out query doesn't erase a previously known scope. +-- On conflict, snr and region_scope are only overwritten when a new +-- non-null value is supplied. +INSERT INTO node_neighbors (node_id, neighbor_id, iata, observation_count, snr, region_scope) +VALUES ($1, $2, $3, 1, $4, $5) ON CONFLICT (node_id, neighbor_id, iata) DO UPDATE SET last_seen = NOW(), observation_count = node_neighbors.observation_count + 1, - snr = COALESCE(EXCLUDED.snr, node_neighbors.snr); + snr = COALESCE(EXCLUDED.snr, node_neighbors.snr), + region_scope = COALESCE(EXCLUDED.region_scope, node_neighbors.region_scope); + +-- name: UpdateObserverRegionScope :exec +-- Records the observer's own OTA-reported region scope, from the "self" +-- field of a /neighbors report. Always known (not queried OTA), so this +-- unconditionally overwrites, unlike the neighbor-side region_scope. +UPDATE observers SET region_scope = $2 WHERE id = $1; -- name: GetNodeNeighbors :many -- Returns the neighbors of a node with details, ordered by most recently seen. @@ -1126,6 +1568,34 @@ WHERE ns.iata = $1 AND n.node_type IN (2, 3) AND ns.prefix_1 = ANY($2::bytea[]); +-- name: ResolveEndpointHashes :many +-- Logical endpoints can be any advertised role, unlike intermediate relay hops. +-- Endpoint hashes are always one byte; use the existing (iata, prefix_1) index. +-- LIMIT 1 keeps generic plans on a node PK lookup per candidate instead of +-- flattening the join into a scan of all nodes. The PK already guarantees one row. +SELECT ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM node_short_ids ns +CROSS JOIN LATERAL ( + SELECT id, name, latitude, longitude, public_key + FROM nodes WHERE id = ns.node_id + LIMIT 1 +) n +WHERE ns.iata = $1 + AND ns.prefix_1 = ANY($2::bytea[]); + +-- name: ResolveEndpointHashPairs :many +-- Batch form of ResolveEndpointHashes for a page of packets. Matches the cross +-- product of IATAs and hashes; callers pick out the pairs they asked for. +SELECT ns.iata, ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM node_short_ids ns +CROSS JOIN LATERAL ( + SELECT id, name, latitude, longitude, public_key + FROM nodes WHERE id = ns.node_id + LIMIT 1 +) n +WHERE ns.iata = ANY(@iatas::bpchar[]) + AND ns.prefix_1 = ANY(@hashes::bytea[]); + -- name: ResolvePathHashesP2 :many SELECT ns.prefix_4 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key FROM node_short_ids ns @@ -1150,49 +1620,70 @@ WHERE ns.iata = $1 AND n.node_type IN (2, 3) AND ns.prefix_4 = ANY($2::bytea[]); --- name: RefreshHourlyStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hourly_iata_stats; - --- name: RefreshTopNodes :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_nodes_by_iata; - --- name: RefreshTopObservers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_observers_by_iata; - --- name: RefreshPayloadBreakdown :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_payload_breakdown_by_iata; - --- name: RefreshTopTalkers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_talkers_by_iata; - --- name: RefreshTopAdvertisers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata; - -- name: RefreshRadioPresets :exec REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets; --- name: ReconfirmRoutes :exec --- Delete known_routes where any hop node has departed from node_short_ids for --- that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node). -DELETE FROM known_routes kr -WHERE EXISTS ( - SELECT 1 - FROM unnest(kr.node_ids) AS hop_node_id - WHERE NOT EXISTS ( - SELECT 1 FROM node_short_ids ns - WHERE ns.node_id = hop_node_id - AND ns.iata = kr.iata +-- name: AmbiguousPrefixes :many +-- Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. +SELECT iata::text AS iata, 1::int AS len, prefix_1 AS prefix FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 2::int, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 3::int, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 4::int, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1; + +-- name: ReconfirmRoutes :one +-- Checks one batch of least-recently-reconfirmed routes: deletes those with a departed +-- hop node or a hop prefix now matching >1 node in that IATA (length-aware: +-- 1/2/3/4-byte hop prefixes check prefix_1/2/3/4; ambiguity set supplied by AmbiguousPrefixes), +-- and stamps the survivors. +WITH batch AS MATERIALIZED ( + SELECT r.iata, r.path_key, r.node_ids, r.hash_prefix + FROM known_routes r + WHERE r.last_reconfirmed_at < @before + ORDER BY r.last_reconfirmed_at + LIMIT @batch_size + FOR UPDATE OF r SKIP LOCKED +), +amb AS ( + SELECT a.iata::char(3) AS iata, a.len, a.p + FROM ROWS FROM (unnest(@amb_iata::text[]), unnest(@amb_len::int[]), unnest(@amb_prefix::bytea[])) AS a(iata, len, p) +), +dead AS ( + SELECT b.iata, b.path_key + FROM batch b + WHERE EXISTS ( + SELECT 1 + FROM unnest(b.node_ids) AS hop_node_id + WHERE NOT EXISTS ( + SELECT 1 FROM node_short_ids ns + WHERE ns.node_id = hop_node_id + AND ns.iata = b.iata + ) ) + UNION + SELECT DISTINCT b.iata, b.path_key + FROM batch b + CROSS JOIN LATERAL unnest(b.hash_prefix) AS hp + JOIN amb a ON a.iata = b.iata AND a.len = length(hp) AND a.p = hp +), +deleted AS ( + DELETE FROM known_routes kr + USING dead d + WHERE kr.iata = d.iata AND kr.path_key = d.path_key +), +updated AS ( + UPDATE known_routes kr + SET last_reconfirmed_at = GREATEST(NOW(), @before::timestamptz) + FROM batch b + WHERE kr.iata = b.iata AND kr.path_key = b.path_key + AND NOT EXISTS ( + SELECT 1 FROM dead d + WHERE d.iata = b.iata AND d.path_key = b.path_key + ) ) -OR EXISTS ( - SELECT 1 - FROM unnest(kr.hash_prefix) AS hop_prefix - WHERE ( - SELECT COUNT(*) FROM node_short_ids ns - WHERE ns.iata = kr.iata - AND ns.prefix_4 = hop_prefix - ) > 1 -); +SELECT count(*) FROM batch; -- name: ReconfirmNeighbors :exec -- Delete node_neighbors where the neighbor has departed from node_short_ids @@ -1212,3 +1703,134 @@ OR ( AND iata = nn.iata ) ) > 1; + +-- name: CreateAccount :one +INSERT INTO accounts (name) VALUES (sqlc.arg(name)) +ON CONFLICT (name) WHERE deactivated_at IS NULL DO NOTHING +RETURNING id, name, created_at, deactivated_at; + +-- name: ListAccounts :many +SELECT id, name, created_at, deactivated_at FROM accounts +ORDER BY created_at DESC, id DESC; + +-- name: GetAccount :one +SELECT id, name, created_at, deactivated_at FROM accounts WHERE id = $1; + +-- name: DeactivateAccount :one +-- Lock the current row before deciding the outcome, including when another +-- deactivation commits while this statement is waiting for its row lock. +WITH target AS MATERIALIZED ( + SELECT a.id, a.deactivated_at FROM accounts a WHERE a.id = $1 FOR UPDATE +), changed AS ( + UPDATE accounts a SET deactivated_at = NOW() + FROM target t WHERE a.id = t.id AND t.deactivated_at IS NULL + RETURNING a.id +) +SELECT EXISTS(SELECT 1 FROM target) AS found, + EXISTS(SELECT 1 FROM changed) AS deactivated; + +-- name: GetObserverActivityLiveSummary :one +-- Two indexed ranges, bounded to one observer; no legacy presence counters. +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = @observer_id::uuid + AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly; + +-- name: GetScopeCatalogue :one +SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; + +-- name: ListScopeCatalogues :many +SELECT * FROM meshmapper_scope_catalogues ORDER BY iata, attempted_at; + +-- name: SaveScopeCatalogue :exec +-- One statement commits the validated snapshot and its lookup identities together. +-- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +-- after an error or 304. Imported names never replace existing manual metadata. +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest(@names::text[]) AS name, unnest(@keys::bytea[]) AS key, + unnest(@fingerprints::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; + +-- name: ListChannelCatalogues :many +SELECT * FROM meshmapper_channel_catalogues ORDER BY iata, attempted_at; + +-- name: SaveChannelCatalogue :exec +-- NULL payload/etag/checked_at retain the last good list after an error or 304. +INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_channel_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_channel_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_channel_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; + +-- name: DeleteChannelMembersNotIn :exec +DELETE FROM meshmapper_channel_members WHERE iata = @iata AND NOT (key_fingerprint = ANY(@keep::bytea[])); + +-- name: AddChannelMembers :exec +INSERT INTO meshmapper_channel_members (iata, key_fingerprint) +SELECT @iata, unnest(@fingerprints::bytea[]) +ON CONFLICT (iata, key_fingerprint) DO NOTHING; + +-- name: DeleteAllChannelMembers :exec +DELETE FROM meshmapper_channel_members; + +-- name: ListZoneBoundaries :many +SELECT * FROM meshmapper_zone_boundaries ORDER BY iata; + +-- name: SaveZoneBoundary :exec +-- NULL feature/etag/checked_at retain the last good boundary after an error or 304. +INSERT INTO meshmapper_zone_boundaries (iata, url, feature, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(feature)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata) DO UPDATE SET + url = EXCLUDED.url, + feature = COALESCE(EXCLUDED.feature, meshmapper_zone_boundaries.feature), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_boundaries.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_zone_boundaries.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; + +-- name: ListZoneLists :many +SELECT * FROM meshmapper_zone_lists ORDER BY country; + +-- name: SaveZoneList :exec +-- NULL payload/etag/fetched_at retain the last good list after an error or 304. +INSERT INTO meshmapper_zone_lists (country, payload, etag, fetched_at, attempted_at, next_attempt, last_error) +VALUES (@country, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(fetched_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (country) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_zone_lists.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_lists.etag), + fetched_at = COALESCE(EXCLUDED.fetched_at, meshmapper_zone_lists.fetched_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; + +-- name: PruneZoneBoundaries :many +-- Drops imports for IATAs no longer configured, so their manual border returns. +DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY(@keep::text[])) RETURNING iata; diff --git a/db/queries/rollup.sql b/db/queries/rollup.sql new file mode 100644 index 00000000..889199c9 --- /dev/null +++ b/db/queries/rollup.sql @@ -0,0 +1,295 @@ +-- ============================================================ +-- HOURLY ANALYTICS ROLLUPS +-- An hour [H, H+1h) is eligible once now() >= H + 95 min (ingest clamps heard_at to ±30 min). +-- ============================================================ + +-- name: TryRollupLock :one +SELECT pg_try_advisory_lock(25189, 1); + +-- name: RollupUnlock :exec +SELECT pg_advisory_unlock(25189, 1); + +-- name: RegisterRollupHours :exec +-- Adds a 'missing' row for every eligible hour since the earliest retained observation +-- (and not before @since), zero-observation hours included. +INSERT INTO analytics_rollup_hours (hour, status) +SELECT h, 'missing' +-- No observations yet means nothing to register (GREATEST would ignore the NULL MIN). +FROM (SELECT CASE WHEN MIN(heard_at) IS NOT NULL + THEN date_trunc('hour', GREATEST(MIN(heard_at), @since::timestamptz), 'UTC') END AS first_hour + FROM packet_observations) f, + generate_series(f.first_hour, date_trunc('hour', now() - INTERVAL '95 minutes', 'UTC'), INTERVAL '1 hour') h +WHERE f.first_hour IS NOT NULL +ON CONFLICT (hour) DO NOTHING; + +-- name: MarkPartialRollupHours :exec +-- Hours whose raw rows cleanup already started deleting can never be rolled completely. +-- Readers report the new status, so the revision moves too. +WITH marked AS ( + UPDATE analytics_rollup_hours h SET status = 'partial' + FROM analytics_raw_state r + WHERE h.status = 'missing' + AND r.raw_deleted_before IS NOT NULL + AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked); + +-- name: ListMissingRollupHours :many +SELECT hour FROM analytics_rollup_hours +WHERE status = 'missing' +ORDER BY hour +LIMIT $1; + +-- name: ListDirtyRollupHours :many +-- Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours +-- wait: their first roll consumes the entry only if the entry predates its snapshot. +SELECT d.hour FROM analytics_dirty_hours d +JOIN analytics_rollup_hours h ON h.hour = d.hour AND h.status = 'complete' +CROSS JOIN analytics_raw_state r +WHERE r.raw_deleted_before IS NULL OR d.hour > r.raw_deleted_before + INTERVAL '30 minutes' +ORDER BY d.hour +LIMIT $1; + +-- name: DeleteStaleDirtyHours :exec +-- Dirty hours that can no longer be re-rolled. +DELETE FROM analytics_dirty_hours d +WHERE EXISTS (SELECT 1 FROM analytics_rollup_hours h WHERE h.hour = d.hour AND h.status = 'partial') + OR d.hour <= (SELECT raw_deleted_before + INTERVAL '30 minutes' FROM analytics_raw_state); + +-- name: LockRollupHour :one +-- Returns the hour's status, or 'partial' if raw deletion has reached it since registration. +SELECT CASE + WHEN r.raw_deleted_before IS NOT NULL AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + THEN 'partial' + ELSE h.status + END::text AS status +FROM analytics_rollup_hours h CROSS JOIN analytics_raw_state r +WHERE h.hour = $1 +FOR UPDATE OF h; + +-- name: SetRollupHourPartial :exec +WITH marked AS ( + UPDATE analytics_rollup_hours SET status = 'partial' WHERE hour = $1 AND status = 'missing' RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked); + +-- name: FillRollupObs :exec +INSERT INTO rollup_obs (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, + path_len, rssi, snr, airtime_ms, payload_type, route_type, origin_pubkey, scope_id) +SELECT po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, + COALESCE(octet_length(po.path_bytes), 0), po.rssi, po.snr, po.airtime_ms, po.payload_type, + p.route_type, p.origin_pubkey, p.scope_id +FROM packet_observations po +JOIN packets p ON p.packet_hash = po.packet_hash +WHERE po.heard_at >= $1::timestamptz AND po.heard_at < $1::timestamptz + INTERVAL '1 hour'; + +-- name: DeleteRollupHour :exec +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour = @hour::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour = @hour::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour = @hour::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour = @hour::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour = @hour::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour = @hour::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour = @hour::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour = @hour::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour = @hour::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = @hour::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = @hour::timestamptz), + l AS (DELETE FROM analytics_hourly_scope_observers tl WHERE tl.hour = @hour::timestamptz) +DELETE FROM analytics_hourly_scope_nodes tm WHERE tm.hour = @hour::timestamptz; + +-- name: RollIATAObservations :exec +INSERT INTO analytics_hourly_iata_observations (hour, iata, observation_count) +SELECT @hour::timestamptz, iata, count(*) FROM rollup_obs GROUP BY iata; + +-- name: RollPayloadBreakdown :exec +INSERT INTO analytics_hourly_payload_breakdown (hour, iata, payload_type, count) +SELECT @hour::timestamptz, iata, payload_type, count(*) +FROM rollup_obs WHERE payload_type IS NOT NULL +GROUP BY iata, payload_type; + +-- name: RollSignal :exec +-- kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. +-- rssi=0 AND snr=0 means "not reported". +INSERT INTO analytics_hourly_signal (hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) +SELECT @hour::timestamptz, b.iata, grouping(b.snr_bin, b.rssi_bin)::integer, + COALESCE(b.snr_bin, -1)::integer, COALESCE(b.rssi_bin, -1)::integer, + count(*), count(b.snr), COALESCE(sum(b.snr), 0)::double precision, + count(b.rssi), COALESCE(sum(b.rssi), 0)::double precision +FROM ( + SELECT s.iata, s.snr, s.rssi, + width_bucket(s.snr, -30, 30, 12) AS snr_bin, + width_bucket(s.rssi, -140, 0, 14) AS rssi_bin + FROM ( + SELECT ro.iata, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + AND ro.snr > '-Infinity'::real AND ro.snr < 'Infinity'::real + THEN ro.snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + THEN ro.rssi::double precision END AS rssi + FROM rollup_obs ro + ) s +) b +GROUP BY GROUPING SETS ((b.iata), (b.iata, b.snr_bin), (b.iata, b.rssi_bin)); + +-- name: RollPaths :exec +-- category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). +INSERT INTO analytics_hourly_paths (hour, iata, category, hash_bytes, entries, receptions) +SELECT @hour::timestamptz, c.iata, c.category, c.hash_bytes, c.entries, count(*) +FROM ( + SELECT k.iata, k.category, + CASE WHEN k.category = 0 THEN k.hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN k.category = 0 THEN k.hop_count ELSE 0 END::integer AS entries + FROM ( + SELECT iata, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND path_len = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM rollup_obs + ) k +) c +GROUP BY c.iata, c.category, c.hash_bytes, c.entries; + +-- name: RollObserverActivity :exec +INSERT INTO analytics_hourly_observer_activity (hour, observer_id, payload_type, observations, airtime_ms, airtime_n, + snr_sum, snr_n, snr_min, rssi_sum, rssi_n) +SELECT @hour::timestamptz, observer_id, COALESCE(payload_type, -1)::smallint, + count(*), sum(airtime_ms)::real, count(airtime_ms), + sum(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + count(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), + min(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + sum(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + count(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)) +FROM rollup_obs +GROUP BY observer_id, COALESCE(payload_type, -1); + +-- name: RollObserverIdentity :exec +INSERT INTO analytics_hourly_observer_identity (hour, iata, observer_id, observation_count, display_name, observer_type) +SELECT @hour::timestamptz, r.iata, r.observer_id, r.n, o.display_name, o.observer_type +FROM (SELECT iata, observer_id, count(*) AS n FROM rollup_obs GROUP BY iata, observer_id) r +LEFT JOIN observers o ON o.id = r.observer_id; + +-- name: RollAdvertHearings :exec +INSERT INTO analytics_hourly_advert_hearings (hour, iata, origin_pubkey, observations, last_heard, name, node_type) +SELECT @hour::timestamptz, r.iata, r.origin_pubkey, r.n, r.last_heard, n.name, n.node_type +FROM (SELECT iata, origin_pubkey, count(*) AS n, max(heard_at) AS last_heard + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY iata, origin_pubkey) r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey; + +-- name: RollAdvertSets :exec +-- Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. +INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) +SELECT @hour::timestamptz, origin_pubkey, iatas, count(*), + count(*) FILTER (WHERE route_type IN (0, 1)), + count(*) FILTER (WHERE route_type IN (2, 3)) +FROM (SELECT origin_pubkey, route_type, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY packet_hash, origin_pubkey, route_type) p +GROUP BY origin_pubkey, iatas; + +-- name: RollTalkerSets :exec +INSERT INTO analytics_hourly_talker_sets (hour, sender_name, iatas, messages, last_sent) +SELECT @hour::timestamptz, sender_name, iatas, count(*), max(sent_at) +FROM (SELECT cm.sender_name, max(cm.sent_at) AS sent_at, array_agg(DISTINCT ro.iata ORDER BY ro.iata) AS iatas + FROM rollup_obs ro + JOIN channel_messages cm ON cm.packet_hash = ro.packet_hash + WHERE cm.sender_name IS NOT NULL + GROUP BY cm.id, cm.sender_name) m +GROUP BY sender_name, iatas; + +-- name: RollPacketSets :exec +INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) +SELECT @hour::timestamptz, iatas, count(*) +FROM (SELECT array_agg(DISTINCT iata ORDER BY iata) AS iatas FROM rollup_obs GROUP BY packet_hash) p +GROUP BY iatas; + +-- name: RollScopeSets :exec +INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) +SELECT @hour::timestamptz, iatas, scope_id, count(*) +FROM (SELECT scope_id, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs WHERE scope_id IS NOT NULL + GROUP BY packet_hash, scope_id) p +GROUP BY iatas, scope_id; + +-- name: RollScopeObservers :exec +INSERT INTO analytics_hourly_scope_observers (hour, iata, scope_id, observer_id) +SELECT DISTINCT @hour::timestamptz, iata, scope_id, observer_id +FROM rollup_obs WHERE scope_id IS NOT NULL; + +-- name: RollScopeNodes :exec +INSERT INTO analytics_hourly_scope_nodes (hour, iata, scope_id, origin_pubkey) +SELECT DISTINCT @hour::timestamptz, iata, scope_id, origin_pubkey +FROM rollup_obs WHERE payload_type = 4 AND origin_pubkey IS NOT NULL AND scope_id IS NOT NULL; + +-- name: RollupContentHash :one +-- Order-independent fingerprint of every family row for the hour. +SELECT md5(concat_ws('|', + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_iata_observations t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_payload_breakdown t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_signal t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_paths t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_activity t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_identity t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_hearings t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_talker_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_packet_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_observers t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_nodes t WHERE t.hour = @hour::timestamptz) +))::text AS content_hash; + +-- name: FinishRollupHour :one +-- Marks the hour complete and bumps the revision only when its content changed. +WITH prev AS ( + SELECT ph.content_hash FROM analytics_rollup_hours ph WHERE ph.hour = @hour::timestamptz +), upd AS ( + UPDATE analytics_rollup_hours uh + SET status = 'complete', rolled_at = now(), content_hash = @content_hash::text + WHERE uh.hour = @hour::timestamptz +), undirty AS ( + DELETE FROM analytics_dirty_hours dh WHERE dh.hour = @hour::timestamptz +), rev AS ( + UPDATE analytics_state SET revision = revision + 1 + WHERE (SELECT content_hash FROM prev) IS DISTINCT FROM @content_hash::text + RETURNING revision +) +SELECT EXISTS (SELECT 1 FROM rev)::boolean AS changed; + +-- name: GetAnalyticsRevision :one +-- Both counters only grow, so their sum changes whenever rolled content or coverage does. +SELECT (s.revision + r.coverage)::bigint FROM analytics_state s CROSS JOIN analytics_raw_state r; + +-- name: OldestMissingRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'missing'; + +-- name: DeleteOldRollups :exec +-- Bumps coverage when hours go, so cached responses that included them are not reused. +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour < @cutoff::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour < @cutoff::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour < @cutoff::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour < @cutoff::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour < @cutoff::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour < @cutoff::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour < @cutoff::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour < @cutoff::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour < @cutoff::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour < @cutoff::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour < @cutoff::timestamptz), + l AS (DELETE FROM analytics_dirty_hours tl WHERE tl.hour < @cutoff::timestamptz), + n AS (DELETE FROM analytics_hourly_scope_observers tn WHERE tn.hour < @cutoff::timestamptz), + o AS (DELETE FROM analytics_hourly_scope_nodes tob WHERE tob.hour < @cutoff::timestamptz), + m AS (DELETE FROM analytics_rollup_hours tm WHERE tm.hour < @cutoff::timestamptz RETURNING 1) +UPDATE analytics_raw_state SET coverage = coverage + 1 WHERE EXISTS (SELECT 1 FROM m); diff --git a/db/queries/route_evidence.sql b/db/queries/route_evidence.sql new file mode 100644 index 00000000..dcc056fd --- /dev/null +++ b/db/queries/route_evidence.sql @@ -0,0 +1,23 @@ +-- name: GetRouteEvidenceRoute :one +SELECT * FROM known_routes +WHERE iata = @iata::bpchar AND path_key = @path_key::bytea; + +-- name: ListRouteEvidence :many +-- Leading index equalities and the time/ID boundary bound both custom and generic plans. +-- Full-byte equality is required even when the compact digest matches. TRACE path bytes +-- carry readings; unclassified legacy observations cannot be safely called ordinary paths. +SELECT po.id, po.packet_hash, po.observer_id, o.display_name AS observer_name, + po.heard_at, po.payload_type, po.rssi, po.snr +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.iata = @iata::bpchar + AND po.hash_size = @hash_size::smallint + AND decode(md5(po.path_bytes), 'hex') = @path_digest::bytea + AND po.path_bytes = @path_bytes::bytea + AND po.hop_count >= 2 + AND po.hop_count = @hop_count::smallint + AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 + AND po.heard_at >= @since::timestamptz AND po.heard_at < @until::timestamptz + AND (po.heard_at, po.id) < (@before_at::timestamptz, @before_id::bigint) +ORDER BY po.heard_at DESC, po.id DESC +LIMIT @page_limit::integer; diff --git a/db/queries/series.sql b/db/queries/series.sql new file mode 100644 index 00000000..154d8db1 --- /dev/null +++ b/db/queries/series.sql @@ -0,0 +1,94 @@ +-- name: GetStatsSeries :many +-- One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours. +-- Counts sum; observers, IATAs and scopes are distinct across the window; packets count +-- once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate. +WITH hours AS ( + SELECT g.h::timestamptz AS hour, COALESCE(r.status, 'missing')::text AS status + FROM generate_series(@since::timestamptz, @until::timestamptz, INTERVAL '1 hour') g(h) + LEFT JOIN analytics_rollup_hours r ON r.hour = g.h + WHERE g.h < @until::timestamptz +), complete AS ( + SELECT hour FROM hours WHERE status = 'complete' +), obs AS ( + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND o.iata = ANY(@iatas::bpchar[]) +), obs_agg AS ( + SELECT hour, sum(observation_count)::bigint AS observations, count(DISTINCT iata)::bigint AS active_iatas + FROM obs GROUP BY GROUPING SETS ((), (hour)) +), pkt AS ( + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND s.iatas && @iatas::bpchar[] +), pkt_agg AS ( + SELECT hour, sum(packets)::bigint AS unique_packets FROM pkt GROUP BY GROUPING SETS ((), (hour)) +), obsr AS ( + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND i.iata = ANY(@iatas::bpchar[]) +), obsr_agg AS ( + SELECT hour, count(DISTINCT observer_id)::bigint AS active_observers FROM obsr GROUP BY GROUPING SETS ((), (hour)) +), scp AS ( + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND s.iatas && @iatas::bpchar[] +), scp_agg AS ( + SELECT hour, sum(packets)::bigint AS scoped_packets, count(DISTINCT scope_id)::bigint AS active_scopes + FROM scp GROUP BY GROUPING SETS ((), (hour)) +), pth AS ( + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND cardinality(@iatas::bpchar[]) > 0 AND p.iata = ANY(@iatas::bpchar[]) +), pth_agg AS ( + SELECT hour, max(entries)::integer AS max_path_entries FROM pth GROUP BY GROUPING SETS ((), (hour)) +), sig AS ( + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND cardinality(@iatas::bpchar[]) > 0 AND s.iata = ANY(@iatas::bpchar[]) +), sig_agg AS ( + SELECT hour, sum(snr_sum)::double precision AS snr_sum, sum(snr_samples)::bigint AS snr_samples, + sum(rssi_sum)::double precision AS rssi_sum, sum(rssi_samples)::bigint AS rssi_samples + FROM sig GROUP BY GROUPING SETS ((), (hour)) +), keys AS ( + SELECT hour, status FROM hours + UNION ALL + SELECT NULL::timestamptz, 'summary'::text +) +SELECT k.hour, k.status, + COALESCE(o.observations, 0)::bigint AS observations, + COALESCE(p.unique_packets, 0)::bigint AS unique_packets, + COALESCE(ob.active_observers, 0)::bigint AS active_observers, + COALESCE(o.active_iatas, 0)::bigint AS active_iatas, + COALESCE(s.scoped_packets, 0)::bigint AS scoped_packets, + COALESCE(s.active_scopes, 0)::bigint AS active_scopes, + COALESCE(pa.max_path_entries, 0)::integer AS max_path_entries, + COALESCE(sg.snr_sum, 0)::double precision AS snr_sum, + COALESCE(sg.snr_samples, 0)::bigint AS snr_samples, + COALESCE(sg.rssi_sum, 0)::double precision AS rssi_sum, + COALESCE(sg.rssi_samples, 0)::bigint AS rssi_samples +FROM keys k +LEFT JOIN obs_agg o ON o.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pkt_agg p ON p.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN obsr_agg ob ON ob.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN scp_agg s ON s.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pth_agg pa ON pa.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN sig_agg sg ON sg.hour IS NOT DISTINCT FROM k.hour +ORDER BY k.hour NULLS LAST; + +-- name: GetEarliestCompleteRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete'; + +-- name: GetLatestCompleteRollupHour :one +SELECT max(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete'; diff --git a/db/queries/signal.sql b/db/queries/signal.sql new file mode 100644 index 00000000..ecd20cf1 --- /dev/null +++ b/db/queries/signal.sql @@ -0,0 +1,29 @@ +-- name: GetSignalStats :many +-- Read the hourly rollup, never observations on an HTTP request. +-- Weight averages by sample counts instead of averaging regional/hourly means. +WITH readings AS ( + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz + AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz + AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) +) +SELECT (3 + 4 * grouping(hour))::integer AS kind, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + -1::integer AS snr_bin, -1::integer AS rssi_bin, + COALESCE(sum(receptions), 0)::bigint AS receptions, + COALESCE(sum(snr_samples), 0)::bigint AS snr_samples, + COALESCE(sum(snr_sum) / NULLIF(sum(snr_samples), 0), 0)::double precision AS snr_average, + COALESCE(sum(rssi_samples), 0)::bigint AS rssi_samples, + COALESCE(sum(rssi_sum) / NULLIF(sum(rssi_samples), 0), 0)::double precision AS rssi_average +FROM readings WHERE kind = 3 +GROUP BY GROUPING SETS ((), (hour)) +UNION ALL +SELECT (kind + 4)::integer, 'epoch'::timestamptz, snr_bin, rssi_bin, + sum(receptions)::bigint, 0::bigint, 0::double precision, 0::bigint, 0::double precision +FROM readings WHERE kind IN (1, 2) +GROUP BY kind, snr_bin, rssi_bin +ORDER BY kind, hour, snr_bin, rssi_bin; + diff --git a/db/reconfirm_integration_test.go b/db/reconfirm_integration_test.go new file mode 100644 index 00000000..0b52206b --- /dev/null +++ b/db/reconfirm_integration_test.go @@ -0,0 +1,334 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "fmt" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func ambiguity(t *testing.T, ctx context.Context, store *Store) AmbiguousPrefixes { + t.Helper() + amb, err := store.AmbiguousPrefixes(ctx) + if err != nil { + t.Fatal(err) + } + return amb +} + +func TestReconfirmReleasesEarlierBatchPostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + before := time.Now() + if n, err := store.ReconfirmRoutes(ctx, 1, before, ambiguity(t, ctx, store)); err != nil || n != 1 { + t.Fatalf("first batch: %d, %v", n, err) + } + gate, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer gate.Rollback(context.Background()) + key := time.Now().UnixNano() + if _, err := gate.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, key); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, fmt.Sprintf(` +CREATE FUNCTION pause_second_route() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN IF NEW.path_key=int4send(2) THEN PERFORM pg_advisory_xact_lock(%d); END IF; RETURN NEW; END $$; +CREATE TRIGGER pause_second_route BEFORE UPDATE ON known_routes FOR EACH ROW EXECUTE FUNCTION pause_second_route();`, key)) + if err != nil { + t.Fatal(err) + } + finished := make(chan error, 1) + amb := ambiguity(t, ctx, store) + go func() { + _, err := store.ReconfirmRoutes(ctx, 1, before, amb) + finished <- err + }() + deadline := time.Now().Add(3 * time.Second) + for { + var waiting bool + if err := pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted AND database=(SELECT oid FROM pg_database WHERE datname=current_database()))`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting { + break + } + if time.Now().After(deadline) { + t.Fatal("second batch did not reach the test gate") + } + time.Sleep(time.Millisecond) + } + upsertCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + err = sqlc.New(pool).UpsertKnownRoute(upsertCtx, sqlc.UpsertKnownRouteParams{ + PathKey: []byte{0, 0, 0, 1}, NodeIds: []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000001")}, + HashPrefix: [][]byte{{0x11}}, Iata: "SEA", HopCount: 1, + }) + if err != nil { + t.Fatalf("completed batch still blocks ingest: %v", err) + } + if err := gate.Rollback(ctx); err != nil { + t.Fatal(err) + } + if err := <-finished; err != nil { + t.Fatal(err) + } + var count int + if err := pool.QueryRow(ctx, `SELECT observation_count FROM known_routes WHERE path_key=int4send(1)`).Scan(&count); err != nil || count != 2 { + t.Fatalf("concurrent observation increment lost: count %d, error %v", count, err) + } +} + +func TestReconfirmPreservesRouteDataPostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + before := time.Now().UTC().Truncate(time.Microsecond) + _, err := pool.Exec(ctx, ` +TRUNCATE known_routes, node_short_ids; +INSERT INTO node_short_ids (node_id,iata,prefix_4) VALUES + (md5('n1')::uuid,'SEA','\x11223344'), (md5('n2')::uuid,'SEA','\x11223355'), + (md5('n3')::uuid,'SEA','\xaabbccdd'), (md5('n4')::uuid,'SEA','\xaabbccdd'), + (md5('n5')::uuid,'SEA','\x55667788'), (md5('n6')::uuid,'SFO','\x55667799'); +INSERT INTO known_routes (id,path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen,observation_count,last_reconfirmed_at) +SELECT id,int4send(id),ARRAY[md5(node)::uuid],ARRAY[decode(prefix,'hex')],iata,1, + '2025-12-01','2026-01-01',42,'2026-01-01' +FROM (VALUES + (1,'n1','11223344','SEA'), (2,'n2','11223355','SEA'), + (3,'n5','55','SEA'), (4,'n5','5566','SEA'), (5,'n5','556677','SEA'), (6,'n5','55667788','SEA'), + (7,'n1','11','SEA'), (8,'n1','1122','SEA'), (9,'n1','112233','SEA'), (10,'n3','aabbccdd','SEA'), + (11,'missing','55','SEA'), (12,'n6','55','SEA'), + (13,'n1','11223344','SEA'), (14,'n5','55667788','SEA'), (15,'n6','55','SFO'), + (16,'n5','55','SEA'), (17,'n5','55','SEA') +) AS routes(id,node,prefix,iata); +UPDATE known_routes SET node_ids=node_ids || md5('n5')::uuid, hash_prefix=hash_prefix || '\x55667788'::bytea, hop_count=2 WHERE id=13; +UPDATE known_routes SET node_ids=node_ids || md5('missing')::uuid, hash_prefix=hash_prefix || '\x99'::bytea, hop_count=2 WHERE id=14;`) + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE known_routes SET last_reconfirmed_at=$1::timestamptz + CASE WHEN id=16 THEN interval '1 hour' ELSE interval '0' END WHERE id IN (16,17)`, before); err != nil { + t.Fatal(err) + } + snapshot := func() map[int][]byte { + t.Helper() + rows, err := pool.Query(ctx, `SELECT id, to_jsonb(r)-'last_reconfirmed_at' FROM known_routes r`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + result := map[int][]byte{} + for rows.Next() { + var id int + var data []byte + if err := rows.Scan(&id, &data); err != nil { + t.Fatal(err) + } + result[id] = data + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return result + } + original := snapshot() + var checked int64 + for range 10 { + n, err := store.ReconfirmRoutes(ctx, 3, before, ambiguity(t, ctx, store)) + if err != nil { + t.Fatal(err) + } + checked += n + if n == 0 { + break + } + } + if checked != 15 { + t.Fatalf("checked %d routes; want 15 once each, including deleted routes", checked) + } + after := snapshot() + want := []int{1, 2, 3, 4, 5, 6, 13, 15, 16, 17} + if len(after) != len(want) { + t.Fatalf("surviving routes: %v", after) + } + for _, id := range want { + if !bytes.Equal(original[id], after[id]) { + t.Errorf("route %d lost or changed fields: before %s, after %s", id, original[id], after[id]) + } + } + var stamped int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes WHERE id NOT IN (16,17) AND last_reconfirmed_at >= $1`, before).Scan(&stamped); err != nil { + t.Fatal(err) + } + if stamped != 8 { + t.Errorf("validated survivors = %d, want 8", stamped) + } +} + +func TestReconfirmCancelledBatchRollsBackPostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + before := time.Now() + if _, err := store.ReconfirmRoutes(ctx, 1, before, ambiguity(t, ctx, store)); err != nil { + t.Fatal(err) + } + _, err := pool.Exec(ctx, ` +CREATE FUNCTION pause_reconfirm() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN PERFORM pg_sleep(1); RETURN NEW; END $$; +CREATE TRIGGER pause_reconfirm BEFORE UPDATE ON known_routes FOR EACH ROW EXECUTE FUNCTION pause_reconfirm();`) + if err != nil { + t.Fatal(err) + } + batchCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + if _, err := store.ReconfirmRoutes(batchCtx, 1, before, ambiguity(t, ctx, store)); err == nil { + t.Fatal("expected batch cancellation") + } + // Waiting for this DDL also waits for cancellation to release the row lock. + if _, err := pool.Exec(ctx, `DROP TRIGGER pause_reconfirm ON known_routes`); err != nil { + t.Fatal(err) + } + var total, stamped int + if err := pool.QueryRow(ctx, `SELECT count(*),count(*) FILTER (WHERE last_reconfirmed_at >= $1) FROM known_routes`, before).Scan(&total, &stamped); err != nil { + t.Fatal(err) + } + if total != 2 || stamped != 1 { + t.Fatalf("cancellation changed committed work: total %d, checked %d", total, stamped) + } + if n, err := store.ReconfirmRoutes(ctx, 1, before, ambiguity(t, ctx, store)); err != nil || n != 1 { + t.Fatalf("cancelled route was not available to retry: %d, %v", n, err) + } +} + +func TestReconfirmFutureCutoffDoesNotRepeatPostgres(t *testing.T) { + ctx, _, store := reconfirmPool(t) + before := time.Now().Add(time.Hour) + if n, err := store.ReconfirmRoutes(ctx, 2, before, ambiguity(t, ctx, store)); err != nil || n != 2 { + t.Fatalf("first batch: %d, %v", n, err) + } + if n, err := store.ReconfirmRoutes(ctx, 2, before, ambiguity(t, ctx, store)); err != nil || n != 0 { + t.Fatalf("completed routes consumed the run budget twice: %d, %v", n, err) + } +} + +func TestReconfirmCancellationDoesNotPartiallyDeletePostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + _, err := pool.Exec(ctx, ` +UPDATE known_routes SET node_ids=ARRAY[md5('departed')::uuid] WHERE path_key=int4send(1); +CREATE FUNCTION pause_valid_route() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN PERFORM pg_sleep(1); RETURN NEW; END $$; +CREATE TRIGGER pause_valid_route BEFORE UPDATE ON known_routes FOR EACH ROW EXECUTE FUNCTION pause_valid_route();`) + if err != nil { + t.Fatal(err) + } + before := time.Now() + batchCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + if _, err := store.ReconfirmRoutes(batchCtx, 2, before, ambiguity(t, ctx, store)); err == nil { + t.Fatal("expected batch cancellation") + } + if _, err := pool.Exec(ctx, `DROP TRIGGER pause_valid_route ON known_routes`); err != nil { + t.Fatal(err) + } + var count int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes`).Scan(&count); err != nil || count != 2 { + t.Fatalf("cancelled batch partially deleted routes: %d, %v", count, err) + } + if n, err := store.ReconfirmRoutes(ctx, 2, before, ambiguity(t, ctx, store)); err != nil || n != 2 { + t.Fatalf("retry did not process both routes: %d, %v", n, err) + } + if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes WHERE path_key=int4send(2)`).Scan(&count); err != nil || count != 1 { + t.Fatalf("retry lost the valid route: %d, %v", count, err) + } +} + +// A private schema lets independent connections exercise row locks. +func reconfirmPool(t *testing.T) (context.Context, *pgxpool.Pool, *Store) { + t.Helper() + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { admin.Close(context.Background()) }) + schema := pgx.Identifier{"reconfirm_" + uuid.NewString()}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.Background(), "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Error(err) + } + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.MaxConns = 4 + cfg.ConnConfig.RuntimeParams["search_path"] = schema + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + _, err = pool.Exec(ctx, ` +CREATE TABLE known_routes (LIKE public.known_routes INCLUDING ALL); +CREATE TABLE node_short_ids (LIKE public.node_short_ids INCLUDING ALL); +INSERT INTO node_short_ids (node_id, iata, prefix_4) VALUES + ('00000000-0000-0000-0000-000000000001', 'SEA', '\x11223344'); +INSERT INTO known_routes (path_key,node_ids,hash_prefix,iata,hop_count,last_reconfirmed_at) +SELECT int4send(i), ARRAY['00000000-0000-0000-0000-000000000001'::uuid], ARRAY['\x11'::bytea], 'SEA', 1, + '2026-01-01'::timestamptz + i * interval '1 second' +FROM generate_series(1,2) i;`) + if err != nil { + t.Fatal(err) + } + return ctx, pool, New(pool, 0, 0) +} + +func TestReconfirmSkipsBusyRoutePostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + busy, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer busy.Rollback(context.Background()) + if _, err := busy.Exec(ctx, `SELECT 1 FROM known_routes WHERE path_key=int4send(1) FOR UPDATE`); err != nil { + t.Fatal(err) + } + batchCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + if _, err := store.ReconfirmRoutes(batchCtx, 1, time.Now(), ambiguity(t, ctx, store)); err != nil { + t.Fatalf("maintenance should skip the busy route and validate the next one: %v", err) + } + var checked, preserved int + if err := pool.QueryRow(ctx, `SELECT count(*) FILTER (WHERE last_reconfirmed_at > '2026-02-01'), count(*) FROM known_routes`).Scan(&checked, &preserved); err != nil { + t.Fatal(err) + } + if checked != 1 || preserved != 2 { + t.Fatalf("checked %d, preserved %d; want 1 and 2", checked, preserved) + } + if err := busy.Rollback(ctx); err != nil { + t.Fatal(err) + } + if _, err := store.ReconfirmRoutes(ctx, 1, time.Now(), ambiguity(t, ctx, store)); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes WHERE last_reconfirmed_at > '2026-02-01'`).Scan(&checked); err != nil { + t.Fatal(err) + } + if checked != 2 { + t.Fatalf("deferred route was not validated after release: checked %d", checked) + } +} diff --git a/db/retention_integration_test.go b/db/retention_integration_test.go new file mode 100644 index 00000000..cff7dae0 --- /dev/null +++ b/db/retention_integration_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +func retentionTx(t *testing.T) (context.Context, pgx.Tx) { + t.Helper() + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close(context.Background()) }) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { tx.Rollback(context.Background()) }) + return ctx, tx +} + +func countRows(t *testing.T, ctx context.Context, tx pgx.Tx, sql string) int { + t.Helper() + var n int + if err := tx.QueryRow(ctx, sql).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// More than one batch of old packets; observations go with them. +func TestDeleteOldPacketsBatchesPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + _, err := tx.Exec(ctx, ` +INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x01'); +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +SELECT int4send(i), 4, 0, 1, '\x00', '\x00', + CASE WHEN i <= 2500 THEN '2026-01-01'::timestamptz ELSE '2026-02-01'::timestamptz END, + CASE WHEN i <= 2500 THEN '2026-01-01'::timestamptz ELSE '2026-02-01'::timestamptz END +FROM generate_series(1, 2510) i; +INSERT INTO packet_observations (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +SELECT packet_hash, '00000000-0000-0000-0000-000000000001', 'YVR', last_heard_at, 0, 1, 0 FROM packets;`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + if err := store.DeleteOldPackets(ctx, time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM packets`); n != 10 { + t.Errorf("packets left = %d, want 10", n) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM packet_observations`); n != 10 { + t.Errorf("observations left = %d, want 10", n) + } +} + +// Covers grace set longer than retention too. +func TestDeleteOldRoutesBatchesPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + // 1-10500 past retention, 10501-10600 rare, 10601-10700 well observed, rest fresh. + _, err := tx.Exec(ctx, ` +CREATE TEMP TABLE known_routes (LIKE public.known_routes INCLUDING ALL) ON COMMIT DROP; +INSERT INTO known_routes (id, path_key, node_ids, hash_prefix, iata, hop_count, last_seen, observation_count) +SELECT i, decode(md5(i::text), 'hex'), ARRAY[md5(i::text)::uuid], ARRAY['\x01'::bytea], 'YYZ', 2, + CASE WHEN i <= 10500 THEN '2026-01-01'::timestamptz + WHEN i <= 10700 THEN '2026-01-20'::timestamptz + ELSE '2026-02-01'::timestamptz END, + CASE WHEN i BETWEEN 10501 AND 10600 THEN 1 ELSE 10 END +FROM generate_series(1, 10710) i;`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + retention := time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC) + grace := time.Date(2026, 1, 25, 0, 0, 0, 0, time.UTC) + if err := store.DeleteOldRoutes(ctx, retention, 3, grace); err != nil { + t.Fatal(err) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM known_routes`); n != 110 { + t.Errorf("routes left = %d, want 110 (well-observed + fresh)", n) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM known_routes WHERE observation_count = 1`); n != 0 { + t.Errorf("rare routes past grace left = %d, want 0", n) + } + + // Grace longer than retention: retention alone decides. + if err := store.DeleteOldRoutes(ctx, time.Date(2026, 1, 25, 0, 0, 0, 0, time.UTC), 3, retention); err != nil { + t.Fatal(err) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM known_routes`); n != 10 { + t.Errorf("routes left = %d, want 10 fresh", n) + } +} diff --git a/db/rollup.go b/db/rollup.go new file mode 100644 index 00000000..fc8e3fd7 --- /dev/null +++ b/db/rollup.go @@ -0,0 +1,204 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "fmt" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Must match db/sqlc_scratch/rollup_obs.sql, which only exists so sqlc can type-check rollup.sql. +const createRollupObs = `CREATE TEMP TABLE rollup_obs ( + packet_hash bytea NOT NULL, observer_id uuid NOT NULL, iata character(3) NOT NULL, + heard_at timestamptz NOT NULL, path_length_byte smallint NOT NULL, hash_size smallint NOT NULL, + hop_count smallint NOT NULL, path_len integer NOT NULL, rssi smallint, snr real, airtime_ms real, + payload_type smallint, route_type smallint NOT NULL, origin_pubkey bytea, scope_id integer +) ON COMMIT DROP` + +// rollupDelay is how long after an hour closes it becomes eligible to roll (ingest clamps +// heard_at to ±30 min; RegisterRollupHours uses the same bound). +const rollupDelay = 95 * time.Minute + +// RollOutcome is what RollHour did with an hour. +type RollOutcome int + +const ( + RollSkipped RollOutcome = iota // unregistered, or lost a serialization race; retried next pass + RollPartial // raw rows already deleted; never rolled + RollComplete // rolled and marked complete +) + +// RollupSession holds the rollup advisory lock on one dedicated connection. +type RollupSession struct { + conn *pgxpool.Conn + q *sqlc.Queries +} + +// BeginRollup takes the rollup lock. ok is false when another process holds it. +func (s *Store) BeginRollup(ctx context.Context) (*RollupSession, bool, error) { + if s.pool == nil { + return nil, false, errors.New("rollup: store has no connection pool") + } + conn, err := s.pool.Acquire(ctx) + if err != nil { + return nil, false, err + } + q := sqlc.New(conn) + ok, err := q.TryRollupLock(ctx) + if err != nil || !ok { + conn.Release() + return nil, false, err + } + return &RollupSession{conn: conn, q: q}, true, nil +} + +// Close releases the lock. If unlocking fails the connection is closed so the lock can't leak. +func (r *RollupSession) Close(ctx context.Context) { + if err := r.q.RollupUnlock(ctx); err != nil { + _ = r.conn.Hijack().Close(context.Background()) + return + } + r.conn.Release() +} + +// RegisterHours adds a 'missing' row for every eligible hour since max(since, earliest observation). +func (r *RollupSession) RegisterHours(ctx context.Context, since time.Time) error { + return r.q.RegisterRollupHours(ctx, ts(since)) +} + +func (r *RollupSession) MarkPartialHours(ctx context.Context) error { + return r.q.MarkPartialRollupHours(ctx) +} + +// MissingHours returns up to limit unrolled hours, oldest first. +func (r *RollupSession) MissingHours(ctx context.Context, limit int32) ([]time.Time, error) { + rows, err := r.q.ListMissingRollupHours(ctx, limit) + return times(rows), err +} + +// DirtyHours drops dirty hours that can't be re-rolled and returns up to limit of the rest. +func (r *RollupSession) DirtyHours(ctx context.Context, limit int32) ([]time.Time, error) { + if err := r.q.DeleteStaleDirtyHours(ctx); err != nil { + return nil, err + } + rows, err := r.q.ListDirtyRollupHours(ctx, limit) + return times(rows), err +} + +// RollHour rebuilds every rollup family for one hour in a single REPEATABLE READ transaction. +// changed reports whether the hour's content differs from its previous roll. +func (r *RollupSession) RollHour(ctx context.Context, hour time.Time) (outcome RollOutcome, changed bool, err error) { + tx, err := r.conn.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + if err != nil { + return RollSkipped, false, err + } + defer func() { + _ = tx.Rollback(context.Background()) + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "40001" { + outcome, changed, err = RollSkipped, false, nil + } + }() + outcome, changed, err = rollHour(ctx, tx, hour) + if err != nil { + return RollSkipped, false, err + } + return outcome, changed, tx.Commit(ctx) +} + +// rollHour does RollHour's work inside tx; the caller commits. +func rollHour(ctx context.Context, tx pgx.Tx, hour time.Time) (RollOutcome, bool, error) { + q := sqlc.New(tx) + h := ts(hour) + status, err := q.LockRollupHour(ctx, h) + if errors.Is(err, pgx.ErrNoRows) { + return RollSkipped, false, nil + } + if err != nil { + return RollSkipped, false, err + } + if status == "partial" { + return RollPartial, false, q.SetRollupHourPartial(ctx, h) + } + if _, err := tx.Exec(ctx, createRollupObs); err != nil { + return RollSkipped, false, err + } + if err := q.FillRollupObs(ctx, h); err != nil { + return RollSkipped, false, err + } + if _, err := tx.Exec(ctx, "ANALYZE rollup_obs"); err != nil { + return RollSkipped, false, err + } + if err := q.DeleteRollupHour(ctx, h); err != nil { + return RollSkipped, false, err + } + for name, roll := range map[string]func(context.Context, pgtype.Timestamptz) error{ + "iata observations": q.RollIATAObservations, + "payload breakdown": q.RollPayloadBreakdown, + "signal": q.RollSignal, + "paths": q.RollPaths, + "observer activity": q.RollObserverActivity, + "observer identity": q.RollObserverIdentity, + "advert hearings": q.RollAdvertHearings, + "advert sets": q.RollAdvertSets, + "talker sets": q.RollTalkerSets, + "packet sets": q.RollPacketSets, + "scope sets": q.RollScopeSets, + "scope observers": q.RollScopeObservers, + "scope nodes": q.RollScopeNodes, + } { + if err := roll(ctx, h); err != nil { + return RollSkipped, false, fmt.Errorf("roll %s: %w", name, err) + } + } + // Dropped here as well as ON COMMIT so a caller can roll several hours in one transaction. + if _, err := tx.Exec(ctx, "DROP TABLE rollup_obs"); err != nil { + return RollSkipped, false, err + } + hash, err := q.RollupContentHash(ctx, h) + if err != nil { + return RollSkipped, false, err + } + changed, err := q.FinishRollupHour(ctx, sqlc.FinishRollupHourParams{Hour: h, ContentHash: hash}) + if err != nil { + return RollSkipped, false, err + } + return RollComplete, changed, nil +} + +// AnalyticsRevision changes whenever rolled content, an hour's status or rollup coverage changes. +func (s *Store) AnalyticsRevision(ctx context.Context) (int64, error) { + return s.q.GetAnalyticsRevision(ctx) +} + +func ts(t time.Time) pgtype.Timestamptz { + return pgtype.Timestamptz{Time: t, Valid: true} +} + +func times(rows []pgtype.Timestamptz) []time.Time { + out := make([]time.Time, len(rows)) + for i, r := range rows { + out[i] = r.Time + } + return out +} + +// OldestMissingRollupHour returns the earliest hour still waiting to be rolled; ok is false if none. +func (s *Store) OldestMissingRollupHour(ctx context.Context) (hour time.Time, ok bool, err error) { + h, err := s.q.OldestMissingRollupHour(ctx) + return h.Time, h.Valid, err +} + +// DeleteOldRollups drops rolled hours, their bookkeeping and queued re-rolls before cutoff. +func (s *Store) DeleteOldRollups(ctx context.Context, cutoff time.Time) error { + return s.q.DeleteOldRollups(ctx, ts(cutoff)) +} diff --git a/db/rollup_integration_test.go b/db/rollup_integration_test.go new file mode 100644 index 00000000..d71d73bb --- /dev/null +++ b/db/rollup_integration_test.go @@ -0,0 +1,524 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// rollupFixture loads observations across two hours, h0 and h0+1h, and returns h0. +// h0+2h is left empty so a zero-observation hour gets registered too. +func rollupFixture(t *testing.T, ctx context.Context, pool *pgxpool.Pool) time.Time { + t.Helper() + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + var h0 time.Time + if err := pool.QueryRow(ctx, "SELECT date_trunc('hour', now(), 'UTC') - INTERVAL '4 hours'").Scan(&h0); err != nil { + t.Fatal(err) + } + _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YVR'), ('YYZ'), ('YOW'); +INSERT INTO transport_scopes (id, name, transport_key, key_fingerprint) +VALUES (1, 'scope-one', decode(repeat('01', 16), 'hex'), decode(repeat('01', 8), 'hex')); +INSERT INTO observers (id, public_key, display_name, observer_type) VALUES + ('00000000-0000-0000-0000-00000000000a', '\x0a', 'obs a', 'repeater'), + ('00000000-0000-0000-0000-00000000000b', '\x0b', 'obs b', NULL); +INSERT INTO nodes (public_key, node_type, name) VALUES ('\xee', 2, 'node e'); +INSERT INTO channels (id, channel_hash) VALUES (1, '\x11'); +-- hash, payload type, route type, origin, scope +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, origin_pubkey, scope_id, raw_payload, raw_header, first_heard_at, last_heard_at) +SELECT hash, pt, rt, 0, origin, scope, '\x00', '\x00', $1::timestamptz, $1::timestamptz FROM (VALUES + ('\x01'::bytea, 4, 1, '\xee'::bytea, NULL::int), -- flood advert, two IATAs + ('\x02', 4, 2, '\xee', 1), -- direct advert, scoped, one IATA + ('\x03', 5, 1, NULL, 1), -- scoped channel message, two IATAs + ('\x04', 5, 1, NULL, 1), -- scoped channel message, one IATA + ('\x05', 2, 1, NULL, NULL), -- straddles h0 and h0+1h + ('\x06', 9, 1, NULL, NULL), -- TRACE + ('\x07', 2, 1, NULL, NULL) -- invalid path +) v(hash, pt, rt, origin, scope); +INSERT INTO channel_messages (id, channel_id, packet_hash, sender_name, sent_at) VALUES + (1, 1, '\x03', 'alice', $1::timestamptz + INTERVAL '5 minutes'), + (2, 1, '\x04', 'alice', $1::timestamptz + INTERVAL '6 minutes'); +SELECT setval(pg_get_serial_sequence('channel_messages', 'id'), 100); +INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, airtime_ms, payload_type) +SELECT n, hash, obs::uuid, iata, $1::timestamptz + mins * INTERVAL '1 minute', plb, hs, hc, pb, rssi, snr, air, pt FROM (VALUES + (1, '\x01'::bytea, '00000000-0000-0000-0000-00000000000a', 'YVR', 1, 65, 2, 1, '\xaabb'::bytea, -90, 5.5::real, 12.5::real, 4), + (2, '\x01', '00000000-0000-0000-0000-00000000000b', 'YYZ', 2, 0, 1, 0, NULL, -100, -3.25, NULL, 4), + (3, '\x02', '00000000-0000-0000-0000-00000000000a', 'YVR', 3, 2, 1, 2, '\xaabb', 0, 0, 10, 4), + (4, '\x03', '00000000-0000-0000-0000-00000000000a', 'YVR', 4, 1, 1, 1, '\xaa', -80, 8, 20, 5), + (5, '\x03', '00000000-0000-0000-0000-00000000000b', 'YYZ', 5, 1, 1, 1, '\xbb', -85, 7, NULL, 5), + (6, '\x04', '00000000-0000-0000-0000-00000000000b', 'YYZ', 6, 0, 1, 0, NULL, NULL, NULL, NULL, 5), + (7, '\x05', '00000000-0000-0000-0000-00000000000a', 'YVR', 59, 0, 1, 0, NULL, -120, -12, 5, 2), + (8, '\x05', '00000000-0000-0000-0000-00000000000b', 'YYZ', 61, 0, 1, 0, NULL, -121, -13, 5, 2), + (9, '\x06', '00000000-0000-0000-0000-00000000000a', 'YVR', 7, 3, 1, 3, '\x010203', -95, 2, 30, 9), + (10, '\x07', '00000000-0000-0000-0000-00000000000a', 'YVR', 8, 3, 1, 3, '\xaa', -95, 2, 30, 2) +) v(n, hash, obs, iata, mins, plb, hs, hc, pb, rssi, snr, air, pt);`, pgx.QueryExecModeSimpleProtocol, h0) + if err != nil { + t.Fatal(err) + } + return h0 +} + +// Raw-scan oracles: the SQL the old live views ran, grouped per hour. +var rollupOracles = map[string][2]string{ + "iata observations": { + `SELECT hour, iata, observation_count FROM analytics_hourly_iata_observations`, + `SELECT date_trunc('hour', heard_at, 'UTC'), iata, count(*) FROM packet_observations GROUP BY 1, 2`}, + "payload breakdown": { + `SELECT hour, iata, payload_type, count FROM analytics_hourly_payload_breakdown`, + `SELECT date_trunc('hour', heard_at, 'UTC'), iata, payload_type, count(*) FROM packet_observations + WHERE payload_type IS NOT NULL GROUP BY 1, 2, 3`}, + "signal": { + `SELECT hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal`, + `WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) THEN rssi::double precision END AS rssi + FROM packet_observations + ), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, width_bucket(rssi, -140, 0, 14) AS rssi_bin FROM samples + ) + SELECT hour, iata, grouping(snr_bin, rssi_bin)::integer, COALESCE(snr_bin, -1)::integer, COALESCE(rssi_bin, -1)::integer, + count(*)::bigint, count(snr)::bigint, COALESCE(sum(snr), 0)::double precision, + count(rssi)::bigint, COALESCE(sum(rssi), 0)::double precision + FROM binned GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin))`}, + "paths": { + `SELECT hour, iata, category, hash_bytes, entries, receptions FROM analytics_hourly_paths`, + `WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM packet_observations + ), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified + ) + SELECT hour, iata, category, hash_bytes, entries, count(*)::bigint FROM buckets GROUP BY 1, 2, 3, 4, 5`}, + "observer activity": { + `SELECT hour, observer_id, payload_type, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n + FROM analytics_hourly_observer_activity`, + `SELECT date_trunc('hour', heard_at, 'UTC'), observer_id, COALESCE(payload_type, -1)::smallint, + COUNT(*)::bigint, SUM(airtime_ms)::real, COUNT(airtime_ms)::bigint, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint + FROM packet_observations GROUP BY 1, 2, 3`}, + "observer identity": { + `SELECT hour, iata, observer_id, observation_count, display_name, observer_type FROM analytics_hourly_observer_identity`, + `SELECT date_trunc('hour', po.heard_at, 'UTC'), po.iata, po.observer_id, count(*), o.display_name, o.observer_type + FROM packet_observations po JOIN observers o ON o.id = po.observer_id GROUP BY 1, 2, 3, 5, 6`}, + "scope observers": { + `SELECT hour, iata, scope_id, observer_id FROM analytics_hourly_scope_observers`, + `SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.iata, p.scope_id, po.observer_id + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash WHERE p.scope_id IS NOT NULL`}, + "scope nodes": { + `SELECT hour, iata, scope_id, origin_pubkey FROM analytics_hourly_scope_nodes`, + `SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.iata, p.scope_id, p.origin_pubkey + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.payload_type = 4 AND p.origin_pubkey IS NOT NULL AND p.scope_id IS NOT NULL`}, + "advert hearings": { + `SELECT hour, iata, origin_pubkey, observations, last_heard, name, node_type FROM analytics_hourly_advert_hearings`, + `SELECT date_trunc('hour', po.heard_at, 'UTC'), po.iata, p.origin_pubkey, count(*), max(po.heard_at), n.name, n.node_type + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + LEFT JOIN nodes n ON n.public_key = p.origin_pubkey + WHERE po.payload_type = 4 AND p.origin_pubkey IS NOT NULL GROUP BY 1, 2, 3, 6, 7`}, +} + +// Set families must give exact distinct counts for any IATA set, summed per hour. +var rollupSetOracles = map[string][2]string{ + "packets": { + `SELECT COALESCE(sum(packets), 0) FROM analytics_hourly_packet_sets WHERE iatas && $1::bpchar[]`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', heard_at, 'UTC'), packet_hash + FROM packet_observations WHERE iata = ANY($1::bpchar[])) d`}, + "scoped packets": { + `SELECT COALESCE(sum(packets), 0) FROM analytics_hourly_scope_sets WHERE iatas && $1::bpchar[] AND scope_id = 1`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.packet_hash + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND p.scope_id = 1) d`}, + "adverts": { + `SELECT COALESCE(sum(advert_packets), 0) FROM analytics_hourly_advert_sets WHERE iatas && $1::bpchar[]`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.packet_hash + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND po.payload_type = 4) d`}, + "flood adverts": { + `SELECT COALESCE(sum(flood_packets), 0) FROM analytics_hourly_advert_sets WHERE iatas && $1::bpchar[]`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.packet_hash + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND po.payload_type = 4 AND p.route_type IN (0, 1)) d`}, + "messages": { + `SELECT COALESCE(sum(messages), 0) FROM analytics_hourly_talker_sets WHERE iatas && $1::bpchar[] AND sender_name = 'alice'`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), cm.id + FROM packet_observations po JOIN channel_messages cm ON cm.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND cm.sender_name = 'alice') d`}, +} + +// rollTxHours registers every hour holding observations (eligible or not) and rolls all +// registered hours inside tx, so tx-scoped fixtures can read rollup-backed stats. +func rollTxHours(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + if _, err := tx.Exec(ctx, `INSERT INTO analytics_rollup_hours (hour, status) + SELECT DISTINCT date_trunc('hour', heard_at, 'UTC'), 'missing' FROM packet_observations + ON CONFLICT (hour) DO NOTHING`); err != nil { + t.Fatal(err) + } + rows, err := tx.Query(ctx, "SELECT hour FROM analytics_rollup_hours ORDER BY hour") + if err != nil { + t.Fatal(err) + } + hours, err := pgx.CollectRows(rows, pgx.RowTo[time.Time]) + if err != nil { + t.Fatal(err) + } + for _, h := range hours { + if outcome, _, err := rollHour(ctx, tx, h); err != nil || outcome != RollComplete { + t.Fatalf("roll %s: outcome %d err %v", h, outcome, err) + } + } +} + +// backfillMessage stores a decrypted-late message the way channel backfill does. +func backfillMessage(t *testing.T, ctx context.Context, store *Store, hash byte, sender string, at time.Time) { + t.Helper() + m, err := store.InsertChannelMessage(ctx, ingest.InsertChannelMessageParams{ + ChannelID: 1, PacketHash: []byte{hash}, SenderName: sender, Content: "late", SentAt: at, Historical: true, + }) + if err != nil || m == nil { + t.Fatalf("backfill message %x: %v %v", hash, m, err) + } +} + +func rowsText(t *testing.T, ctx context.Context, pool *pgxpool.Pool, query string) string { + t.Helper() + var s *string + if err := pool.QueryRow(ctx, "SELECT string_agg(x::text, E'\\n' ORDER BY x::text) FROM ("+query+") x").Scan(&s); err != nil { + t.Fatalf("%v\n%s", err, query) + } + if s == nil { + return "" + } + return *s +} + +func rollAll(t *testing.T, ctx context.Context, r *RollupSession) { + t.Helper() + hours, err := r.MissingHours(ctx, 100) + if err != nil { + t.Fatal(err) + } + for _, h := range hours { + if outcome, _, err := r.RollHour(ctx, h); err != nil || outcome != RollComplete { + t.Fatalf("roll %s: outcome %d, err %v", h, outcome, err) + } + } +} + +func revision(t *testing.T, ctx context.Context, s *Store) int64 { + t.Helper() + rev, err := s.AnalyticsRevision(ctx) + if err != nil { + t.Fatal(err) + } + return rev +} + +func TestRollupHourPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if other, ok, err := store.BeginRollup(ctx); err != nil || ok { + if other != nil { + other.Close(ctx) + } + t.Fatalf("second BeginRollup: ok=%v err=%v, want lock held", ok, err) + } + + if err := r.RegisterHours(ctx, h0.Add(-10*time.Hour)); err != nil { + t.Fatal(err) + } + var first, last time.Time + var eligibleLast bool + if err := pool.QueryRow(ctx, `SELECT min(hour), max(hour), max(hour) = date_trunc('hour', now() - INTERVAL '95 minutes', 'UTC') + FROM analytics_rollup_hours WHERE status = 'missing'`).Scan(&first, &last, &eligibleLast); err != nil { + t.Fatal(err) + } + if !first.Equal(h0) || !eligibleLast { + t.Fatalf("registered hours %s..%s, want %s..now-95m", first, last, h0) + } + + rev0 := revision(t, ctx, store) + rollAll(t, ctx, r) + rev1 := revision(t, ctx, store) + if rev1 == rev0 { + t.Error("first roll did not bump the revision") + } + + for name, q := range rollupOracles { + if got, want := rowsText(t, ctx, pool, q[0]), rowsText(t, ctx, pool, q[1]); got != want { + t.Errorf("%s differs from raw:\nrollup:\n%s\nraw:\n%s", name, got, want) + } + } + for name, q := range rollupSetOracles { + for _, set := range [][]string{{"YVR"}, {"YYZ"}, {"YVR", "YYZ"}, {"YOW"}, {"YOW", "YVR", "YYZ"}} { + var got, want int64 + if err := pool.QueryRow(ctx, q[0], set).Scan(&got); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, q[1], set).Scan(&want); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("%s for %v: rollup %d, raw %d", name, set, got, want) + } + } + } + + // Empty hour: complete, no family rows. + var emptyStatus string + var emptyRows int + if err := pool.QueryRow(ctx, `SELECT status, (SELECT count(*) FROM analytics_hourly_iata_observations WHERE hour = $1) + FROM analytics_rollup_hours WHERE hour = $1`, h0.Add(2*time.Hour)).Scan(&emptyStatus, &emptyRows); err != nil { + t.Fatal(err) + } + if emptyStatus != "complete" || emptyRows != 0 { + t.Errorf("empty hour status %q rows %d, want complete, 0", emptyStatus, emptyRows) + } + + // Re-rolling unchanged data is a no-op for readers. + if outcome, changed, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete || changed { + t.Fatalf("idempotent re-roll: outcome %d changed %v err %v", outcome, changed, err) + } + if revision(t, ctx, store) != rev1 { + t.Error("idempotent re-roll bumped the revision") + } + + // A backfilled message dirties every hour its packet was heard in; re-rolling picks it up. + backfillMessage(t, ctx, store, 0x05, "bob", h0) + dirty, err := r.DirtyHours(ctx, 10) + if err != nil || len(dirty) != 2 || !dirty[0].Equal(h0) || !dirty[1].Equal(h0.Add(time.Hour)) { + t.Fatalf("dirty hours %v, %v; want h0 and h0+1h", dirty, err) + } + for _, h := range dirty { + if outcome, changed, err := r.RollHour(ctx, h); err != nil || outcome != RollComplete || !changed { + t.Fatalf("dirty re-roll %s: outcome %d changed %v err %v", h, outcome, changed, err) + } + } + if revision(t, ctx, store) <= rev1 { + t.Error("changed re-roll did not bump the revision") + } + if dirty, err := r.DirtyHours(ctx, 10); err != nil || len(dirty) != 0 { + t.Errorf("dirty hours after re-roll %v, %v; want none", dirty, err) + } + var bob int64 + if err := pool.QueryRow(ctx, `SELECT COALESCE(sum(messages), 0) FROM analytics_hourly_talker_sets WHERE sender_name = 'bob'`).Scan(&bob); err != nil || bob != 2 { + t.Errorf("bob messages across the straddled hours = %d, %v; want 2", bob, err) + } +} + +func TestRollupPartialHoursPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0.Add(-10*time.Hour)); err != nil { + t.Fatal(err) + } + + // Rolling h0+1h first leaves h0 missing when cleanup reaches it. + if outcome, _, err := r.RollHour(ctx, h0.Add(time.Hour)); err != nil || outcome != RollComplete { + t.Fatalf("roll h0+1h: %d %v", outcome, err) + } + if _, err := pool.Exec(ctx, "UPDATE analytics_raw_state SET raw_deleted_before = $1", h0.Add(20*time.Minute)); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollPartial { + t.Fatalf("roll h0 after raw deletion: outcome %d err %v, want partial", outcome, err) + } + if err := r.MarkPartialHours(ctx); err != nil { + t.Fatal(err) + } + var statuses string + if err := pool.QueryRow(ctx, `SELECT string_agg(status, ',' ORDER BY hour) FROM analytics_rollup_hours WHERE hour IN ($1, $2)`, + h0, h0.Add(time.Hour)).Scan(&statuses); err != nil { + t.Fatal(err) + } + if statuses != "partial,complete" { + t.Errorf("statuses %q, want partial,complete", statuses) + } + + // Only complete hours with intact raw rows are re-rolled; h0's entry is dropped. + backfillMessage(t, ctx, store, 0x01, "carol", h0) + backfillMessage(t, ctx, store, 0x05, "bob", h0) + if dirty, err := r.DirtyHours(ctx, 10); err != nil || len(dirty) != 1 || !dirty[0].Equal(h0.Add(time.Hour)) { + t.Errorf("dirty hours %v, %v; want only h0+1h", dirty, err) + } + if missing, err := r.MissingHours(ctx, 100); err != nil || len(missing) == 0 || !missing[0].After(h0.Add(time.Hour)) { + t.Errorf("missing hours %v, %v; want only hours after h0+1h", missing, err) + } +} + +// A roll that dies mid-transaction leaves the hour missing for the next pass. +func TestRollupInterruptedHourPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + // Breaking a family table makes the roll fail after earlier families were written. + if _, err := pool.Exec(ctx, "ALTER TABLE analytics_hourly_scope_sets ADD CONSTRAINT boom CHECK (packets < 0) NOT VALID"); err != nil { + t.Fatal(err) + } + if _, _, err := r.RollHour(ctx, h0); err == nil { + t.Fatal("roll succeeded despite a failing family") + } + var status string + var rows int + if err := pool.QueryRow(ctx, `SELECT status, (SELECT count(*) FROM analytics_hourly_iata_observations WHERE hour = $1) + FROM analytics_rollup_hours WHERE hour = $1`, h0).Scan(&status, &rows); err != nil { + t.Fatal(err) + } + if status != "missing" || rows != 0 { + t.Errorf("after failed roll: status %q, %d family rows; want missing, 0", status, rows) + } + if _, err := pool.Exec(ctx, "ALTER TABLE analytics_hourly_scope_sets DROP CONSTRAINT boom"); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete { + t.Errorf("retry: outcome %d err %v", outcome, err) + } +} + +// A message backfilled after a first roll's snapshot (but before it commits) is not lost: +// its dirty entry is invisible to that roll, survives it, and triggers a re-roll. +func TestRollupBackfillDuringFirstRollPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + + roll, err := pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + if err != nil { + t.Fatal(err) + } + defer roll.Rollback(ctx) + if _, err := roll.Exec(ctx, "SELECT count(*) FROM channel_messages"); err != nil { // fixes the snapshot + t.Fatal(err) + } + backfillMessage(t, ctx, store, 0x05, "bob", h0) // commits on another connection + if outcome, _, err := rollHour(ctx, roll, h0); err != nil || outcome != RollComplete { + t.Fatalf("first roll: %d %v", outcome, err) + } + if err := roll.Commit(ctx); err != nil { + t.Fatal(err) + } + talkers := func() int64 { + t.Helper() + var n int64 + if err := pool.QueryRow(ctx, "SELECT COALESCE(sum(messages), 0) FROM analytics_hourly_talker_sets WHERE hour = $1 AND sender_name = 'bob'", h0).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + if talkers() != 0 { + t.Fatal("first roll saw the message; the race was not reproduced") + } + dirty, err := r.DirtyHours(ctx, 10) + if err != nil || len(dirty) == 0 || !dirty[0].Equal(h0) { + t.Fatalf("dirty hours %v, %v; want h0 still queued", dirty, err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete { + t.Fatalf("re-roll: %d %v", outcome, err) + } + if talkers() != 1 { + t.Error("re-roll did not pick up the backfilled message") + } +} + +// With no observations there is nothing to roll, so no empty "complete" hours. +func TestRollupRegisterEmptyDatabasePostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + r, ok, err := New(pool, 0, 0).BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, time.Now().Add(-7*24*time.Hour)); err != nil { + t.Fatal(err) + } + var n int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM analytics_rollup_hours").Scan(&n); err != nil || n != 0 { + t.Errorf("registered %d hours on an empty database, err %v; want 0", n, err) + } +} + +// An observation may be heard up to 30 minutes after its packet's last_heard_at, so an hour +// starting exactly raw_deleted_before + 30 min may have lost rows too. +func TestRollupPartialBoundaryPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + r, ok, err := New(pool, 0, 0).BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + h1 := h0.Add(time.Hour) + if _, err := pool.Exec(ctx, "UPDATE analytics_raw_state SET raw_deleted_before = $1", h1.Add(-30*time.Minute)); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h1); err != nil || outcome != RollPartial { + t.Errorf("hour at the boundary: outcome %d err %v, want partial", outcome, err) + } + if outcome, _, err := r.RollHour(ctx, h1.Add(time.Hour)); err != nil || outcome != RollComplete { + t.Errorf("next hour: outcome %d err %v, want complete", outcome, err) + } +} diff --git a/db/rollup_schema_integration_test.go b/db/rollup_schema_integration_test.go new file mode 100644 index 00000000..169e07fe --- /dev/null +++ b/db/rollup_schema_integration_test.go @@ -0,0 +1,117 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +func TestRollupSchemaPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(ctx) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + applyBaseline(t, ctx, tx) + + // Each failing insert runs in a savepoint so the tx stays usable. + bad := map[string]string{ + "duplicate": "'{YVR,YVR}'", + "unsorted": "'{YYZ,YVR}'", + "empty": "'{}'", + "null member": "'{YVR,NULL}'", + } + inserts := map[string]string{ + "packet_sets": "INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) VALUES (now(), %s, 1)", + "scope_sets": "INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) VALUES (now(), %s, 1, 1)", + "advert_sets": "INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) VALUES (now(), '\\x01', %s, 1, 1, 0)", + "talker_sets": "INSERT INTO analytics_hourly_talker_sets (hour, sender_name, iatas, messages, last_sent) VALUES (now(), 'a', %s, 1, now())", + } + for tname, tmpl := range inserts { + for name, lit := range bad { + sql := fmt.Sprintf(tmpl, lit) + if _, err := tx.Exec(ctx, "SAVEPOINT s"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, sql); err == nil { + t.Errorf("%s %s: insert succeeded, want CHECK violation", tname, name) + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT s"); err != nil { + t.Fatal(err) + } + } + if _, err := tx.Exec(ctx, fmt.Sprintf(tmpl, "'{YVR,YYZ}'")); err != nil { + t.Errorf("%s canonical set: %v", tname, err) + } + } + + for _, tbl := range []string{"analytics_state", "analytics_raw_state"} { + var n int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM "+tbl).Scan(&n); err != nil || n != 1 { + t.Errorf("%s rows = %d, err %v; want 1", tbl, n, err) + } + if _, err := tx.Exec(ctx, "SAVEPOINT s"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, "INSERT INTO "+tbl+" (singleton) VALUES (false)"); err == nil { + t.Errorf("%s accepted a second row", tbl) + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT s"); err != nil { + t.Fatal(err) + } + } + var rev int64 + if err := tx.QueryRow(ctx, "SELECT revision FROM analytics_state").Scan(&rev); err != nil || rev != 0 { + t.Errorf("revision = %d, err %v; want 0", rev, err) + } + var rawDeleted *time.Time + if err := tx.QueryRow(ctx, "SELECT raw_deleted_before FROM analytics_raw_state").Scan(&rawDeleted); err != nil || rawDeleted != nil { + t.Errorf("raw_deleted_before = %v, err %v; want NULL", rawDeleted, err) + } + + var heard, brin, obsID, lastIATA *string + if err := tx.QueryRow(ctx, `SELECT to_regclass('idx_observations_heard')::text, to_regclass('idx_observations_heard_brin')::text, + to_regclass('idx_observations_observer_id')::text, to_regclass('idx_observers_last_iata')::text`).Scan(&heard, &brin, &obsID, &lastIATA); err != nil { + t.Fatal(err) + } + if heard == nil || obsID == nil || lastIATA == nil { + t.Errorf("missing index: heard=%v observer_id=%v last_iata=%v", heard, obsID, lastIATA) + } + if brin != nil { + t.Errorf("idx_observations_heard_brin still exists") + } + + var nullable string + if err := tx.QueryRow(ctx, `SELECT is_nullable FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = 'channels' AND column_name = 'message_count'`).Scan(&nullable); err != nil || nullable != "NO" { + t.Errorf("channels.message_count nullable = %q, err %v; want NO", nullable, err) + } + for _, c := range [][2]string{{"packets", "observation_count"}, {"observers", "last_iata"}} { + var n int + if err := tx.QueryRow(ctx, `SELECT count(*) FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1 AND column_name = $2`, c[0], c[1]).Scan(&n); err != nil || n != 1 { + t.Errorf("column %s.%s missing (err %v)", c[0], c[1], err) + } + } + if _, err := tx.Exec(ctx, "INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at) VALUES ('\\x01', now(), now())"); err != nil { + t.Errorf("trace_tags: %v", err) + } +} diff --git a/db/route_evidence.go b/db/route_evidence.go new file mode 100644 index 00000000..e7017ea5 --- /dev/null +++ b/db/route_evidence.go @@ -0,0 +1,93 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "math" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgtype" +) + +func savedRoutePath(prefixes [][]byte, count int32) (int16, []byte, bool) { + if count < 2 || count > 63 || len(prefixes) != int(count) { + return 0, nil, false + } + width := len(prefixes[0]) + if width < 1 || width > 3 { + return 0, nil, false + } + for _, p := range prefixes { + if len(p) != width { + return 0, nil, false + } + } + return int16(width), bytes.Join(prefixes, nil), true +} + +func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + if !api.ValidRouteEvidenceKey(iata, key) || !api.ValidRouteEvidenceWindow(query.Since, query.Until) || query.Limit < 1 || query.Limit > 200 { + return nil, api.ErrRouteEvidenceInput + } + if c := query.Cursor; c != nil { + if c.IATA != iata || c.PathKey != key || !c.Since.Equal(query.Since) || !c.Until.Equal(query.Until) || c.ID <= 0 || c.HeardAt.Before(query.Since) || !c.HeardAt.Before(query.Until) { + return nil, api.ErrRouteEvidenceInput + } + } + keyBytes, _ := hex.DecodeString(key) + row, err := s.q.GetRouteEvidenceRoute(ctx, sqlc.GetRouteEvidenceRouteParams{Iata: iata, PathKey: keyBytes}) + if err != nil { + return nil, err + } + nodes, err := s.GetNodesByIDs(ctx, row.NodeIds) + if err != nil { + return nil, err + } + route := toKnownRoutes([]knownRouteRow{{ID: row.ID, NodeIds: row.NodeIds, HashPrefix: row.HashPrefix, Iata: row.Iata, HopCount: row.HopCount, FirstSeen: row.FirstSeen, LastSeen: row.LastSeen, ObservationCount: row.ObservationCount}}, nodes)[0] + route.PathKey = hex.EncodeToString(row.PathKey) + out := &api.RouteEvidence{Items: []api.RouteObservation{}, Route: route, WindowStart: query.Since.UnixMilli(), WindowEnd: query.Until.UnixMilli(), GeneratedAt: time.Now().UnixMilli(), MatchType: "saved_path_prefixes"} + width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) + if !valid { + return out, nil + } + out.MatchAvailable = true + out.HashSize = width + out.PathBytes = hex.EncodeToString(path) + beforeAt, beforeID := query.Until, int64(math.MaxInt64) + if query.Cursor != nil { + beforeAt, beforeID = query.Cursor.HeardAt, query.Cursor.ID + } + ts := func(t time.Time) pgtype.Timestamptz { return pgtype.Timestamptz{Time: t, Valid: true} } + digest := md5.Sum(path) + rows, err := s.q.ListRouteEvidence(ctx, sqlc.ListRouteEvidenceParams{Iata: iata, HashSize: width, HopCount: int16(row.HopCount), PathDigest: digest[:], PathBytes: path, Since: ts(query.Since), Until: ts(query.Until), BeforeAt: ts(beforeAt), BeforeID: beforeID, PageLimit: query.Limit + 1}) + if err != nil { + return nil, err + } + out.HasMore = len(rows) > int(query.Limit) + if out.HasMore { + rows = rows[:query.Limit] + } + for _, r := range rows { + payload := int16(-1) + if r.PayloadType != nil { + payload = *r.PayloadType + } + snr := r.Snr + if snr != nil && (math.IsNaN(float64(*snr)) || math.IsInf(float64(*snr), 0)) { + snr = nil + } + out.Items = append(out.Items, api.RouteObservation{ID: r.ID, PacketHash: hex.EncodeToString(r.PacketHash), ObserverID: r.ObserverID, ObserverName: r.ObserverName, HeardAt: r.HeardAt.Time.UnixMilli(), PayloadType: payload, PayloadTypeName: api.PayloadTypeName(payload), RSSI: r.Rssi, SNR: snr}) + } + if out.HasMore { + last := rows[len(rows)-1] + cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID}).String() + out.NextPageCursor = &cursor + } + return out, nil +} diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go new file mode 100644 index 00000000..a84b534f --- /dev/null +++ b/db/route_evidence_integration_test.go @@ -0,0 +1,195 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +type evidenceCapture struct { + pgx.Tx + statement string + args []any +} + +func (c *evidenceCapture) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + if strings.Contains(sql, "-- name: ListRouteEvidence") { + c.statement = sql + c.args = args + } + return c.Tx.Query(ctx, sql, args...) +} + +func TestRouteEvidencePostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + _, err := tx.Exec(ctx, ` + INSERT INTO observers(id,public_key,display_name) VALUES ('00000000-0000-0000-0000-000000000001','\x01','One'),('00000000-0000-0000-0000-000000000002','\x02','Two'); + INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa',2,'A'),('00000000-0000-0000-0000-000000000002','\xbb',2,'B'); + INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) + SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(1,10) i;`) + if err != nil { + t.Fatal(err) + } + capture := &evidenceCapture{Tx: tx} + store := &Store{q: sqlc.New(capture)} + ids := []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000001"), uuid.MustParse("00000000-0000-0000-0000-000000000002")} + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } + key := hex.EncodeToString(routePathKey(ids)) + since := time.Now().UTC().Truncate(time.Millisecond).Add(-time.Hour) + until := since.Add(time.Hour) + at := since.Add(time.Minute + 123456*time.Microsecond) + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,snr,path_length_byte) + SELECT i,int4send(i), '00000000-0000-0000-0000-000000000001',CASE WHEN i=5 THEN 'YVR' ELSE 'YOW' END, + CASE WHEN i=9 THEN $2::timestamptz WHEN i=1 THEN $1::timestamptz-interval '2 microseconds' ELSE $1::timestamptz END, + CASE WHEN i=4 THEN 2 ELSE 1 END,CASE WHEN i=4 THEN 1 WHEN i=10 THEN 3 ELSE 2 END, + CASE WHEN i=6 THEN '\xabbb'::bytea WHEN i=10 THEN '\xaabbcc'::bytea ELSE '\xaabb'::bytea END, + CASE WHEN i=7 THEN 9 WHEN i=8 THEN NULL ELSE 4 END,CASE WHEN i=1 THEN 'NaN'::real ELSE 0 END,0 FROM generate_series(1,10) i; + `, at, until) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,path_length_byte) VALUES (11,int4send(1),'00000000-0000-0000-0000-000000000002','YOW',$1,1,2,'\xaabb',4,0)`, at.Add(-time.Microsecond)); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `UPDATE packet_observations SET path_length_byte=((hash_size-1)<<6)|hop_count`); err != nil { + t.Fatal(err) + } + query := api.RouteEvidenceQuery{Since: since, Until: until, Limit: 2} + first, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil { + t.Fatal(err) + } + if !first.HasMore || first.NextPageCursor == nil || len(first.Items) != 2 || first.Items[0].ID != 3 || first.Items[1].ID != 2 || first.Route.PathKey != key || first.HashSize != 1 || first.PathBytes != "aabb" { + t.Fatalf("first page: %+v", first) + } + query.Cursor, err = api.ParseRouteEvidenceCursor(*first.NextPageCursor) + if err != nil { + t.Fatal(err) + } + second, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil { + t.Fatal(err) + } + if second.HasMore || len(second.Items) != 2 || second.Items[0].ID != 11 || second.Items[1].ID != 1 || second.Items[1].SNR != nil { + t.Fatalf("precise second page: %+v", second) + } + if _, err = json.Marshal(second); err != nil { + t.Fatalf("non-finite JSON: %v", err) + } + query.Cursor.IATA = "YVR" + if _, err = store.GetRouteEvidence(ctx, "YOW", key, query); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatal("cross-route cursor accepted") + } + query.Cursor = nil + if _, err = store.GetRouteEvidence(ctx, "YVR", key, query); !errors.Is(err, pgx.ErrNoRows) { + t.Fatal("unknown route did not return not-found") + } + if _, err = tx.Exec(ctx, "DELETE FROM packets"); err != nil { + t.Fatal(err) + } + empty, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || !empty.MatchAvailable || len(empty.Items) != 0 || empty.HasMore { + t.Fatalf("expired evidence: %+v %v", empty, err) + } + + // Many unrelated paths must not turn this into a raw-table or route-ID scan. + started := time.Now() + _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) + SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(100,200099) i;`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,path_bytes,payload_type) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',$1,2,1,2, + CASE WHEN get_byte(packet_hash,3)=0 THEN '\xaabb'::bytea ELSE packet_hash END,4 FROM packets;`, at) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, "ANALYZE packet_observations; ANALYZE observers;"); err != nil { + t.Fatal(err) + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err = tx.Exec(ctx, "SET LOCAL plan_cache_mode = "+mode); err != nil { + t.Fatal(err) + } + result, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || len(result.Items) != 2 { + t.Fatalf("indexed read: %v", err) + } + var plan string + if err = tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+capture.statement, capture.args...).Scan(&plan); err != nil { + t.Fatal(err) + } + if !strings.Contains(plan, "idx_observations_route_evidence") { + t.Fatalf("missing evidence index: %s", plan) + } + t.Logf("%s plan: %s", mode, plan) + } + t.Logf("200000-row fixture insert/analyze/read: %s", time.Since(started)) + if _, err = tx.Exec(ctx, `UPDATE known_routes SET hash_prefix=ARRAY['\xaa'::bytea]`); err != nil { + t.Fatal(err) + } + unavailable, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || unavailable.MatchAvailable || len(unavailable.Items) != 0 { + t.Fatalf("malformed stored path: %+v %v", unavailable, err) + } +} + +func TestRouteEvidenceIndexPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT int4send(n),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(0,2) n; +INSERT INTO packet_observations(packet_hash,observer_id,iata,path_length_byte,hash_size,path_bytes,heard_at,payload_type,hop_count) +SELECT int4send(n),'00000000-0000-0000-0000-000000000001','YOW',n,1,'\xaabb',NOW(),4,n FROM generate_series(0,2) n; +ANALYZE packet_observations`); err != nil { + t.Fatal(err) + } + var indexed float32 + if err := tx.QueryRow(ctx, `SELECT reltuples FROM pg_class WHERE oid='idx_observations_route_evidence'::regclass`).Scan(&indexed); err != nil || indexed != 1 { + t.Fatalf("0/1-hop rows entered the route index: %v %v", indexed, err) + } +} + +func TestRouteEvidenceFollowsPrefixWidthPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +VALUES (int4send(1),4,0,1,'\x00','\x00',NOW(),NOW()); +INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,path_length_byte) +VALUES (1,int4send(1),'00000000-0000-0000-0000-000000000001','YOW',NOW(),2,2,'\xaa11bb22',4,(1<<6)|2)`); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + ids := []uuid.UUID{uuid.New(), uuid.New()} + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa, 0x11}, {0xbb, 0x22}}, "YOW", 2); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + got, err := store.GetRouteEvidence(ctx, "YOW", hex.EncodeToString(routePathKey(ids)), api.RouteEvidenceQuery{Since: now.Add(-time.Hour), Until: now.Add(time.Minute), Limit: 10}) + if err != nil { + t.Fatal(err) + } + if got.HashSize != 2 || got.PathBytes != "aa11bb22" || len(got.Items) != 1 || got.Route.ObservationCount != 2 { + t.Fatalf("evidence kept the first prefix width: %+v", got) + } +} diff --git a/db/route_evidence_test.go b/db/route_evidence_test.go new file mode 100644 index 00000000..868b565b --- /dev/null +++ b/db/route_evidence_test.go @@ -0,0 +1,23 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "bytes" + "testing" +) + +func TestSavedRoutePath(t *testing.T) { + size, path, ok := savedRoutePath([][]byte{{0xaa}, {0xbb}}, 2) + if !ok || size != 1 || !bytes.Equal(path, []byte{0xaa, 0xbb}) { + t.Fatal("valid path lost") + } + for _, prefixes := range [][][]byte{nil, {{1}}, {{1}, {2, 3}}, {{}, {2}}, {{1, 2, 3, 4}, {5, 6, 7, 8}}} { + if _, _, ok := savedRoutePath(prefixes, int32(len(prefixes))); ok { + t.Fatalf("accepted malformed path %x", prefixes) + } + } + if _, _, ok := savedRoutePath([][]byte{{1}, {2}}, 3); ok { + t.Fatal("accepted inconsistent hop count") + } +} diff --git a/db/routes.go b/db/routes.go index 54e88876..73b17c54 100644 --- a/db/routes.go +++ b/db/routes.go @@ -5,7 +5,9 @@ package db import ( "context" + "crypto/md5" "encoding/hex" + "strings" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -14,29 +16,46 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +// routePathKey is the route's identity digest: md5 over the comma-joined +// node UUIDs, matching Postgres's decode(md5(array_to_string(node_ids, ',')), 'hex'). +func routePathKey(nodeIDs []uuid.UUID) []byte { + parts := make([]string, len(nodeIDs)) + for i, id := range nodeIDs { + parts[i] = id.String() + } + sum := md5.Sum([]byte(strings.Join(parts, ","))) + return sum[:] +} + func (s *Store) UpsertKnownRoute(ctx context.Context, nodeIDs []uuid.UUID, hashPrefix [][]byte, iata string, hopCount int32) error { return s.q.UpsertKnownRoute(ctx, sqlc.UpsertKnownRouteParams{ + PathKey: routePathKey(nodeIDs), NodeIds: nodeIDs, HashPrefix: hashPrefix, Iata: iata, - HopCount: int32(hopCount), + HopCount: hopCount, }) } -func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) { +func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]api.KnownRoute, error) { var cursorTS pgtype.Timestamptz if !cursor.IsZero() { cursorTS = pgtype.Timestamptz{Time: cursor, Valid: true} } - rows, err := s.q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{ + sqlRows, err := s.q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{ Column1: iata, Column2: hopCount, Column3: cursorTS, Limit: limit, + Column5: cursorID, }) if err != nil { return nil, err } + rows := make([]knownRouteRow, len(sqlRows)) + for i, r := range sqlRows { + rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount} + } ids := collectNodeIDs(rows) nodes, err := s.GetNodesByIDs(ctx, ids) if err != nil { @@ -54,7 +73,7 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st if err != nil { return nil, err } - rows, err := s.q.SearchKnownRoutes(ctx, sqlc.SearchKnownRoutesParams{ + sqlRows, err := s.q.SearchKnownRoutes(ctx, sqlc.SearchKnownRoutesParams{ Iata: iata, Column2: fromBytes, Column3: toBytes, @@ -62,6 +81,10 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st if err != nil { return nil, err } + rows := make([]knownRouteRow, len(sqlRows)) + for i, r := range sqlRows { + rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount} + } ids := collectNodeIDs(rows) nodes, err := s.GetNodesByIDs(ctx, ids) if err != nil { @@ -96,6 +119,7 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st hops = append(hops, hop) } items = append(items, api.KnownRoute{ + PathKey: hex.EncodeToString(routePathKey(r.NodeIds)), ID: r.ID, IATA: r.Iata, HopCount: int32(len(hops)), @@ -109,13 +133,17 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st } func (s *Store) GetKnownRoutesByNode(ctx context.Context, iata string, nodeID uuid.UUID) ([]api.KnownRoute, error) { - rows, err := s.q.GetKnownRoutesByNode(ctx, sqlc.GetKnownRoutesByNodeParams{ + sqlRows, err := s.q.GetKnownRoutesByNode(ctx, sqlc.GetKnownRoutesByNodeParams{ Iata: iata, Column2: nodeID, }) if err != nil { return nil, err } + rows := make([]knownRouteRow, len(sqlRows)) + for i, r := range sqlRows { + rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount} + } ids := collectNodeIDs(rows) nodes, err := s.GetNodesByIDs(ctx, ids) if err != nil { @@ -260,8 +288,52 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t return results, nil } -func (s *Store) ReconfirmRoutes(ctx context.Context) error { - return s.q.ReconfirmRoutes(ctx) +// AmbiguousPrefixes is the per-IATA set of hop prefixes that resolve to more than one node. +type AmbiguousPrefixes struct { + IATAs []string + Lens []int32 + Prefixes [][]byte +} + +func (s *Store) AmbiguousPrefixes(ctx context.Context) (AmbiguousPrefixes, error) { + rows, err := s.q.AmbiguousPrefixes(ctx) + if err != nil { + return AmbiguousPrefixes{}, err + } + amb := AmbiguousPrefixes{IATAs: make([]string, 0, len(rows)), Lens: make([]int32, 0, len(rows)), Prefixes: make([][]byte, 0, len(rows))} + for _, r := range rows { + amb.IATAs = append(amb.IATAs, r.Iata) + amb.Lens = append(amb.Lens, r.Len) + amb.Prefixes = append(amb.Prefixes, r.Prefix) + } + return amb, nil +} + +// ReconfirmRoutes checks the batchSize least-recently-reconfirmed routes, +// deleting stale or ambiguous ones and stamping the survivors. +func (s *Store) ReconfirmRoutes(ctx context.Context, batchSize int32, before time.Time, amb AmbiguousPrefixes) (int64, error) { + return s.q.ReconfirmRoutes(ctx, sqlc.ReconfirmRoutesParams{ + BatchSize: batchSize, + Before: pgtype.Timestamptz{Time: before, Valid: true}, + AmbIata: amb.IATAs, + AmbLen: amb.Lens, + AmbPrefix: amb.Prefixes, + }) +} + +const routeDeleteBatch = 10000 + +// DeleteOldRoutes prunes routes per the retention rule: unconditionally past +// retentionCutoff, and past graceCutoff when observed fewer than minObservations times. +func (s *Store) DeleteOldRoutes(ctx context.Context, retentionCutoff time.Time, minObservations int64, graceCutoff time.Time) error { + return deleteInBatches(ctx, routeDeleteBatch, func(ctx context.Context, n int32) (int64, error) { + return s.q.DeleteOldRoutes(ctx, sqlc.DeleteOldRoutesParams{ + RetentionCutoff: pgtype.Timestamptz{Time: retentionCutoff, Valid: true}, + GraceCutoff: pgtype.Timestamptz{Time: graceCutoff, Valid: true}, + MinObservations: minObservations, + BatchSize: n, + }) + }) } // extractFromNode returns the portion of a route starting at the given node. @@ -274,7 +346,20 @@ func extractFromNode(hops []api.RouteHop, nodeID uuid.UUID) []api.RouteHop { return hops } -func toKnownRoutes(rows []sqlc.KnownRoute, nodes map[uuid.UUID]*api.ResolvedNode) []api.KnownRoute { +// knownRouteRow normalizes the per-query sqlc row structs (identical +// columns, distinct generated types) so the helpers below share one body. +type knownRouteRow struct { + ID int64 + NodeIds []uuid.UUID + HashPrefix [][]byte + Iata string + HopCount int32 + FirstSeen pgtype.Timestamptz + LastSeen pgtype.Timestamptz + ObservationCount int64 +} + +func toKnownRoutes(rows []knownRouteRow, nodes map[uuid.UUID]*api.ResolvedNode) []api.KnownRoute { items := make([]api.KnownRoute, 0, len(rows)) for _, r := range rows { hops := make([]api.RouteHop, 0, len(r.NodeIds)) @@ -289,6 +374,7 @@ func toKnownRoutes(rows []sqlc.KnownRoute, nodes map[uuid.UUID]*api.ResolvedNode hops = append(hops, hop) } items = append(items, api.KnownRoute{ + PathKey: hex.EncodeToString(routePathKey(r.NodeIds)), ID: r.ID, IATA: r.Iata, HopCount: r.HopCount, @@ -301,7 +387,7 @@ func toKnownRoutes(rows []sqlc.KnownRoute, nodes map[uuid.UUID]*api.ResolvedNode return items } -func collectNodeIDs(rows []sqlc.KnownRoute) []uuid.UUID { +func collectNodeIDs(rows []knownRouteRow) []uuid.UUID { seen := make(map[uuid.UUID]struct{}) var ids []uuid.UUID for _, r := range rows { diff --git a/db/routes_integration_test.go b/db/routes_integration_test.go new file mode 100644 index 00000000..7e6594c4 --- /dev/null +++ b/db/routes_integration_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + _ "embed" + "encoding/json" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +//go:embed queries/queries.sql +var routeTestQueries string + +// All synthetic rows are rolled back. The sparse +// region sits behind 100,000 newer routes to expose global timestamp scans. +func TestListKnownRoutesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + _, err = tx.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('GPT'); +INSERT INTO known_routes (id,path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) +SELECT i, decode(md5(i::text),'hex'), ARRAY[md5(i::text)::uuid], ARRAY['\x01'::bytea], + CASE WHEN i > 100000 THEN 'GPT' ELSE 'YYZ' END, i % 4 + 2, + '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second', + '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second' +FROM generate_series(1,100100) i; +ANALYZE known_routes;`) + if err != nil { + t.Fatal(err) + } + queries := strings.ReplaceAll(routeTestQueries, "\r\n", "\n") + querySQL := strings.Split(strings.Split(queries, "-- name: ListKnownRoutes :many\n")[1], "\n-- name:")[0] + if _, err := tx.Exec(ctx, "PREPARE route_page AS "+querySQL); err != nil { + t.Fatal(err) + } + q := sqlc.New(tx) + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + t.Run(mode, func(t *testing.T) { + if _, err := tx.Exec(ctx, "SET LOCAL plan_cache_mode = "+mode); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, iata string + hop, cursor, limit, first, stride, count int + }{ + {"global", "", 0, 0, 3, 1, 1, 3}, + {"common", "YYZ", 0, 0, 3, 1, 1, 3}, + {"sparse", "GPT", 0, 0, 50, 100001, 1, 50}, + {"sparse hop", "GPT", 2, 0, 50, 100004, 4, 25}, + {"global hop", "", 2, 0, 3, 4, 4, 3}, + {"global cursor", "", 0, 3, 3, 4, 1, 3}, + {"sparse cursor", "GPT", 0, 100010, 3, 100011, 1, 3}, + {"sparse hop cursor", "GPT", 2, 100010, 3, 100012, 4, 3}, + {"partial page", "GPT", 0, 100098, 50, 100099, 1, 2}, + {"end", "GPT", 0, 100100, 50, 0, 1, 0}, + {"unknown", "ZZZ", 0, 0, 50, 0, 1, 0}, + {"lowercase", "gpt", 0, 0, 50, 0, 1, 0}, + {"trailing space", "GPT ", 0, 0, 50, 0, 1, 0}, + {"overlong", "GPTX", 0, 0, 50, 0, 1, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + cursor := pgtype.Timestamptz{} + if tc.cursor != 0 { + cursor = pgtype.Timestamptz{Time: anchor.Add(-time.Duration(tc.cursor) * time.Second), Valid: true} + } + rows, err := q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{Column1: tc.iata, Column2: int32(tc.hop), Column3: cursor, Limit: int32(tc.limit)}) + if err != nil { + t.Fatal(err) + } + if len(rows) != tc.count { + t.Fatalf("got %d rows, want %d", len(rows), tc.count) + } + for i, r := range rows { + want := int64(tc.first + i*tc.stride) + wantIATA := "YYZ" + if want > 100000 { + wantIATA = "GPT" + } + if r.ID != want || r.Iata != wantIATA || r.HopCount != int32(want%4+2) || r.ObservationCount != 1 || + len(r.NodeIds) != 1 || len(r.HashPrefix) != 1 || len(r.HashPrefix[0]) != 1 || r.HashPrefix[0][0] != 1 || + !r.LastSeen.Time.Equal(anchor.Add(-time.Duration(want)*time.Second)) || !r.FirstSeen.Time.Equal(r.LastSeen.Time) { + t.Errorf("unexpected route %d at row %d; want %d", r.ID, i, want) + } + } + }) + } + for _, iata := range []string{"GPT", "YYZ", ""} { + var planJSON []byte + if err := tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE route_page('"+iata+"',0,NULL,50,0)").Scan(&planJSON); err != nil { + t.Fatal(err) + } + var plans []struct { + Plan map[string]any + ExecutionTime float64 `json:"Execution Time"` + } + if err := json.Unmarshal(planJSON, &plans); err != nil { + t.Fatal(err) + } + var visited float64 + var walk func(map[string]any) + walk = func(p map[string]any) { + if p["Relation Name"] == "known_routes" { + rows, _ := p["Actual Rows"].(float64) + removed, _ := p["Rows Removed by Filter"].(float64) + loops, _ := p["Actual Loops"].(float64) + visited += (rows + removed) * loops + } + if children, ok := p["Plans"].([]any); ok { + for _, child := range children { + walk(child.(map[string]any)) + } + } + } + walk(plans[0].Plan) + t.Logf("100,100 routes / 50-row page IATA=%q: %.3f ms, %.0f rows examined", iata, plans[0].ExecutionTime, visited) + if visited > 200 { + t.Errorf("page IATA=%q examined %.0f routes; budget 200", iata, visited) + } + } + }) + } +} diff --git a/db/routes_pagination_integration_test.go b/db/routes_pagination_integration_test.go new file mode 100644 index 00000000..81513292 --- /dev/null +++ b/db/routes_pagination_integration_test.go @@ -0,0 +1,78 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "slices" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +// Routes upserted in one batch share last_seen; paging must not skip any of them. +func TestKnownRoutePaginationTiesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + // Seven routes in one millisecond (two with sub-ms offsets), one older, one in another IATA. + if _, err := tx.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YYZ'), ('YOW') ON CONFLICT DO NOTHING; +INSERT INTO known_routes (id,path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) +SELECT i, decode(md5(i::text),'hex'), ARRAY[]::uuid[], ARRAY['\x01'::bytea], + CASE WHEN i = 9 THEN 'YOW' ELSE 'YYZ' END, 2, + '2026-01-01 00:00:00+00'::timestamptz, + CASE WHEN i = 8 THEN '2026-01-01 00:00:04+00'::timestamptz + WHEN i IN (3, 6) THEN '2026-01-01 00:00:05.000400+00'::timestamptz + ELSE '2026-01-01 00:00:05+00'::timestamptz END +FROM generate_series(1,9) i;`); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + for _, iata := range []string{"", "YYZ"} { + t.Run("iata="+iata, func(t *testing.T) { + var got []int64 + var cursor time.Time + var cursorID int64 + for page := 0; page < 10; page++ { + routes, err := store.ListKnownRoutes(ctx, iata, 0, cursor, cursorID, 3) + if err != nil { + t.Fatal(err) + } + if len(routes) == 0 { + break + } + for _, r := range routes { + got = append(got, r.ID) + } + last := routes[len(routes)-1] + cursor, cursorID = time.UnixMilli(last.LastSeen), last.ID + } + want := []int64{7, 6, 5, 4, 3, 2, 1, 8} + if iata == "" { + want = []int64{9, 7, 6, 5, 4, 3, 2, 1, 8} + } + if !slices.Equal(got, want) { + t.Fatalf("paged ids = %v, want %v", got, want) + } + }) + } +} diff --git a/db/routes_test.go b/db/routes_test.go index 1df7a6ec..034d57f4 100644 --- a/db/routes_test.go +++ b/db/routes_test.go @@ -4,12 +4,71 @@ package db import ( + "bytes" + "context" + "encoding/hex" "testing" + "time" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/google/uuid" + "go.uber.org/mock/gomock" ) +func TestRoutePathKey_MatchesPostgresDigest(t *testing.T) { + // Golden vector: Postgres computes + // decode(md5(array_to_string(node_ids, ',')), 'hex') + // over lowercase-hyphenated UUIDs. Migration 024 backfilled with that + // expression; this pins the Go side to the identical bytes. + a := uuid.MustParse("00000000-0000-0000-0000-000000000001") + b := uuid.MustParse("00000000-0000-0000-0000-000000000002") + got := hex.EncodeToString(routePathKey([]uuid.UUID{a, b})) + want := "f097439148601d9f3291c474f82fa64c" + if got != want { + t.Errorf("routePathKey = %s, want %s", got, want) + } +} + +func TestUpsertKnownRoute_ComputesPathKey(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + + a := uuid.MustParse("00000000-0000-0000-0000-000000000001") + b := uuid.MustParse("00000000-0000-0000-0000-000000000002") + wantKey, _ := hex.DecodeString("f097439148601d9f3291c474f82fa64c") + + mock.EXPECT().UpsertKnownRoute(gomock.Any(), gomock.Cond(func(p sqlc.UpsertKnownRouteParams) bool { + return bytes.Equal(p.PathKey, wantKey) + })).Return(nil) + + if err := store.UpsertKnownRoute(context.Background(), []uuid.UUID{a, b}, [][]byte{{0x37}, {0xd8}}, "PRG", 2); err != nil { + t.Fatal(err) + } +} + +func TestDeleteOldRoutes_PassesCutoffs(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + + retention := time.Date(2026, 7, 23, 0, 0, 0, 0, time.UTC) + grace := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC) + + mock.EXPECT().DeleteOldRoutes(gomock.Any(), gomock.Cond(func(p sqlc.DeleteOldRoutesParams) bool { + return p.RetentionCutoff.Time.Equal(retention) && p.MinObservations == 3 && p.GraceCutoff.Time.Equal(grace) && + p.BatchSize == routeDeleteBatch + })).Return(int64(0), nil) + + if err := store.DeleteOldRoutes(context.Background(), retention, 3, grace); err != nil { + t.Fatal(err) + } +} + func TestExtractFromNode_Found(t *testing.T) { a, b, c := uuid.New(), uuid.New(), uuid.New() hops := []api.RouteHop{ diff --git a/db/scope_stats_integration_test.go b/db/scope_stats_integration_test.go new file mode 100644 index 00000000..8a07435d --- /dev/null +++ b/db/scope_stats_integration_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "reflect" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +// Packets come from the rolled IATA sets (each packet once per hour, however many requested +// IATAs heard it); observers are current memberships filtered by their latest IATA. +func TestScopeStatsPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + var h time.Time + if err := pool.QueryRow(ctx, "SELECT date_trunc('hour', now(), 'UTC') - INTERVAL '4 hours'").Scan(&h); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YVR'), ('YYJ'), ('YYZ'); +INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) +SELECT i,name,decode(repeat('00',16),'hex'),decode(lpad(to_hex(i),16,'0'),'hex') +FROM (VALUES (1,'#a'),(2,'#b'),(3,'#unused')) v(i,name); +INSERT INTO observers (id,public_key,last_iata) +SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),2,'0'),'hex'),iata +FROM (VALUES (1,'YVR'),(2,'YYJ'),(3,'YVR'),(4,'YVR'),(5,'YYZ')) v(i,iata); +-- every packet is an advert from its own node, so per-scope nodes equal per-scope packets +INSERT INTO packets (packet_hash,scope_id,payload_type,payload_version,route_type,origin_pubkey,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),scope_id,4,0,1,decode(lpad(to_hex(100+i),64,'0'),'hex'),'\x00','\x00',$1::timestamptz,$1::timestamptz +FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,2),(8,NULL)) v(i,scope_id); +INSERT INTO observer_scopes (observer_id,scope_id) +SELECT md5(i::text)::uuid,scope_id FROM (VALUES (1,1),(2,1),(4,1),(5,1),(1,2),(2,2),(3,2)) v(i,scope_id); +-- packet 4 is never heard, so it never reaches the rollup +INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type) +SELECT id,decode(lpad(to_hex(packet),2,'0'),'hex'),md5(observer::text)::uuid,iata, + $1::timestamptz+id*interval '1 second',0,1,0,4 +FROM (VALUES (1,1,1,'YVR'),(2,1,4,'YVR'),(3,1,2,'YYJ'),(4,2,2,'YVR'), + (5,3,1,'YYZ'),(6,5,3,'YVR'),(7,6,1,'YYJ'),(8,7,2,'YYZ'),(9,8,1,'YVR')) v(id,packet,observer,iata); +INSERT INTO nodes (id,public_key,node_type,default_scope_id) +SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),2,scope_id +FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,NULL)) v(i,scope_id); +INSERT INTO node_iatas (node_id,iata) +SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YYZ'),(5,'YYJ'),(6,'YVR'),(6,'YYJ'),(7,'YVR')) v(i,iata); +`, pgx.QueryExecModeSimpleProtocol, h); err != nil { + t.Fatal(err) + } + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + if err := r.RegisterHours(ctx, h); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + r.Close(ctx) + + for _, tc := range []struct { + name string + iatas []string + since time.Time + a, b [3]int64 // packets, observers, nodes + ah, bh int64 // distinct observers heard carrying the scope in hour h + }{ + {"global", nil, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}, 3, 3}, + {"empty filter", []string{}, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}, 3, 3}, + {"YVR", []string{"YVR"}, time.Time{}, [3]int64{2, 2, 2}, [3]int64{1, 2, 1}, 3, 1}, + {"YYJ", []string{"YYJ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 1, 2}, 1, 1}, + {"YYZ", []string{"YYZ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 0, 0}, 1, 1}, + {"overlapping IATAs", []string{"YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}, 3, 2}, + {"duplicate IATAs", []string{"YYJ", "YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}, 3, 2}, + {"unknown IATA", []string{"ZZZ"}, time.Time{}, [3]int64{}, [3]int64{}, 0, 0}, + {"window after the traffic", nil, h.Add(time.Hour), [3]int64{0, 4, 4}, [3]int64{0, 3, 2}, 0, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + rows, err := store.GetScopeStats(ctx, tc.iatas, tc.since) + if err != nil { + t.Fatal(err) + } + // All fixture traffic is in hour h; hours with no packets are omitted. + // Each packet has its own origin, so active nodes per hour equal packets per hour. + hourly := func(packets, observers int64) []api.ScopeHour { + if packets == 0 && observers == 0 { + return []api.ScopeHour{} + } + return []api.ScopeHour{{Hour: h.UnixMilli(), Packets: packets, Observers: observers, Nodes: packets}} + } + want := []api.ScopeStats{ + {Name: "#a", PacketCount: tc.a[0], ObserverCount: tc.a[1], NodeCount: tc.a[2], Hourly: hourly(tc.a[0], tc.ah)}, + {Name: "#b", PacketCount: tc.b[0], ObserverCount: tc.b[1], NodeCount: tc.b[2], Hourly: hourly(tc.b[0], tc.bh)}, + {Name: "#unused", Hourly: []api.ScopeHour{}}, + } + if !reflect.DeepEqual(rows, want) { + t.Fatalf("counts = %+v, want %+v", rows, want) + } + }) + } + + detail, err := store.GetScopeByName(ctx, "#a") + if err != nil { + t.Fatal(err) + } + want := &api.ScopeDetail{Name: "#a", PacketCount: 3, ObserverCount: 4, NodeCount: 4, IATACount: 3, IATAs: []string{"YVR", "YYJ", "YYZ"}} + if !reflect.DeepEqual(detail, want) { + t.Errorf("scope detail %+v, want %+v", detail, want) + } + + if _, err := pool.Exec(ctx, "TRUNCATE transport_scopes CASCADE"); err != nil { + t.Fatal(err) + } + rows, err := store.GetScopeStats(ctx, nil, time.Time{}) + if err != nil || rows == nil || len(rows) != 0 { + t.Fatalf("empty roster = %+v, %v", rows, err) + } +} diff --git a/db/scopes.go b/db/scopes.go index af5cf9a2..a9b1cb20 100644 --- a/db/scopes.go +++ b/db/scopes.go @@ -49,24 +49,6 @@ func (s *Store) GetScopeNames(ctx context.Context) ([]string, error) { return s.q.GetScopeNames(ctx) } -// GetScopesByIATAs returns scope summaries filtered by the given IATA codes. -func (s *Store) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) { - rows, err := s.q.GetScopesByIATAs(ctx, iatas) - if err != nil { - return nil, err - } - items := make([]api.ScopeSummary, 0, len(rows)) - for _, r := range rows { - items = append(items, api.ScopeSummary{ - Name: r.Name, - ObserverCount: r.ObserverCount, - NodeCount: r.NodeCount, - IATACount: r.IataCount, - }) - } - return items, nil -} - // GetScopeByName returns full detail for a single scope by its normalized name. func (s *Store) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { row, err := s.q.GetScopeByName(ctx, name) diff --git a/db/scopes_test.go b/db/scopes_test.go index 890e93ad..c581deea 100644 --- a/db/scopes_test.go +++ b/db/scopes_test.go @@ -43,29 +43,6 @@ func TestGetTransportScopes(t *testing.T) { } } -func TestGetScopesByIATAs(t *testing.T) { - ctrl := gomock.NewController(t) - mock := mockdb.NewMockQuerier(ctrl) - - mock.EXPECT(). - GetScopesByIATAs(gomock.Any(), []string{"YVR", "YYJ"}). - Return([]sqlc.GetScopesByIATAsRow{ - {Name: "default", ObserverCount: 3, NodeCount: 10, IataCount: 2}, - }, nil) - - store := &Store{q: mock} - items, err := store.GetScopesByIATAs(context.Background(), []string{"YVR", "YYJ"}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(items) != 1 { - t.Fatalf("expected 1 item, got %d", len(items)) - } - if items[0].IATACount != 2 { - t.Errorf("expected IATACount 2, got %d", items[0].IATACount) - } -} - func TestGetScopeByName(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/db/signal.go b/db/signal.go new file mode 100644 index 00000000..0edd4cf1 --- /dev/null +++ b/db/signal.go @@ -0,0 +1,68 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + rows, err := s.q.GetSignalStats(ctx, sqlc.GetSignalStatsParams{ + Since: pgtype.Timestamptz{Time: since, Valid: true}, Until: pgtype.Timestamptz{Time: until, Valid: true}, Iatas: iatas, + }) + if err != nil { + return nil, err + } + stats := &api.SignalStats{Since: since.UnixMilli(), Until: until.UnixMilli(), + SNR: api.SignalMetric{Histogram: signalBins(-30, 5, 12)}, RSSI: api.SignalMetric{Histogram: signalBins(-140, 10, 14)}, Hourly: []api.SignalHour{}} + for _, row := range rows { + // GROUPING bits distinguish real null buckets from dimensions not in the set. + switch row.Kind { + case 7: + stats.Receptions = row.Receptions + stats.SNR.Samples, stats.SNR.Average = row.SnrSamples, signalAverage(row.SnrAverage, row.SnrSamples) + stats.RSSI.Samples, stats.RSSI.Average = row.RssiSamples, signalAverage(row.RssiAverage, row.RssiSamples) + case 3: + stats.Hourly = append(stats.Hourly, api.SignalHour{Hour: row.Hour.Time.UnixMilli(), Receptions: row.Receptions, + SNRSamples: row.SnrSamples, SNRAverage: signalAverage(row.SnrAverage, row.SnrSamples), + RSSISamples: row.RssiSamples, RSSIAverage: signalAverage(row.RssiAverage, row.RssiSamples)}) + case 5: + if row.SnrBin >= 0 && int(row.SnrBin) < len(stats.SNR.Histogram) { + stats.SNR.Histogram[row.SnrBin].Count = row.Receptions + } + case 6: + if row.RssiBin >= 0 && int(row.RssiBin) < len(stats.RSSI.Histogram) { + stats.RSSI.Histogram[row.RssiBin].Count = row.Receptions + } + } + } + return stats, nil +} + +func signalAverage(value float64, count int64) *float64 { + if count == 0 { + return nil + } + return &value +} + +func signalBins(lower, width float64, count int) []api.SignalBin { + bins := make([]api.SignalBin, count+2) + for i := range bins { + if i > 0 { + v := lower + float64(i-1)*width + bins[i].Lower = &v + } + if i <= count { + v := lower + float64(i)*width + bins[i].Upper = &v + } + } + return bins +} diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go new file mode 100644 index 00000000..06555032 --- /dev/null +++ b/db/signal_integration_test.go @@ -0,0 +1,156 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http/httptest" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/jackc/pgx/v5" +) + +// Real view DDL and fixture rows are isolated in a rolled-back private schema. +func TestSignalPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + _, err = tx.Exec(ctx, ` +SET LOCAL TIME ZONE 'America/Vancouver'; +CREATE TEMP TABLE fixture (heard_at timestamptz NOT NULL, iata char(3) NOT NULL, snr real, rssi smallint); +INSERT INTO fixture +SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FROM (VALUES + ('00:30:00','YVR','0',-100), ('00:45:00','YVR','0',0), ('00:59:00','YVR',NULL,NULL), + ('01:00:00','YVR','-30',-140), ('01:10:00','YVR','-30.25',-141), ('01:20:00','YVR','30',0), + ('01:30:00','YVR','5',NULL), ('01:40:00','YVR',NULL,-90), ('01:50:00','YVR','NaN',-80), + ('02:00:00','YVR','Infinity',-70), ('02:10:00','YVR','-Infinity',-60), + ('02:20:00','YYJ','10',-50), ('03:00:00','YYZ','-5',-110), + ('04:00:00','YVR','20',-40), ('04:30:00','YVR','20',-40), + ('03:10:00','YVR','0',NULL), ('03:20:00','YVR',NULL,0) +) v(at,iata,snr,rssi); +ALTER TABLE fixture ADD COLUMN n serial; +INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT int4send(n),4,0,1,'\x00','\x00',heard_at,heard_at FROM fixture; +INSERT INTO packet_observations (packet_hash,observer_id,heard_at,iata,snr,rssi,path_length_byte,hash_size,hop_count) +SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi,0,1,0 FROM fixture;`) + if err != nil { + t.Fatal(err) + } + since := time.Now().UTC().Truncate(24 * time.Hour).Add(-48 * time.Hour) + if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { + t.Fatal(err) + } + rollTxHours(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + until := since.Add(4 * time.Hour) + for _, tc := range []struct { + name string + iatas []string + receptions, snr, rssi int64 + }{ + {"global", nil, 15, 7, 10}, {"empty filter", []string{}, 15, 7, 10}, {"YVR", []string{"YVR"}, 13, 5, 8}, + {"duplicate/overlapping IATAs", []string{"YVR", "YYJ", "YVR"}, 14, 6, 9}, + {"unknown IATA", []string{"ZZZ"}, 0, 0, 0}, {"empty region", []string{""}, 0, 0, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := store.GetSignalStats(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + if got.Receptions != tc.receptions || got.SNR.Samples != tc.snr || got.RSSI.Samples != tc.rssi { + t.Fatalf("counts: %+v", got) + } + for _, metric := range []api.SignalMetric{got.SNR, got.RSSI} { + var total int64 + for _, bin := range metric.Histogram { + total += bin.Count + } + if total != metric.Samples || (metric.Average == nil) != (metric.Samples == 0) { + t.Fatalf("histogram/average: %+v", metric) + } + } + var receptions, snr, rssi int64 + for i, hour := range got.Hourly { + if hour.Hour%3600000 != 0 || hour.Hour < since.Truncate(time.Hour).UnixMilli() || hour.Hour >= until.UnixMilli() || (i > 0 && hour.Hour <= got.Hourly[i-1].Hour) { + t.Fatalf("UTC order/window: %+v", got.Hourly) + } + receptions += hour.Receptions + snr += hour.SNRSamples + rssi += hour.RSSISamples + } + if receptions != got.Receptions || snr != got.SNR.Samples || rssi != got.RSSI.Samples { + t.Fatal("hourly totals differ") + } + if _, err := json.Marshal(got); err != nil { + t.Fatal(err) + } + if tc.name == "YVR" { + if math.Abs(*got.SNR.Average-(-5.05)) > 1e-9 || *got.RSSI.Average != -85.125 { + t.Fatalf("unweighted/missing readings: %+v", got) + } + if got.SNR.Histogram[0].Count != 1 || got.SNR.Histogram[1].Count != 1 || got.SNR.Histogram[7].Count != 1 || got.SNR.Histogram[8].Count != 1 || got.SNR.Histogram[13].Count != 1 { + t.Fatalf("SNR boundaries/zero: %+v", got.SNR.Histogram) + } + if got.RSSI.Histogram[0].Count != 1 || got.RSSI.Histogram[1].Count != 1 || got.RSSI.Histogram[15].Count != 1 { + t.Fatalf("RSSI boundaries: %+v", got.RSSI.Histogram) + } + last := got.Hourly[len(got.Hourly)-1] + if last.Receptions != 2 || last.SNRAverage != nil || last.RSSIAverage != nil { + t.Fatalf("missing-only hour: %+v", last) + } + } + }) + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err = tx.Exec(ctx, "SET LOCAL plan_cache_mode="+mode); err != nil { + t.Fatal(err) + } + got, err := store.GetSignalStats(ctx, since, until, []string{"YVR"}) + if err != nil || got.Receptions != 13 || got.SNR.Samples != 5 || got.RSSI.Samples != 8 { + t.Fatalf("%s: %+v %v", mode, got, err) + } + } + w := httptest.NewRecorder() + request := httptest.NewRequest("GET", fmt.Sprintf("/signal?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx) + handlers.StatsRouter(store, handlers.StatsOptions{}).ServeHTTP(w, request) + var response api.SignalStats + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 13 || response.SNR.Samples != 5 { + t.Fatalf("PostgreSQL HTTP response: status=%d body=%s error=%v", w.Code, w.Body.String(), err) + } + if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { + t.Fatal(err) + } + stale, err := store.GetSignalStats(ctx, since, until, nil) + if err != nil || stale.Receptions != 15 { + t.Fatalf("snapshot lost: %+v %v", stale, err) + } + rollTxHours(t, ctx, tx) + got, err := store.GetSignalStats(ctx, since, until, nil) + if err != nil || got.Receptions != 0 || got.Hourly == nil || len(got.Hourly) != 0 || got.SNR.Average != nil { + t.Fatalf("empty: %+v %v", got, err) + } +} diff --git a/db/signal_test.go b/db/signal_test.go new file mode 100644 index 00000000..35b24f69 --- /dev/null +++ b/db/signal_test.go @@ -0,0 +1,21 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import "testing" + +func TestSignalBinsAndMissingAverage(t *testing.T) { + bins := signalBins(-30, 5, 12) + if len(bins) != 14 || bins[0].Lower != nil || *bins[0].Upper != -30 || *bins[13].Lower != 30 || bins[13].Upper != nil { + t.Fatalf("bounds: %+v", bins) + } + for i := 1; i < 13; i++ { + if *bins[i].Upper-*bins[i].Lower != 5 || *bins[i].Lower != *bins[i-1].Upper { + t.Fatal("non-contiguous buckets") + } + } + if signalAverage(0, 0) != nil || *signalAverage(0, 1) != 0 { + t.Fatal("zero must be distinct from no samples") + } +} diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 91f7e0fb..b34664f2 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -source=db/sqlc/querier.go -destination=db/sqlc/mock/querier.go -package=mockdb +// mockgen -source db/sqlc/querier.go -destination db/sqlc/mock/querier.go -package mockdb // // Package mockdb is a generated GoMock package. @@ -11,6 +11,7 @@ package mockdb import ( context "context" + json "encoding/json" reflect "reflect" db "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -43,6 +44,149 @@ func (m *MockQuerier) EXPECT() *MockQuerierMockRecorder { return m.recorder } +// AddChannelConfigScopes mocks base method. +func (m *MockQuerier) AddChannelConfigScopes(ctx context.Context, arg db.AddChannelConfigScopesParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddChannelConfigScopes", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddChannelConfigScopes indicates an expected call of AddChannelConfigScopes. +func (mr *MockQuerierMockRecorder) AddChannelConfigScopes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChannelConfigScopes", reflect.TypeOf((*MockQuerier)(nil).AddChannelConfigScopes), ctx, arg) +} + +// AddChannelMembers mocks base method. +func (m *MockQuerier) AddChannelMembers(ctx context.Context, arg db.AddChannelMembersParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddChannelMembers", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddChannelMembers indicates an expected call of AddChannelMembers. +func (mr *MockQuerierMockRecorder) AddChannelMembers(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChannelMembers", reflect.TypeOf((*MockQuerier)(nil).AddChannelMembers), ctx, arg) +} + +// AddIATAs mocks base method. +func (m *MockQuerier) AddIATAs(ctx context.Context, iatas []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddIATAs", ctx, iatas) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddIATAs indicates an expected call of AddIATAs. +func (mr *MockQuerierMockRecorder) AddIATAs(ctx, iatas any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddIATAs", reflect.TypeOf((*MockQuerier)(nil).AddIATAs), ctx, iatas) +} + +// AddRegionIATAs mocks base method. +func (m *MockQuerier) AddRegionIATAs(ctx context.Context, arg db.AddRegionIATAsParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddRegionIATAs", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddRegionIATAs indicates an expected call of AddRegionIATAs. +func (mr *MockQuerierMockRecorder) AddRegionIATAs(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRegionIATAs", reflect.TypeOf((*MockQuerier)(nil).AddRegionIATAs), ctx, arg) +} + +// AmbiguousPrefixes mocks base method. +func (m *MockQuerier) AmbiguousPrefixes(ctx context.Context) ([]db.AmbiguousPrefixesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AmbiguousPrefixes", ctx) + ret0, _ := ret[0].([]db.AmbiguousPrefixesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AmbiguousPrefixes indicates an expected call of AmbiguousPrefixes. +func (mr *MockQuerierMockRecorder) AmbiguousPrefixes(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AmbiguousPrefixes", reflect.TypeOf((*MockQuerier)(nil).AmbiguousPrefixes), ctx) +} + +// CreateAccount mocks base method. +func (m *MockQuerier) CreateAccount(ctx context.Context, name string) (db.Account, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateAccount", ctx, name) + ret0, _ := ret[0].(db.Account) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateAccount indicates an expected call of CreateAccount. +func (mr *MockQuerierMockRecorder) CreateAccount(ctx, name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateAccount", reflect.TypeOf((*MockQuerier)(nil).CreateAccount), ctx, name) +} + +// DeactivateAccount mocks base method. +func (m *MockQuerier) DeactivateAccount(ctx context.Context, id uuid.UUID) (db.DeactivateAccountRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeactivateAccount", ctx, id) + ret0, _ := ret[0].(db.DeactivateAccountRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeactivateAccount indicates an expected call of DeactivateAccount. +func (mr *MockQuerierMockRecorder) DeactivateAccount(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeactivateAccount", reflect.TypeOf((*MockQuerier)(nil).DeactivateAccount), ctx, id) +} + +// DeleteAllChannelMembers mocks base method. +func (m *MockQuerier) DeleteAllChannelMembers(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAllChannelMembers", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAllChannelMembers indicates an expected call of DeleteAllChannelMembers. +func (mr *MockQuerierMockRecorder) DeleteAllChannelMembers(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllChannelMembers", reflect.TypeOf((*MockQuerier)(nil).DeleteAllChannelMembers), ctx) +} + +// DeleteChannelConfigScopes mocks base method. +func (m *MockQuerier) DeleteChannelConfigScopes(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteChannelConfigScopes", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteChannelConfigScopes indicates an expected call of DeleteChannelConfigScopes. +func (mr *MockQuerierMockRecorder) DeleteChannelConfigScopes(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChannelConfigScopes", reflect.TypeOf((*MockQuerier)(nil).DeleteChannelConfigScopes), ctx) +} + +// DeleteChannelMembersNotIn mocks base method. +func (m *MockQuerier) DeleteChannelMembersNotIn(ctx context.Context, arg db.DeleteChannelMembersNotInParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteChannelMembersNotIn", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteChannelMembersNotIn indicates an expected call of DeleteChannelMembersNotIn. +func (mr *MockQuerierMockRecorder) DeleteChannelMembersNotIn(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChannelMembersNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteChannelMembersNotIn), ctx, arg) +} + // DeleteOldChannelIATAs mocks base method. func (m *MockQuerier) DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error { m.ctrl.T.Helper() @@ -71,18 +215,63 @@ func (mr *MockQuerierMockRecorder) DeleteOldNodes(ctx, lastSeen any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldNodes", reflect.TypeOf((*MockQuerier)(nil).DeleteOldNodes), ctx, lastSeen) } +// DeleteOldObservers mocks base method. +func (m *MockQuerier) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldObservers", ctx, lastSeen) + ret0, _ := ret[0].([]uuid.UUID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteOldObservers indicates an expected call of DeleteOldObservers. +func (mr *MockQuerierMockRecorder) DeleteOldObservers(ctx, lastSeen any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldObservers", reflect.TypeOf((*MockQuerier)(nil).DeleteOldObservers), ctx, lastSeen) +} + // DeleteOldPackets mocks base method. -func (m *MockQuerier) DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { +func (m *MockQuerier) DeleteOldPackets(ctx context.Context, arg db.DeleteOldPacketsParams) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldPackets", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteOldPackets indicates an expected call of DeleteOldPackets. +func (mr *MockQuerierMockRecorder) DeleteOldPackets(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldPackets", reflect.TypeOf((*MockQuerier)(nil).DeleteOldPackets), ctx, arg) +} + +// DeleteOldRollups mocks base method. +func (m *MockQuerier) DeleteOldRollups(ctx context.Context, cutoff pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteOldPackets", ctx, lastHeardAt) + ret := m.ctrl.Call(m, "DeleteOldRollups", ctx, cutoff) ret0, _ := ret[0].(error) return ret0 } -// DeleteOldPackets indicates an expected call of DeleteOldPackets. -func (mr *MockQuerierMockRecorder) DeleteOldPackets(ctx, lastHeardAt any) *gomock.Call { +// DeleteOldRollups indicates an expected call of DeleteOldRollups. +func (mr *MockQuerierMockRecorder) DeleteOldRollups(ctx, cutoff any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldRollups", reflect.TypeOf((*MockQuerier)(nil).DeleteOldRollups), ctx, cutoff) +} + +// DeleteOldRoutes mocks base method. +func (m *MockQuerier) DeleteOldRoutes(ctx context.Context, arg db.DeleteOldRoutesParams) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldRoutes", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteOldRoutes indicates an expected call of DeleteOldRoutes. +func (mr *MockQuerierMockRecorder) DeleteOldRoutes(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldPackets", reflect.TypeOf((*MockQuerier)(nil).DeleteOldPackets), ctx, lastHeardAt) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldRoutes", reflect.TypeOf((*MockQuerier)(nil).DeleteOldRoutes), ctx, arg) } // DeleteOldTelemetry mocks base method. @@ -113,6 +302,121 @@ func (mr *MockQuerierMockRecorder) DeleteOldTraceIATAs(ctx, lastHeard any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceIATAs", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceIATAs), ctx, lastHeard) } +// DeleteOldTraceTags mocks base method. +func (m *MockQuerier) DeleteOldTraceTags(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldTraceTags", ctx, lastHeardAt) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteOldTraceTags indicates an expected call of DeleteOldTraceTags. +func (mr *MockQuerierMockRecorder) DeleteOldTraceTags(ctx, lastHeardAt any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceTags", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceTags), ctx, lastHeardAt) +} + +// DeleteRegionIATAsNotIn mocks base method. +func (m *MockQuerier) DeleteRegionIATAsNotIn(ctx context.Context, arg db.DeleteRegionIATAsNotInParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRegionIATAsNotIn", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteRegionIATAsNotIn indicates an expected call of DeleteRegionIATAsNotIn. +func (mr *MockQuerierMockRecorder) DeleteRegionIATAsNotIn(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRegionIATAsNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteRegionIATAsNotIn), ctx, arg) +} + +// DeleteRollupHour mocks base method. +func (m *MockQuerier) DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRollupHour", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteRollupHour indicates an expected call of DeleteRollupHour. +func (mr *MockQuerierMockRecorder) DeleteRollupHour(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRollupHour", reflect.TypeOf((*MockQuerier)(nil).DeleteRollupHour), ctx, hour) +} + +// DeleteStaleDirtyHours mocks base method. +func (m *MockQuerier) DeleteStaleDirtyHours(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteStaleDirtyHours", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteStaleDirtyHours indicates an expected call of DeleteStaleDirtyHours. +func (mr *MockQuerierMockRecorder) DeleteStaleDirtyHours(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStaleDirtyHours", reflect.TypeOf((*MockQuerier)(nil).DeleteStaleDirtyHours), ctx) +} + +// FillRollupObs mocks base method. +func (m *MockQuerier) FillRollupObs(ctx context.Context, dollar_1 pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FillRollupObs", ctx, dollar_1) + ret0, _ := ret[0].(error) + return ret0 +} + +// FillRollupObs indicates an expected call of FillRollupObs. +func (mr *MockQuerierMockRecorder) FillRollupObs(ctx, dollar_1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FillRollupObs", reflect.TypeOf((*MockQuerier)(nil).FillRollupObs), ctx, dollar_1) +} + +// FinishRollupHour mocks base method. +func (m *MockQuerier) FinishRollupHour(ctx context.Context, arg db.FinishRollupHourParams) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinishRollupHour", ctx, arg) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FinishRollupHour indicates an expected call of FinishRollupHour. +func (mr *MockQuerierMockRecorder) FinishRollupHour(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinishRollupHour", reflect.TypeOf((*MockQuerier)(nil).FinishRollupHour), ctx, arg) +} + +// GetAccount mocks base method. +func (m *MockQuerier) GetAccount(ctx context.Context, id uuid.UUID) (db.Account, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAccount", ctx, id) + ret0, _ := ret[0].(db.Account) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAccount indicates an expected call of GetAccount. +func (mr *MockQuerierMockRecorder) GetAccount(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccount", reflect.TypeOf((*MockQuerier)(nil).GetAccount), ctx, id) +} + +// GetAnalyticsRevision mocks base method. +func (m *MockQuerier) GetAnalyticsRevision(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAnalyticsRevision", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAnalyticsRevision indicates an expected call of GetAnalyticsRevision. +func (mr *MockQuerierMockRecorder) GetAnalyticsRevision(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAnalyticsRevision", reflect.TypeOf((*MockQuerier)(nil).GetAnalyticsRevision), ctx) +} + // GetChannelByID mocks base method. func (m *MockQuerier) GetChannelByID(ctx context.Context, id int32) (db.Channel, error) { m.ctrl.T.Helper() @@ -143,11 +447,26 @@ func (mr *MockQuerierMockRecorder) GetCrossIATANeighbors(ctx, arg any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCrossIATANeighbors", reflect.TypeOf((*MockQuerier)(nil).GetCrossIATANeighbors), ctx, arg) } +// GetEarliestCompleteRollupHour mocks base method. +func (m *MockQuerier) GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetEarliestCompleteRollupHour", ctx) + ret0, _ := ret[0].(pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetEarliestCompleteRollupHour indicates an expected call of GetEarliestCompleteRollupHour. +func (mr *MockQuerierMockRecorder) GetEarliestCompleteRollupHour(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEarliestCompleteRollupHour", reflect.TypeOf((*MockQuerier)(nil).GetEarliestCompleteRollupHour), ctx) +} + // GetHourlyStats mocks base method. -func (m *MockQuerier) GetHourlyStats(ctx context.Context, arg db.GetHourlyStatsParams) ([]db.MvHourlyIataStat, error) { +func (m *MockQuerier) GetHourlyStats(ctx context.Context, arg db.GetHourlyStatsParams) ([]db.GetHourlyStatsRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetHourlyStats", ctx, arg) - ret0, _ := ret[0].([]db.MvHourlyIataStat) + ret0, _ := ret[0].([]db.GetHourlyStatsRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -174,10 +493,10 @@ func (mr *MockQuerierMockRecorder) GetIATA(ctx, iata any) *gomock.Call { } // GetIATABorder mocks base method. -func (m *MockQuerier) GetIATABorder(ctx context.Context, iata string) ([]byte, error) { +func (m *MockQuerier) GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetIATABorder", ctx, iata) - ret0, _ := ret[0].([]byte) + ret0, _ := ret[0].(json.RawMessage) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -189,10 +508,10 @@ func (mr *MockQuerierMockRecorder) GetIATABorder(ctx, iata any) *gomock.Call { } // GetKnownRoutesByNode mocks base method. -func (m *MockQuerier) GetKnownRoutesByNode(ctx context.Context, arg db.GetKnownRoutesByNodeParams) ([]db.KnownRoute, error) { +func (m *MockQuerier) GetKnownRoutesByNode(ctx context.Context, arg db.GetKnownRoutesByNodeParams) ([]db.GetKnownRoutesByNodeRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetKnownRoutesByNode", ctx, arg) - ret0, _ := ret[0].([]db.KnownRoute) + ret0, _ := ret[0].([]db.GetKnownRoutesByNodeRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -203,6 +522,21 @@ func (mr *MockQuerierMockRecorder) GetKnownRoutesByNode(ctx, arg any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetKnownRoutesByNode", reflect.TypeOf((*MockQuerier)(nil).GetKnownRoutesByNode), ctx, arg) } +// GetLatestCompleteRollupHour mocks base method. +func (m *MockQuerier) GetLatestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLatestCompleteRollupHour", ctx) + ret0, _ := ret[0].(pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLatestCompleteRollupHour indicates an expected call of GetLatestCompleteRollupHour. +func (mr *MockQuerierMockRecorder) GetLatestCompleteRollupHour(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLatestCompleteRollupHour", reflect.TypeOf((*MockQuerier)(nil).GetLatestCompleteRollupHour), ctx) +} + // GetNodeByID mocks base method. func (m *MockQuerier) GetNodeByID(ctx context.Context, id uuid.UUID) (db.GetNodeByIDRow, error) { m.ctrl.T.Helper() @@ -263,6 +597,96 @@ func (mr *MockQuerierMockRecorder) GetNodesByIDs(ctx, dollar_1 any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodesByIDs", reflect.TypeOf((*MockQuerier)(nil).GetNodesByIDs), ctx, dollar_1) } +// GetNodesByPubkeys mocks base method. +func (m *MockQuerier) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]db.GetNodesByPubkeysRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNodesByPubkeys", ctx, pubkeys) + ret0, _ := ret[0].([]db.GetNodesByPubkeysRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNodesByPubkeys indicates an expected call of GetNodesByPubkeys. +func (mr *MockQuerierMockRecorder) GetNodesByPubkeys(ctx, pubkeys any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodesByPubkeys", reflect.TypeOf((*MockQuerier)(nil).GetNodesByPubkeys), ctx, pubkeys) +} + +// GetObserverActivityHourly mocks base method. +func (m *MockQuerier) GetObserverActivityHourly(ctx context.Context, arg db.GetObserverActivityHourlyParams) ([]db.GetObserverActivityHourlyRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityHourly", ctx, arg) + ret0, _ := ret[0].([]db.GetObserverActivityHourlyRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityHourly indicates an expected call of GetObserverActivityHourly. +func (mr *MockQuerierMockRecorder) GetObserverActivityHourly(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityHourly", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityHourly), ctx, arg) +} + +// GetObserverActivityHourlyPayloadTypes mocks base method. +func (m *MockQuerier) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg db.GetObserverActivityHourlyPayloadTypesParams) ([]db.GetObserverActivityHourlyPayloadTypesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityHourlyPayloadTypes", ctx, arg) + ret0, _ := ret[0].([]db.GetObserverActivityHourlyPayloadTypesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityHourlyPayloadTypes indicates an expected call of GetObserverActivityHourlyPayloadTypes. +func (mr *MockQuerierMockRecorder) GetObserverActivityHourlyPayloadTypes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityHourlyPayloadTypes", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityHourlyPayloadTypes), ctx, arg) +} + +// GetObserverActivityLiveSummary mocks base method. +func (m *MockQuerier) GetObserverActivityLiveSummary(ctx context.Context, arg db.GetObserverActivityLiveSummaryParams) (db.GetObserverActivityLiveSummaryRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityLiveSummary", ctx, arg) + ret0, _ := ret[0].(db.GetObserverActivityLiveSummaryRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityLiveSummary indicates an expected call of GetObserverActivityLiveSummary. +func (mr *MockQuerierMockRecorder) GetObserverActivityLiveSummary(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityLiveSummary", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityLiveSummary), ctx, arg) +} + +// GetObserverActivityRaw mocks base method. +func (m *MockQuerier) GetObserverActivityRaw(ctx context.Context, arg db.GetObserverActivityRawParams) ([]db.GetObserverActivityRawRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityRaw", ctx, arg) + ret0, _ := ret[0].([]db.GetObserverActivityRawRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityRaw indicates an expected call of GetObserverActivityRaw. +func (mr *MockQuerierMockRecorder) GetObserverActivityRaw(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityRaw", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityRaw), ctx, arg) +} + +// GetObserverActivityRawPayloadTypes mocks base method. +func (m *MockQuerier) GetObserverActivityRawPayloadTypes(ctx context.Context, arg db.GetObserverActivityRawPayloadTypesParams) ([]db.GetObserverActivityRawPayloadTypesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityRawPayloadTypes", ctx, arg) + ret0, _ := ret[0].([]db.GetObserverActivityRawPayloadTypesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityRawPayloadTypes indicates an expected call of GetObserverActivityRawPayloadTypes. +func (mr *MockQuerierMockRecorder) GetObserverActivityRawPayloadTypes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityRawPayloadTypes", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityRawPayloadTypes), ctx, arg) +} + // GetObserverBrokers mocks base method. func (m *MockQuerier) GetObserverBrokers(ctx context.Context, observerID uuid.UUID) ([]db.GetObserverBrokersRow, error) { m.ctrl.T.Helper() @@ -308,19 +732,34 @@ func (mr *MockQuerierMockRecorder) GetObserverByPubkey(ctx, publicKey any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverByPubkey", reflect.TypeOf((*MockQuerier)(nil).GetObserverByPubkey), ctx, publicKey) } +// GetObserverComparison mocks base method. +func (m *MockQuerier) GetObserverComparison(ctx context.Context, arg db.GetObserverComparisonParams) (db.GetObserverComparisonRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverComparison", ctx, arg) + ret0, _ := ret[0].(db.GetObserverComparisonRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverComparison indicates an expected call of GetObserverComparison. +func (mr *MockQuerierMockRecorder) GetObserverComparison(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverComparison", reflect.TypeOf((*MockQuerier)(nil).GetObserverComparison), ctx, arg) +} + // GetObserverLastIATA mocks base method. -func (m *MockQuerier) GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) { +func (m *MockQuerier) GetObserverLastIATA(ctx context.Context, id uuid.UUID) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetObserverLastIATA", ctx, observerID) + ret := m.ctrl.Call(m, "GetObserverLastIATA", ctx, id) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetObserverLastIATA indicates an expected call of GetObserverLastIATA. -func (mr *MockQuerierMockRecorder) GetObserverLastIATA(ctx, observerID any) *gomock.Call { +func (mr *MockQuerierMockRecorder) GetObserverLastIATA(ctx, id any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverLastIATA", reflect.TypeOf((*MockQuerier)(nil).GetObserverLastIATA), ctx, observerID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverLastIATA", reflect.TypeOf((*MockQuerier)(nil).GetObserverLastIATA), ctx, id) } // GetObserverRadio mocks base method. @@ -368,21 +807,6 @@ func (mr *MockQuerierMockRecorder) GetObserverTelemetry(ctx, arg any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverTelemetry", reflect.TypeOf((*MockQuerier)(nil).GetObserverTelemetry), ctx, arg) } -// GetObserverTelemetryBucketed mocks base method. -func (m *MockQuerier) GetObserverTelemetryBucketed(ctx context.Context, arg db.GetObserverTelemetryBucketedParams) ([]db.GetObserverTelemetryBucketedRow, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetObserverTelemetryBucketed", ctx, arg) - ret0, _ := ret[0].([]db.GetObserverTelemetryBucketedRow) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetObserverTelemetryBucketed indicates an expected call of GetObserverTelemetryBucketed. -func (mr *MockQuerierMockRecorder) GetObserverTelemetryBucketed(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverTelemetryBucketed", reflect.TypeOf((*MockQuerier)(nil).GetObserverTelemetryBucketed), ctx, arg) -} - // GetPacketByHash mocks base method. func (m *MockQuerier) GetPacketByHash(ctx context.Context, packetHash []byte) (db.GetPacketByHashRow, error) { m.ctrl.T.Helper() @@ -398,34 +822,34 @@ func (mr *MockQuerierMockRecorder) GetPacketByHash(ctx, packetHash any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPacketByHash", reflect.TypeOf((*MockQuerier)(nil).GetPacketByHash), ctx, packetHash) } -// GetPacketObservationCount mocks base method. -func (m *MockQuerier) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) { +// GetPacketsByTraceTag mocks base method. +func (m *MockQuerier) GetPacketsByTraceTag(ctx context.Context, decode string) ([]db.GetPacketsByTraceTagRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPacketObservationCount", ctx, packetHash) - ret0, _ := ret[0].(int64) + ret := m.ctrl.Call(m, "GetPacketsByTraceTag", ctx, decode) + ret0, _ := ret[0].([]db.GetPacketsByTraceTagRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetPacketObservationCount indicates an expected call of GetPacketObservationCount. -func (mr *MockQuerierMockRecorder) GetPacketObservationCount(ctx, packetHash any) *gomock.Call { +// GetPacketsByTraceTag indicates an expected call of GetPacketsByTraceTag. +func (mr *MockQuerierMockRecorder) GetPacketsByTraceTag(ctx, decode any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPacketObservationCount", reflect.TypeOf((*MockQuerier)(nil).GetPacketObservationCount), ctx, packetHash) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPacketsByTraceTag", reflect.TypeOf((*MockQuerier)(nil).GetPacketsByTraceTag), ctx, decode) } -// GetPacketsByTraceTag mocks base method. -func (m *MockQuerier) GetPacketsByTraceTag(ctx context.Context, decode string) ([]db.GetPacketsByTraceTagRow, error) { +// GetPathStats mocks base method. +func (m *MockQuerier) GetPathStats(ctx context.Context, arg db.GetPathStatsParams) ([]db.GetPathStatsRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPacketsByTraceTag", ctx, decode) - ret0, _ := ret[0].([]db.GetPacketsByTraceTagRow) + ret := m.ctrl.Call(m, "GetPathStats", ctx, arg) + ret0, _ := ret[0].([]db.GetPathStatsRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetPacketsByTraceTag indicates an expected call of GetPacketsByTraceTag. -func (mr *MockQuerierMockRecorder) GetPacketsByTraceTag(ctx, decode any) *gomock.Call { +// GetPathStats indicates an expected call of GetPathStats. +func (mr *MockQuerierMockRecorder) GetPathStats(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPacketsByTraceTag", reflect.TypeOf((*MockQuerier)(nil).GetPacketsByTraceTag), ctx, decode) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPathStats", reflect.TypeOf((*MockQuerier)(nil).GetPathStats), ctx, arg) } // GetRadioPresets mocks base method. @@ -488,6 +912,21 @@ func (mr *MockQuerierMockRecorder) GetRegionIATAs(ctx, regionID any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegionIATAs", reflect.TypeOf((*MockQuerier)(nil).GetRegionIATAs), ctx, regionID) } +// GetRouteEvidenceRoute mocks base method. +func (m *MockQuerier) GetRouteEvidenceRoute(ctx context.Context, arg db.GetRouteEvidenceRouteParams) (db.KnownRoute, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRouteEvidenceRoute", ctx, arg) + ret0, _ := ret[0].(db.KnownRoute) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRouteEvidenceRoute indicates an expected call of GetRouteEvidenceRoute. +func (mr *MockQuerierMockRecorder) GetRouteEvidenceRoute(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRouteEvidenceRoute", reflect.TypeOf((*MockQuerier)(nil).GetRouteEvidenceRoute), ctx, arg) +} + // GetScopeByName mocks base method. func (m *MockQuerier) GetScopeByName(ctx context.Context, name string) (db.GetScopeByNameRow, error) { m.ctrl.T.Helper() @@ -503,6 +942,21 @@ func (mr *MockQuerierMockRecorder) GetScopeByName(ctx, name any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeByName", reflect.TypeOf((*MockQuerier)(nil).GetScopeByName), ctx, name) } +// GetScopeCatalogue mocks base method. +func (m *MockQuerier) GetScopeCatalogue(ctx context.Context, arg db.GetScopeCatalogueParams) (db.MeshmapperScopeCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetScopeCatalogue", ctx, arg) + ret0, _ := ret[0].(db.MeshmapperScopeCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetScopeCatalogue indicates an expected call of GetScopeCatalogue. +func (mr *MockQuerierMockRecorder) GetScopeCatalogue(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).GetScopeCatalogue), ctx, arg) +} + // GetScopeNames mocks base method. func (m *MockQuerier) GetScopeNames(ctx context.Context) ([]string, error) { m.ctrl.T.Helper() @@ -519,63 +973,78 @@ func (mr *MockQuerierMockRecorder) GetScopeNames(ctx any) *gomock.Call { } // GetScopeStats mocks base method. -func (m *MockQuerier) GetScopeStats(ctx context.Context) ([]db.GetScopeStatsRow, error) { +func (m *MockQuerier) GetScopeStats(ctx context.Context, arg db.GetScopeStatsParams) ([]db.GetScopeStatsRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetScopeStats", ctx) + ret := m.ctrl.Call(m, "GetScopeStats", ctx, arg) ret0, _ := ret[0].([]db.GetScopeStatsRow) ret1, _ := ret[1].(error) return ret0, ret1 } // GetScopeStats indicates an expected call of GetScopeStats. -func (mr *MockQuerierMockRecorder) GetScopeStats(ctx any) *gomock.Call { +func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, arg) } -// GetScopesByIATAs mocks base method. -func (m *MockQuerier) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]db.GetScopesByIATAsRow, error) { +// GetScopeStatsHourly mocks base method. +func (m *MockQuerier) GetScopeStatsHourly(ctx context.Context, arg db.GetScopeStatsHourlyParams) ([]db.GetScopeStatsHourlyRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetScopesByIATAs", ctx, dollar_1) - ret0, _ := ret[0].([]db.GetScopesByIATAsRow) + ret := m.ctrl.Call(m, "GetScopeStatsHourly", ctx, arg) + ret0, _ := ret[0].([]db.GetScopeStatsHourlyRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetScopesByIATAs indicates an expected call of GetScopesByIATAs. -func (mr *MockQuerierMockRecorder) GetScopesByIATAs(ctx, dollar_1 any) *gomock.Call { +// GetScopeStatsHourly indicates an expected call of GetScopeStatsHourly. +func (mr *MockQuerierMockRecorder) GetScopeStatsHourly(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopesByIATAs", reflect.TypeOf((*MockQuerier)(nil).GetScopesByIATAs), ctx, dollar_1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStatsHourly", reflect.TypeOf((*MockQuerier)(nil).GetScopeStatsHourly), ctx, arg) } -// GetStatsNodeTypes mocks base method. -func (m *MockQuerier) GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]db.GetStatsNodeTypesRow, error) { +// GetSignalStats mocks base method. +func (m *MockQuerier) GetSignalStats(ctx context.Context, arg db.GetSignalStatsParams) ([]db.GetSignalStatsRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetStatsNodeTypes", ctx, dollar_1) - ret0, _ := ret[0].([]db.GetStatsNodeTypesRow) + ret := m.ctrl.Call(m, "GetSignalStats", ctx, arg) + ret0, _ := ret[0].([]db.GetSignalStatsRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetStatsNodeTypes indicates an expected call of GetStatsNodeTypes. -func (mr *MockQuerierMockRecorder) GetStatsNodeTypes(ctx, dollar_1 any) *gomock.Call { +// GetSignalStats indicates an expected call of GetSignalStats. +func (mr *MockQuerierMockRecorder) GetSignalStats(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsNodeTypes", reflect.TypeOf((*MockQuerier)(nil).GetStatsNodeTypes), ctx, dollar_1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSignalStats", reflect.TypeOf((*MockQuerier)(nil).GetSignalStats), ctx, arg) } -// GetStatsOverview mocks base method. -func (m *MockQuerier) GetStatsOverview(ctx context.Context, dollar_1 []string) (db.GetStatsOverviewRow, error) { +// GetStatsClockDrift mocks base method. +func (m *MockQuerier) GetStatsClockDrift(ctx context.Context, arg db.GetStatsClockDriftParams) ([]db.GetStatsClockDriftRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetStatsOverview", ctx, dollar_1) - ret0, _ := ret[0].(db.GetStatsOverviewRow) + ret := m.ctrl.Call(m, "GetStatsClockDrift", ctx, arg) + ret0, _ := ret[0].([]db.GetStatsClockDriftRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetStatsOverview indicates an expected call of GetStatsOverview. -func (mr *MockQuerierMockRecorder) GetStatsOverview(ctx, dollar_1 any) *gomock.Call { +// GetStatsClockDrift indicates an expected call of GetStatsClockDrift. +func (mr *MockQuerierMockRecorder) GetStatsClockDrift(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsOverview", reflect.TypeOf((*MockQuerier)(nil).GetStatsOverview), ctx, dollar_1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsClockDrift", reflect.TypeOf((*MockQuerier)(nil).GetStatsClockDrift), ctx, arg) +} + +// GetStatsNodeTypes mocks base method. +func (m *MockQuerier) GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]db.GetStatsNodeTypesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStatsNodeTypes", ctx, dollar_1) + ret0, _ := ret[0].([]db.GetStatsNodeTypesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetStatsNodeTypes indicates an expected call of GetStatsNodeTypes. +func (mr *MockQuerierMockRecorder) GetStatsNodeTypes(ctx, dollar_1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsNodeTypes", reflect.TypeOf((*MockQuerier)(nil).GetStatsNodeTypes), ctx, dollar_1) } // GetStatsPayloadBreakdown mocks base method. @@ -593,6 +1062,21 @@ func (mr *MockQuerierMockRecorder) GetStatsPayloadBreakdown(ctx, arg any) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).GetStatsPayloadBreakdown), ctx, arg) } +// GetStatsSeries mocks base method. +func (m *MockQuerier) GetStatsSeries(ctx context.Context, arg db.GetStatsSeriesParams) ([]db.GetStatsSeriesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStatsSeries", ctx, arg) + ret0, _ := ret[0].([]db.GetStatsSeriesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetStatsSeries indicates an expected call of GetStatsSeries. +func (mr *MockQuerierMockRecorder) GetStatsSeries(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsSeries", reflect.TypeOf((*MockQuerier)(nil).GetStatsSeries), ctx, arg) +} + // GetStatsTopAdvertisers mocks base method. func (m *MockQuerier) GetStatsTopAdvertisers(ctx context.Context, arg db.GetStatsTopAdvertisersParams) ([]db.GetStatsTopAdvertisersRow, error) { m.ctrl.T.Helper() @@ -639,10 +1123,10 @@ func (mr *MockQuerierMockRecorder) GetStatsTopTalkers(ctx, arg any) *gomock.Call } // GetTopNodes mocks base method. -func (m *MockQuerier) GetTopNodes(ctx context.Context, arg db.GetTopNodesParams) ([]db.MvTopNodesByIatum, error) { +func (m *MockQuerier) GetTopNodes(ctx context.Context, arg db.GetTopNodesParams) ([]db.GetTopNodesRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetTopNodes", ctx, arg) - ret0, _ := ret[0].([]db.MvTopNodesByIatum) + ret0, _ := ret[0].([]db.GetTopNodesRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -684,10 +1168,10 @@ func (mr *MockQuerierMockRecorder) GetTransportScopes(ctx any) *gomock.Call { } // InsertChannelMessage mocks base method. -func (m *MockQuerier) InsertChannelMessage(ctx context.Context, arg db.InsertChannelMessageParams) (int64, error) { +func (m *MockQuerier) InsertChannelMessage(ctx context.Context, arg db.InsertChannelMessageParams) (db.InsertChannelMessageRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InsertChannelMessage", ctx, arg) - ret0, _ := ret[0].(int64) + ret0, _ := ret[0].(db.InsertChannelMessageRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -699,10 +1183,10 @@ func (mr *MockQuerierMockRecorder) InsertChannelMessage(ctx, arg any) *gomock.Ca } // InsertObservation mocks base method. -func (m *MockQuerier) InsertObservation(ctx context.Context, arg db.InsertObservationParams) (db.PacketObservation, error) { +func (m *MockQuerier) InsertObservation(ctx context.Context, arg db.InsertObservationParams) (db.InsertObservationRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InsertObservation", ctx, arg) - ret0, _ := ret[0].(db.PacketObservation) + ret0, _ := ret[0].(db.InsertObservationRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -727,6 +1211,21 @@ func (mr *MockQuerierMockRecorder) InsertObserverTelemetry(ctx, arg any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertObserverTelemetry", reflect.TypeOf((*MockQuerier)(nil).InsertObserverTelemetry), ctx, arg) } +// ListAccounts mocks base method. +func (m *MockQuerier) ListAccounts(ctx context.Context) ([]db.Account, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListAccounts", ctx) + ret0, _ := ret[0].([]db.Account) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListAccounts indicates an expected call of ListAccounts. +func (mr *MockQuerierMockRecorder) ListAccounts(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAccounts", reflect.TypeOf((*MockQuerier)(nil).ListAccounts), ctx) +} + // ListAllChannelMessages mocks base method. func (m *MockQuerier) ListAllChannelMessages(ctx context.Context, arg db.ListAllChannelMessagesParams) ([]db.ListAllChannelMessagesRow, error) { m.ctrl.T.Helper() @@ -742,6 +1241,21 @@ func (mr *MockQuerierMockRecorder) ListAllChannelMessages(ctx, arg any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAllChannelMessages", reflect.TypeOf((*MockQuerier)(nil).ListAllChannelMessages), ctx, arg) } +// ListChannelCatalogues mocks base method. +func (m *MockQuerier) ListChannelCatalogues(ctx context.Context) ([]db.MeshmapperChannelCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListChannelCatalogues", ctx) + ret0, _ := ret[0].([]db.MeshmapperChannelCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListChannelCatalogues indicates an expected call of ListChannelCatalogues. +func (mr *MockQuerierMockRecorder) ListChannelCatalogues(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannelCatalogues", reflect.TypeOf((*MockQuerier)(nil).ListChannelCatalogues), ctx) +} + // ListChannelMessages mocks base method. func (m *MockQuerier) ListChannelMessages(ctx context.Context, arg db.ListChannelMessagesParams) ([]db.ListChannelMessagesRow, error) { m.ctrl.T.Helper() @@ -787,6 +1301,51 @@ func (mr *MockQuerierMockRecorder) ListChannels(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannels", reflect.TypeOf((*MockQuerier)(nil).ListChannels), ctx, arg) } +// ListChannelsAfter mocks base method. +func (m *MockQuerier) ListChannelsAfter(ctx context.Context, arg db.ListChannelsAfterParams) ([]db.Channel, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListChannelsAfter", ctx, arg) + ret0, _ := ret[0].([]db.Channel) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListChannelsAfter indicates an expected call of ListChannelsAfter. +func (mr *MockQuerierMockRecorder) ListChannelsAfter(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannelsAfter", reflect.TypeOf((*MockQuerier)(nil).ListChannelsAfter), ctx, arg) +} + +// ListDirtyRollupHours mocks base method. +func (m *MockQuerier) ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDirtyRollupHours", ctx, limit) + ret0, _ := ret[0].([]pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDirtyRollupHours indicates an expected call of ListDirtyRollupHours. +func (mr *MockQuerierMockRecorder) ListDirtyRollupHours(ctx, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDirtyRollupHours", reflect.TypeOf((*MockQuerier)(nil).ListDirtyRollupHours), ctx, limit) +} + +// ListHeardIATAs mocks base method. +func (m *MockQuerier) ListHeardIATAs(ctx context.Context) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListHeardIATAs", ctx) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListHeardIATAs indicates an expected call of ListHeardIATAs. +func (mr *MockQuerierMockRecorder) ListHeardIATAs(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListHeardIATAs", reflect.TypeOf((*MockQuerier)(nil).ListHeardIATAs), ctx) +} + // ListIATAs mocks base method. func (m *MockQuerier) ListIATAs(ctx context.Context) ([]db.IataCode, error) { m.ctrl.T.Helper() @@ -803,391 +1362,812 @@ func (mr *MockQuerierMockRecorder) ListIATAs(ctx any) *gomock.Call { } // ListKnownRoutes mocks base method. -func (m *MockQuerier) ListKnownRoutes(ctx context.Context, arg db.ListKnownRoutesParams) ([]db.KnownRoute, error) { +func (m *MockQuerier) ListKnownRoutes(ctx context.Context, arg db.ListKnownRoutesParams) ([]db.ListKnownRoutesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListKnownRoutes", ctx, arg) + ret0, _ := ret[0].([]db.ListKnownRoutesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListKnownRoutes indicates an expected call of ListKnownRoutes. +func (mr *MockQuerierMockRecorder) ListKnownRoutes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListKnownRoutes", reflect.TypeOf((*MockQuerier)(nil).ListKnownRoutes), ctx, arg) +} + +// ListMessagesAfterID mocks base method. +func (m *MockQuerier) ListMessagesAfterID(ctx context.Context, arg db.ListMessagesAfterIDParams) ([]db.ListMessagesAfterIDRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMessagesAfterID", ctx, arg) + ret0, _ := ret[0].([]db.ListMessagesAfterIDRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMessagesAfterID indicates an expected call of ListMessagesAfterID. +func (mr *MockQuerierMockRecorder) ListMessagesAfterID(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMessagesAfterID", reflect.TypeOf((*MockQuerier)(nil).ListMessagesAfterID), ctx, arg) +} + +// ListMissingRollupHours mocks base method. +func (m *MockQuerier) ListMissingRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMissingRollupHours", ctx, limit) + ret0, _ := ret[0].([]pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMissingRollupHours indicates an expected call of ListMissingRollupHours. +func (mr *MockQuerierMockRecorder) ListMissingRollupHours(ctx, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMissingRollupHours", reflect.TypeOf((*MockQuerier)(nil).ListMissingRollupHours), ctx, limit) +} + +// ListNodeObservations mocks base method. +func (m *MockQuerier) ListNodeObservations(ctx context.Context, arg db.ListNodeObservationsParams) ([]db.ListNodeObservationsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListNodeObservations", ctx, arg) + ret0, _ := ret[0].([]db.ListNodeObservationsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListNodeObservations indicates an expected call of ListNodeObservations. +func (mr *MockQuerierMockRecorder) ListNodeObservations(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListNodeObservations", reflect.TypeOf((*MockQuerier)(nil).ListNodeObservations), ctx, arg) +} + +// ListNodes mocks base method. +func (m *MockQuerier) ListNodes(ctx context.Context, arg db.ListNodesParams) ([]db.ListNodesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListNodes", ctx, arg) + ret0, _ := ret[0].([]db.ListNodesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListNodes indicates an expected call of ListNodes. +func (mr *MockQuerierMockRecorder) ListNodes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListNodes", reflect.TypeOf((*MockQuerier)(nil).ListNodes), ctx, arg) +} + +// ListObservationsForPacket mocks base method. +func (m *MockQuerier) ListObservationsForPacket(ctx context.Context, packetHash []byte) ([]db.ListObservationsForPacketRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListObservationsForPacket", ctx, packetHash) + ret0, _ := ret[0].([]db.ListObservationsForPacketRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListObservationsForPacket indicates an expected call of ListObservationsForPacket. +func (mr *MockQuerierMockRecorder) ListObservationsForPacket(ctx, packetHash any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObservationsForPacket", reflect.TypeOf((*MockQuerier)(nil).ListObservationsForPacket), ctx, packetHash) +} + +// ListObserverAdverts mocks base method. +func (m *MockQuerier) ListObserverAdverts(ctx context.Context, arg db.ListObserverAdvertsParams) ([]db.ListObserverAdvertsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListObserverAdverts", ctx, arg) + ret0, _ := ret[0].([]db.ListObserverAdvertsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListObserverAdverts indicates an expected call of ListObserverAdverts. +func (mr *MockQuerierMockRecorder) ListObserverAdverts(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObserverAdverts", reflect.TypeOf((*MockQuerier)(nil).ListObserverAdverts), ctx, arg) +} + +// ListObservers mocks base method. +func (m *MockQuerier) ListObservers(ctx context.Context, arg db.ListObserversParams) ([]db.ListObserversRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListObservers", ctx, arg) + ret0, _ := ret[0].([]db.ListObserversRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListObservers indicates an expected call of ListObservers. +func (mr *MockQuerierMockRecorder) ListObservers(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObservers", reflect.TypeOf((*MockQuerier)(nil).ListObservers), ctx, arg) +} + +// ListPackets mocks base method. +func (m *MockQuerier) ListPackets(ctx context.Context, arg db.ListPacketsParams) ([]db.ListPacketsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListPackets", ctx, arg) + ret0, _ := ret[0].([]db.ListPacketsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListPackets indicates an expected call of ListPackets. +func (mr *MockQuerierMockRecorder) ListPackets(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPackets", reflect.TypeOf((*MockQuerier)(nil).ListPackets), ctx, arg) +} + +// ListPacketsAfterID mocks base method. +func (m *MockQuerier) ListPacketsAfterID(ctx context.Context, arg db.ListPacketsAfterIDParams) ([]db.ListPacketsAfterIDRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListPacketsAfterID", ctx, arg) + ret0, _ := ret[0].([]db.ListPacketsAfterIDRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListPacketsAfterID indicates an expected call of ListPacketsAfterID. +func (mr *MockQuerierMockRecorder) ListPacketsAfterID(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPacketsAfterID", reflect.TypeOf((*MockQuerier)(nil).ListPacketsAfterID), ctx, arg) +} + +// ListPacketsByIATAs mocks base method. +func (m *MockQuerier) ListPacketsByIATAs(ctx context.Context, arg db.ListPacketsByIATAsParams) ([]db.ListPacketsByIATAsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListPacketsByIATAs", ctx, arg) + ret0, _ := ret[0].([]db.ListPacketsByIATAsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListPacketsByIATAs indicates an expected call of ListPacketsByIATAs. +func (mr *MockQuerierMockRecorder) ListPacketsByIATAs(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPacketsByIATAs", reflect.TypeOf((*MockQuerier)(nil).ListPacketsByIATAs), ctx, arg) +} + +// ListRegionState mocks base method. +func (m *MockQuerier) ListRegionState(ctx context.Context) ([]db.ListRegionStateRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRegionState", ctx) + ret0, _ := ret[0].([]db.ListRegionStateRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRegionState indicates an expected call of ListRegionState. +func (mr *MockQuerierMockRecorder) ListRegionState(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegionState", reflect.TypeOf((*MockQuerier)(nil).ListRegionState), ctx) +} + +// ListRegions mocks base method. +func (m *MockQuerier) ListRegions(ctx context.Context) ([]db.ListRegionsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRegions", ctx) + ret0, _ := ret[0].([]db.ListRegionsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRegions indicates an expected call of ListRegions. +func (mr *MockQuerierMockRecorder) ListRegions(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegions", reflect.TypeOf((*MockQuerier)(nil).ListRegions), ctx) +} + +// ListRouteEvidence mocks base method. +func (m *MockQuerier) ListRouteEvidence(ctx context.Context, arg db.ListRouteEvidenceParams) ([]db.ListRouteEvidenceRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRouteEvidence", ctx, arg) + ret0, _ := ret[0].([]db.ListRouteEvidenceRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRouteEvidence indicates an expected call of ListRouteEvidence. +func (mr *MockQuerierMockRecorder) ListRouteEvidence(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRouteEvidence", reflect.TypeOf((*MockQuerier)(nil).ListRouteEvidence), ctx, arg) +} + +// ListScopeCatalogues mocks base method. +func (m *MockQuerier) ListScopeCatalogues(ctx context.Context) ([]db.MeshmapperScopeCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListScopeCatalogues", ctx) + ret0, _ := ret[0].([]db.MeshmapperScopeCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListScopeCatalogues indicates an expected call of ListScopeCatalogues. +func (mr *MockQuerierMockRecorder) ListScopeCatalogues(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListScopeCatalogues", reflect.TypeOf((*MockQuerier)(nil).ListScopeCatalogues), ctx) +} + +// ListTraceTags mocks base method. +func (m *MockQuerier) ListTraceTags(ctx context.Context, arg db.ListTraceTagsParams) ([]db.ListTraceTagsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListTraceTags", ctx, arg) + ret0, _ := ret[0].([]db.ListTraceTagsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListTraceTags indicates an expected call of ListTraceTags. +func (mr *MockQuerierMockRecorder) ListTraceTags(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTraceTags", reflect.TypeOf((*MockQuerier)(nil).ListTraceTags), ctx, arg) +} + +// ListUndecryptedGroupTextPackets mocks base method. +func (m *MockQuerier) ListUndecryptedGroupTextPackets(ctx context.Context) ([]db.ListUndecryptedGroupTextPacketsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListUndecryptedGroupTextPackets", ctx) + ret0, _ := ret[0].([]db.ListUndecryptedGroupTextPacketsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListUndecryptedGroupTextPackets indicates an expected call of ListUndecryptedGroupTextPackets. +func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPackets(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPackets", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPackets), ctx) +} + +// ListUndecryptedGroupTextPacketsByHash mocks base method. +func (m *MockQuerier) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]db.ListUndecryptedGroupTextPacketsByHashRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListUndecryptedGroupTextPacketsByHash", ctx, hashes) + ret0, _ := ret[0].([]db.ListUndecryptedGroupTextPacketsByHashRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListUndecryptedGroupTextPacketsByHash indicates an expected call of ListUndecryptedGroupTextPacketsByHash. +func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPacketsByHash(ctx, hashes any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPacketsByHash", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPacketsByHash), ctx, hashes) +} + +// ListZoneBoundaries mocks base method. +func (m *MockQuerier) ListZoneBoundaries(ctx context.Context) ([]db.MeshmapperZoneBoundary, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListZoneBoundaries", ctx) + ret0, _ := ret[0].([]db.MeshmapperZoneBoundary) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListZoneBoundaries indicates an expected call of ListZoneBoundaries. +func (mr *MockQuerierMockRecorder) ListZoneBoundaries(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneBoundaries", reflect.TypeOf((*MockQuerier)(nil).ListZoneBoundaries), ctx) +} + +// ListZoneLists mocks base method. +func (m *MockQuerier) ListZoneLists(ctx context.Context) ([]db.MeshmapperZoneList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListZoneLists", ctx) + ret0, _ := ret[0].([]db.MeshmapperZoneList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListZoneLists indicates an expected call of ListZoneLists. +func (mr *MockQuerierMockRecorder) ListZoneLists(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneLists", reflect.TypeOf((*MockQuerier)(nil).ListZoneLists), ctx) +} + +// LockRollupHour mocks base method. +func (m *MockQuerier) LockRollupHour(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LockRollupHour", ctx, hour) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LockRollupHour indicates an expected call of LockRollupHour. +func (mr *MockQuerierMockRecorder) LockRollupHour(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockRollupHour", reflect.TypeOf((*MockQuerier)(nil).LockRollupHour), ctx, hour) +} + +// MarkPartialRollupHours mocks base method. +func (m *MockQuerier) MarkPartialRollupHours(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkPartialRollupHours", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkPartialRollupHours indicates an expected call of MarkPartialRollupHours. +func (mr *MockQuerierMockRecorder) MarkPartialRollupHours(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkPartialRollupHours", reflect.TypeOf((*MockQuerier)(nil).MarkPartialRollupHours), ctx) +} + +// OldestMissingRollupHour mocks base method. +func (m *MockQuerier) OldestMissingRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OldestMissingRollupHour", ctx) + ret0, _ := ret[0].(pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// OldestMissingRollupHour indicates an expected call of OldestMissingRollupHour. +func (mr *MockQuerierMockRecorder) OldestMissingRollupHour(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OldestMissingRollupHour", reflect.TypeOf((*MockQuerier)(nil).OldestMissingRollupHour), ctx) +} + +// PruneImportedRegions mocks base method. +func (m *MockQuerier) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PruneImportedRegions", ctx, keep) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PruneImportedRegions indicates an expected call of PruneImportedRegions. +func (mr *MockQuerierMockRecorder) PruneImportedRegions(ctx, keep any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneImportedRegions", reflect.TypeOf((*MockQuerier)(nil).PruneImportedRegions), ctx, keep) +} + +// PruneZoneBoundaries mocks base method. +func (m *MockQuerier) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PruneZoneBoundaries", ctx, keep) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PruneZoneBoundaries indicates an expected call of PruneZoneBoundaries. +func (mr *MockQuerierMockRecorder) PruneZoneBoundaries(ctx, keep any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneZoneBoundaries", reflect.TypeOf((*MockQuerier)(nil).PruneZoneBoundaries), ctx, keep) +} + +// ReconfirmNeighbors mocks base method. +func (m *MockQuerier) ReconfirmNeighbors(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReconfirmNeighbors", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReconfirmNeighbors indicates an expected call of ReconfirmNeighbors. +func (mr *MockQuerierMockRecorder) ReconfirmNeighbors(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmNeighbors", reflect.TypeOf((*MockQuerier)(nil).ReconfirmNeighbors), ctx) +} + +// ReconfirmRoutes mocks base method. +func (m *MockQuerier) ReconfirmRoutes(ctx context.Context, arg db.ReconfirmRoutesParams) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReconfirmRoutes indicates an expected call of ReconfirmRoutes. +func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx, arg) +} + +// RecordTrace mocks base method. +func (m *MockQuerier) RecordTrace(ctx context.Context, arg db.RecordTraceParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecordTrace", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecordTrace indicates an expected call of RecordTrace. +func (mr *MockQuerierMockRecorder) RecordTrace(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordTrace", reflect.TypeOf((*MockQuerier)(nil).RecordTrace), ctx, arg) +} + +// RefreshRadioPresets mocks base method. +func (m *MockQuerier) RefreshRadioPresets(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RefreshRadioPresets", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// RefreshRadioPresets indicates an expected call of RefreshRadioPresets. +func (mr *MockQuerierMockRecorder) RefreshRadioPresets(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshRadioPresets", reflect.TypeOf((*MockQuerier)(nil).RefreshRadioPresets), ctx) +} + +// RegisterRollupHours mocks base method. +func (m *MockQuerier) RegisterRollupHours(ctx context.Context, since pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListKnownRoutes", ctx, arg) - ret0, _ := ret[0].([]db.KnownRoute) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "RegisterRollupHours", ctx, since) + ret0, _ := ret[0].(error) + return ret0 } -// ListKnownRoutes indicates an expected call of ListKnownRoutes. -func (mr *MockQuerierMockRecorder) ListKnownRoutes(ctx, arg any) *gomock.Call { +// RegisterRollupHours indicates an expected call of RegisterRollupHours. +func (mr *MockQuerierMockRecorder) RegisterRollupHours(ctx, since any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListKnownRoutes", reflect.TypeOf((*MockQuerier)(nil).ListKnownRoutes), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterRollupHours", reflect.TypeOf((*MockQuerier)(nil).RegisterRollupHours), ctx, since) } -// ListMessagesAfterID mocks base method. -func (m *MockQuerier) ListMessagesAfterID(ctx context.Context, arg db.ListMessagesAfterIDParams) ([]db.ListMessagesAfterIDRow, error) { +// ResolveEndpointHashPairs mocks base method. +func (m *MockQuerier) ResolveEndpointHashPairs(ctx context.Context, arg db.ResolveEndpointHashPairsParams) ([]db.ResolveEndpointHashPairsRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListMessagesAfterID", ctx, arg) - ret0, _ := ret[0].([]db.ListMessagesAfterIDRow) + ret := m.ctrl.Call(m, "ResolveEndpointHashPairs", ctx, arg) + ret0, _ := ret[0].([]db.ResolveEndpointHashPairsRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListMessagesAfterID indicates an expected call of ListMessagesAfterID. -func (mr *MockQuerierMockRecorder) ListMessagesAfterID(ctx, arg any) *gomock.Call { +// ResolveEndpointHashPairs indicates an expected call of ResolveEndpointHashPairs. +func (mr *MockQuerierMockRecorder) ResolveEndpointHashPairs(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMessagesAfterID", reflect.TypeOf((*MockQuerier)(nil).ListMessagesAfterID), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveEndpointHashPairs", reflect.TypeOf((*MockQuerier)(nil).ResolveEndpointHashPairs), ctx, arg) } -// ListNodeObservations mocks base method. -func (m *MockQuerier) ListNodeObservations(ctx context.Context, arg db.ListNodeObservationsParams) ([]db.ListNodeObservationsRow, error) { +// ResolveEndpointHashes mocks base method. +func (m *MockQuerier) ResolveEndpointHashes(ctx context.Context, arg db.ResolveEndpointHashesParams) ([]db.ResolveEndpointHashesRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListNodeObservations", ctx, arg) - ret0, _ := ret[0].([]db.ListNodeObservationsRow) + ret := m.ctrl.Call(m, "ResolveEndpointHashes", ctx, arg) + ret0, _ := ret[0].([]db.ResolveEndpointHashesRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListNodeObservations indicates an expected call of ListNodeObservations. -func (mr *MockQuerierMockRecorder) ListNodeObservations(ctx, arg any) *gomock.Call { +// ResolveEndpointHashes indicates an expected call of ResolveEndpointHashes. +func (mr *MockQuerierMockRecorder) ResolveEndpointHashes(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListNodeObservations", reflect.TypeOf((*MockQuerier)(nil).ListNodeObservations), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveEndpointHashes", reflect.TypeOf((*MockQuerier)(nil).ResolveEndpointHashes), ctx, arg) } -// ListNodes mocks base method. -func (m *MockQuerier) ListNodes(ctx context.Context, arg db.ListNodesParams) ([]db.ListNodesRow, error) { +// ResolvePathHashesP1 mocks base method. +func (m *MockQuerier) ResolvePathHashesP1(ctx context.Context, arg db.ResolvePathHashesP1Params) ([]db.ResolvePathHashesP1Row, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListNodes", ctx, arg) - ret0, _ := ret[0].([]db.ListNodesRow) + ret := m.ctrl.Call(m, "ResolvePathHashesP1", ctx, arg) + ret0, _ := ret[0].([]db.ResolvePathHashesP1Row) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListNodes indicates an expected call of ListNodes. -func (mr *MockQuerierMockRecorder) ListNodes(ctx, arg any) *gomock.Call { +// ResolvePathHashesP1 indicates an expected call of ResolvePathHashesP1. +func (mr *MockQuerierMockRecorder) ResolvePathHashesP1(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListNodes", reflect.TypeOf((*MockQuerier)(nil).ListNodes), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP1", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP1), ctx, arg) } -// ListObservationsForPacket mocks base method. -func (m *MockQuerier) ListObservationsForPacket(ctx context.Context, packetHash []byte) ([]db.ListObservationsForPacketRow, error) { +// ResolvePathHashesP2 mocks base method. +func (m *MockQuerier) ResolvePathHashesP2(ctx context.Context, arg db.ResolvePathHashesP2Params) ([]db.ResolvePathHashesP2Row, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListObservationsForPacket", ctx, packetHash) - ret0, _ := ret[0].([]db.ListObservationsForPacketRow) + ret := m.ctrl.Call(m, "ResolvePathHashesP2", ctx, arg) + ret0, _ := ret[0].([]db.ResolvePathHashesP2Row) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListObservationsForPacket indicates an expected call of ListObservationsForPacket. -func (mr *MockQuerierMockRecorder) ListObservationsForPacket(ctx, packetHash any) *gomock.Call { +// ResolvePathHashesP2 indicates an expected call of ResolvePathHashesP2. +func (mr *MockQuerierMockRecorder) ResolvePathHashesP2(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObservationsForPacket", reflect.TypeOf((*MockQuerier)(nil).ListObservationsForPacket), ctx, packetHash) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP2", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP2), ctx, arg) } -// ListObserverAdverts mocks base method. -func (m *MockQuerier) ListObserverAdverts(ctx context.Context, arg db.ListObserverAdvertsParams) ([]db.ListObserverAdvertsRow, error) { +// ResolvePathHashesP3 mocks base method. +func (m *MockQuerier) ResolvePathHashesP3(ctx context.Context, arg db.ResolvePathHashesP3Params) ([]db.ResolvePathHashesP3Row, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListObserverAdverts", ctx, arg) - ret0, _ := ret[0].([]db.ListObserverAdvertsRow) + ret := m.ctrl.Call(m, "ResolvePathHashesP3", ctx, arg) + ret0, _ := ret[0].([]db.ResolvePathHashesP3Row) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListObserverAdverts indicates an expected call of ListObserverAdverts. -func (mr *MockQuerierMockRecorder) ListObserverAdverts(ctx, arg any) *gomock.Call { +// ResolvePathHashesP3 indicates an expected call of ResolvePathHashesP3. +func (mr *MockQuerierMockRecorder) ResolvePathHashesP3(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObserverAdverts", reflect.TypeOf((*MockQuerier)(nil).ListObserverAdverts), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP3", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP3), ctx, arg) } -// ListObservers mocks base method. -func (m *MockQuerier) ListObservers(ctx context.Context, arg db.ListObserversParams) ([]db.ListObserversRow, error) { +// ResolvePathHashesP4 mocks base method. +func (m *MockQuerier) ResolvePathHashesP4(ctx context.Context, arg db.ResolvePathHashesP4Params) ([]db.ResolvePathHashesP4Row, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListObservers", ctx, arg) - ret0, _ := ret[0].([]db.ListObserversRow) + ret := m.ctrl.Call(m, "ResolvePathHashesP4", ctx, arg) + ret0, _ := ret[0].([]db.ResolvePathHashesP4Row) ret1, _ := ret[1].(error) return ret0, ret1 } -// ListObservers indicates an expected call of ListObservers. -func (mr *MockQuerierMockRecorder) ListObservers(ctx, arg any) *gomock.Call { +// ResolvePathHashesP4 indicates an expected call of ResolvePathHashesP4. +func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObservers", reflect.TypeOf((*MockQuerier)(nil).ListObservers), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } -// ListPackets mocks base method. -func (m *MockQuerier) ListPackets(ctx context.Context, arg db.ListPacketsParams) ([]db.ListPacketsRow, error) { +// RollAdvertHearings mocks base method. +func (m *MockQuerier) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListPackets", ctx, arg) - ret0, _ := ret[0].([]db.ListPacketsRow) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "RollAdvertHearings", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 } -// ListPackets indicates an expected call of ListPackets. -func (mr *MockQuerierMockRecorder) ListPackets(ctx, arg any) *gomock.Call { +// RollAdvertHearings indicates an expected call of RollAdvertHearings. +func (mr *MockQuerierMockRecorder) RollAdvertHearings(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPackets", reflect.TypeOf((*MockQuerier)(nil).ListPackets), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollAdvertHearings", reflect.TypeOf((*MockQuerier)(nil).RollAdvertHearings), ctx, hour) } -// ListPacketsAfterID mocks base method. -func (m *MockQuerier) ListPacketsAfterID(ctx context.Context, arg db.ListPacketsAfterIDParams) ([]db.ListPacketsAfterIDRow, error) { +// RollAdvertSets mocks base method. +func (m *MockQuerier) RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListPacketsAfterID", ctx, arg) - ret0, _ := ret[0].([]db.ListPacketsAfterIDRow) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "RollAdvertSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 } -// ListPacketsAfterID indicates an expected call of ListPacketsAfterID. -func (mr *MockQuerierMockRecorder) ListPacketsAfterID(ctx, arg any) *gomock.Call { +// RollAdvertSets indicates an expected call of RollAdvertSets. +func (mr *MockQuerierMockRecorder) RollAdvertSets(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPacketsAfterID", reflect.TypeOf((*MockQuerier)(nil).ListPacketsAfterID), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollAdvertSets", reflect.TypeOf((*MockQuerier)(nil).RollAdvertSets), ctx, hour) } -// ListPacketsByIATAs mocks base method. -func (m *MockQuerier) ListPacketsByIATAs(ctx context.Context, arg db.ListPacketsByIATAsParams) ([]db.ListPacketsByIATAsRow, error) { +// RollIATAObservations mocks base method. +func (m *MockQuerier) RollIATAObservations(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListPacketsByIATAs", ctx, arg) - ret0, _ := ret[0].([]db.ListPacketsByIATAsRow) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "RollIATAObservations", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 } -// ListPacketsByIATAs indicates an expected call of ListPacketsByIATAs. -func (mr *MockQuerierMockRecorder) ListPacketsByIATAs(ctx, arg any) *gomock.Call { +// RollIATAObservations indicates an expected call of RollIATAObservations. +func (mr *MockQuerierMockRecorder) RollIATAObservations(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPacketsByIATAs", reflect.TypeOf((*MockQuerier)(nil).ListPacketsByIATAs), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollIATAObservations", reflect.TypeOf((*MockQuerier)(nil).RollIATAObservations), ctx, hour) } -// ListRegions mocks base method. -func (m *MockQuerier) ListRegions(ctx context.Context) ([]db.ListRegionsRow, error) { +// RollObserverActivity mocks base method. +func (m *MockQuerier) RollObserverActivity(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListRegions", ctx) - ret0, _ := ret[0].([]db.ListRegionsRow) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "RollObserverActivity", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 } -// ListRegions indicates an expected call of ListRegions. -func (mr *MockQuerierMockRecorder) ListRegions(ctx any) *gomock.Call { +// RollObserverActivity indicates an expected call of RollObserverActivity. +func (mr *MockQuerierMockRecorder) RollObserverActivity(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegions", reflect.TypeOf((*MockQuerier)(nil).ListRegions), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollObserverActivity", reflect.TypeOf((*MockQuerier)(nil).RollObserverActivity), ctx, hour) } -// ListTraceTags mocks base method. -func (m *MockQuerier) ListTraceTags(ctx context.Context, arg db.ListTraceTagsParams) ([]db.ListTraceTagsRow, error) { +// RollObserverIdentity mocks base method. +func (m *MockQuerier) RollObserverIdentity(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListTraceTags", ctx, arg) - ret0, _ := ret[0].([]db.ListTraceTagsRow) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "RollObserverIdentity", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 } -// ListTraceTags indicates an expected call of ListTraceTags. -func (mr *MockQuerierMockRecorder) ListTraceTags(ctx, arg any) *gomock.Call { +// RollObserverIdentity indicates an expected call of RollObserverIdentity. +func (mr *MockQuerierMockRecorder) RollObserverIdentity(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTraceTags", reflect.TypeOf((*MockQuerier)(nil).ListTraceTags), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollObserverIdentity", reflect.TypeOf((*MockQuerier)(nil).RollObserverIdentity), ctx, hour) } -// ListUndecryptedGroupTextPackets mocks base method. -func (m *MockQuerier) ListUndecryptedGroupTextPackets(ctx context.Context) ([]db.ListUndecryptedGroupTextPacketsRow, error) { +// RollPacketSets mocks base method. +func (m *MockQuerier) RollPacketSets(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListUndecryptedGroupTextPackets", ctx) - ret0, _ := ret[0].([]db.ListUndecryptedGroupTextPacketsRow) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "RollPacketSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 } -// ListUndecryptedGroupTextPackets indicates an expected call of ListUndecryptedGroupTextPackets. -func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPackets(ctx any) *gomock.Call { +// RollPacketSets indicates an expected call of RollPacketSets. +func (mr *MockQuerierMockRecorder) RollPacketSets(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPackets", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPackets), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPacketSets", reflect.TypeOf((*MockQuerier)(nil).RollPacketSets), ctx, hour) } -// ReconfirmNeighbors mocks base method. -func (m *MockQuerier) ReconfirmNeighbors(ctx context.Context) error { +// RollPaths mocks base method. +func (m *MockQuerier) RollPaths(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReconfirmNeighbors", ctx) + ret := m.ctrl.Call(m, "RollPaths", ctx, hour) ret0, _ := ret[0].(error) return ret0 } -// ReconfirmNeighbors indicates an expected call of ReconfirmNeighbors. -func (mr *MockQuerierMockRecorder) ReconfirmNeighbors(ctx any) *gomock.Call { +// RollPaths indicates an expected call of RollPaths. +func (mr *MockQuerierMockRecorder) RollPaths(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmNeighbors", reflect.TypeOf((*MockQuerier)(nil).ReconfirmNeighbors), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPaths", reflect.TypeOf((*MockQuerier)(nil).RollPaths), ctx, hour) } -// ReconfirmRoutes mocks base method. -func (m *MockQuerier) ReconfirmRoutes(ctx context.Context) error { +// RollPayloadBreakdown mocks base method. +func (m *MockQuerier) RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx) + ret := m.ctrl.Call(m, "RollPayloadBreakdown", ctx, hour) ret0, _ := ret[0].(error) return ret0 } -// ReconfirmRoutes indicates an expected call of ReconfirmRoutes. -func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx any) *gomock.Call { +// RollPayloadBreakdown indicates an expected call of RollPayloadBreakdown. +func (mr *MockQuerierMockRecorder) RollPayloadBreakdown(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).RollPayloadBreakdown), ctx, hour) } -// RefreshHourlyStats mocks base method. -func (m *MockQuerier) RefreshHourlyStats(ctx context.Context) error { +// RollScopeNodes mocks base method. +func (m *MockQuerier) RollScopeNodes(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshHourlyStats", ctx) + ret := m.ctrl.Call(m, "RollScopeNodes", ctx, hour) ret0, _ := ret[0].(error) return ret0 } -// RefreshHourlyStats indicates an expected call of RefreshHourlyStats. -func (mr *MockQuerierMockRecorder) RefreshHourlyStats(ctx any) *gomock.Call { +// RollScopeNodes indicates an expected call of RollScopeNodes. +func (mr *MockQuerierMockRecorder) RollScopeNodes(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshHourlyStats", reflect.TypeOf((*MockQuerier)(nil).RefreshHourlyStats), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollScopeNodes", reflect.TypeOf((*MockQuerier)(nil).RollScopeNodes), ctx, hour) } -// RefreshPayloadBreakdown mocks base method. -func (m *MockQuerier) RefreshPayloadBreakdown(ctx context.Context) error { +// RollScopeObservers mocks base method. +func (m *MockQuerier) RollScopeObservers(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshPayloadBreakdown", ctx) + ret := m.ctrl.Call(m, "RollScopeObservers", ctx, hour) ret0, _ := ret[0].(error) return ret0 } -// RefreshPayloadBreakdown indicates an expected call of RefreshPayloadBreakdown. -func (mr *MockQuerierMockRecorder) RefreshPayloadBreakdown(ctx any) *gomock.Call { +// RollScopeObservers indicates an expected call of RollScopeObservers. +func (mr *MockQuerierMockRecorder) RollScopeObservers(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).RefreshPayloadBreakdown), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollScopeObservers", reflect.TypeOf((*MockQuerier)(nil).RollScopeObservers), ctx, hour) } -// RefreshRadioPresets mocks base method. -func (m *MockQuerier) RefreshRadioPresets(ctx context.Context) error { +// RollScopeSets mocks base method. +func (m *MockQuerier) RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshRadioPresets", ctx) + ret := m.ctrl.Call(m, "RollScopeSets", ctx, hour) ret0, _ := ret[0].(error) return ret0 } -// RefreshRadioPresets indicates an expected call of RefreshRadioPresets. -func (mr *MockQuerierMockRecorder) RefreshRadioPresets(ctx any) *gomock.Call { +// RollScopeSets indicates an expected call of RollScopeSets. +func (mr *MockQuerierMockRecorder) RollScopeSets(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshRadioPresets", reflect.TypeOf((*MockQuerier)(nil).RefreshRadioPresets), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollScopeSets", reflect.TypeOf((*MockQuerier)(nil).RollScopeSets), ctx, hour) } -// RefreshTopAdvertisers mocks base method. -func (m *MockQuerier) RefreshTopAdvertisers(ctx context.Context) error { +// RollSignal mocks base method. +func (m *MockQuerier) RollSignal(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopAdvertisers", ctx) + ret := m.ctrl.Call(m, "RollSignal", ctx, hour) ret0, _ := ret[0].(error) return ret0 } -// RefreshTopAdvertisers indicates an expected call of RefreshTopAdvertisers. -func (mr *MockQuerierMockRecorder) RefreshTopAdvertisers(ctx any) *gomock.Call { +// RollSignal indicates an expected call of RollSignal. +func (mr *MockQuerierMockRecorder) RollSignal(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopAdvertisers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopAdvertisers), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollSignal", reflect.TypeOf((*MockQuerier)(nil).RollSignal), ctx, hour) } -// RefreshTopNodes mocks base method. -func (m *MockQuerier) RefreshTopNodes(ctx context.Context) error { +// RollTalkerSets mocks base method. +func (m *MockQuerier) RollTalkerSets(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopNodes", ctx) + ret := m.ctrl.Call(m, "RollTalkerSets", ctx, hour) ret0, _ := ret[0].(error) return ret0 } -// RefreshTopNodes indicates an expected call of RefreshTopNodes. -func (mr *MockQuerierMockRecorder) RefreshTopNodes(ctx any) *gomock.Call { +// RollTalkerSets indicates an expected call of RollTalkerSets. +func (mr *MockQuerierMockRecorder) RollTalkerSets(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopNodes", reflect.TypeOf((*MockQuerier)(nil).RefreshTopNodes), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollTalkerSets", reflect.TypeOf((*MockQuerier)(nil).RollTalkerSets), ctx, hour) } -// RefreshTopObservers mocks base method. -func (m *MockQuerier) RefreshTopObservers(ctx context.Context) error { +// RollupContentHash mocks base method. +func (m *MockQuerier) RollupContentHash(ctx context.Context, hour pgtype.Timestamptz) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopObservers", ctx) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "RollupContentHash", ctx, hour) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// RefreshTopObservers indicates an expected call of RefreshTopObservers. -func (mr *MockQuerierMockRecorder) RefreshTopObservers(ctx any) *gomock.Call { +// RollupContentHash indicates an expected call of RollupContentHash. +func (mr *MockQuerierMockRecorder) RollupContentHash(ctx, hour any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopObservers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopObservers), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollupContentHash", reflect.TypeOf((*MockQuerier)(nil).RollupContentHash), ctx, hour) } -// RefreshTopTalkers mocks base method. -func (m *MockQuerier) RefreshTopTalkers(ctx context.Context) error { +// RollupUnlock mocks base method. +func (m *MockQuerier) RollupUnlock(ctx context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopTalkers", ctx) + ret := m.ctrl.Call(m, "RollupUnlock", ctx) ret0, _ := ret[0].(error) return ret0 } -// RefreshTopTalkers indicates an expected call of RefreshTopTalkers. -func (mr *MockQuerierMockRecorder) RefreshTopTalkers(ctx any) *gomock.Call { +// RollupUnlock indicates an expected call of RollupUnlock. +func (mr *MockQuerierMockRecorder) RollupUnlock(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopTalkers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopTalkers), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollupUnlock", reflect.TypeOf((*MockQuerier)(nil).RollupUnlock), ctx) } -// ResolvePathHashesP1 mocks base method. -func (m *MockQuerier) ResolvePathHashesP1(ctx context.Context, arg db.ResolvePathHashesP1Params) ([]db.ResolvePathHashesP1Row, error) { +// SaveChannelCatalogue mocks base method. +func (m *MockQuerier) SaveChannelCatalogue(ctx context.Context, arg db.SaveChannelCatalogueParams) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResolvePathHashesP1", ctx, arg) - ret0, _ := ret[0].([]db.ResolvePathHashesP1Row) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "SaveChannelCatalogue", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 } -// ResolvePathHashesP1 indicates an expected call of ResolvePathHashesP1. -func (mr *MockQuerierMockRecorder) ResolvePathHashesP1(ctx, arg any) *gomock.Call { +// SaveChannelCatalogue indicates an expected call of SaveChannelCatalogue. +func (mr *MockQuerierMockRecorder) SaveChannelCatalogue(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP1", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP1), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveChannelCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveChannelCatalogue), ctx, arg) } -// ResolvePathHashesP2 mocks base method. -func (m *MockQuerier) ResolvePathHashesP2(ctx context.Context, arg db.ResolvePathHashesP2Params) ([]db.ResolvePathHashesP2Row, error) { +// SaveScopeCatalogue mocks base method. +func (m *MockQuerier) SaveScopeCatalogue(ctx context.Context, arg db.SaveScopeCatalogueParams) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResolvePathHashesP2", ctx, arg) - ret0, _ := ret[0].([]db.ResolvePathHashesP2Row) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "SaveScopeCatalogue", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 } -// ResolvePathHashesP2 indicates an expected call of ResolvePathHashesP2. -func (mr *MockQuerierMockRecorder) ResolvePathHashesP2(ctx, arg any) *gomock.Call { +// SaveScopeCatalogue indicates an expected call of SaveScopeCatalogue. +func (mr *MockQuerierMockRecorder) SaveScopeCatalogue(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP2", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP2), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveScopeCatalogue), ctx, arg) } -// ResolvePathHashesP3 mocks base method. -func (m *MockQuerier) ResolvePathHashesP3(ctx context.Context, arg db.ResolvePathHashesP3Params) ([]db.ResolvePathHashesP3Row, error) { +// SaveZoneBoundary mocks base method. +func (m *MockQuerier) SaveZoneBoundary(ctx context.Context, arg db.SaveZoneBoundaryParams) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResolvePathHashesP3", ctx, arg) - ret0, _ := ret[0].([]db.ResolvePathHashesP3Row) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "SaveZoneBoundary", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 } -// ResolvePathHashesP3 indicates an expected call of ResolvePathHashesP3. -func (mr *MockQuerierMockRecorder) ResolvePathHashesP3(ctx, arg any) *gomock.Call { +// SaveZoneBoundary indicates an expected call of SaveZoneBoundary. +func (mr *MockQuerierMockRecorder) SaveZoneBoundary(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP3", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP3), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneBoundary", reflect.TypeOf((*MockQuerier)(nil).SaveZoneBoundary), ctx, arg) } -// ResolvePathHashesP4 mocks base method. -func (m *MockQuerier) ResolvePathHashesP4(ctx context.Context, arg db.ResolvePathHashesP4Params) ([]db.ResolvePathHashesP4Row, error) { +// SaveZoneList mocks base method. +func (m *MockQuerier) SaveZoneList(ctx context.Context, arg db.SaveZoneListParams) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResolvePathHashesP4", ctx, arg) - ret0, _ := ret[0].([]db.ResolvePathHashesP4Row) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "SaveZoneList", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 } -// ResolvePathHashesP4 indicates an expected call of ResolvePathHashesP4. -func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Call { +// SaveZoneList indicates an expected call of SaveZoneList. +func (mr *MockQuerierMockRecorder) SaveZoneList(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneList", reflect.TypeOf((*MockQuerier)(nil).SaveZoneList), ctx, arg) } // SearchKnownRoutes mocks base method. -func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.KnownRoute, error) { +func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SearchKnownRoutes", ctx, arg) - ret0, _ := ret[0].([]db.KnownRoute) + ret0, _ := ret[0].([]db.SearchKnownRoutesRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -1254,6 +2234,20 @@ func (mr *MockQuerierMockRecorder) SetPacketDecrypted(ctx, packetHash any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPacketDecrypted", reflect.TypeOf((*MockQuerier)(nil).SetPacketDecrypted), ctx, packetHash) } +// SetRollupHourPartial mocks base method. +func (m *MockQuerier) SetRollupHourPartial(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetRollupHourPartial", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetRollupHourPartial indicates an expected call of SetRollupHourPartial. +func (mr *MockQuerierMockRecorder) SetRollupHourPartial(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetRollupHourPartial", reflect.TypeOf((*MockQuerier)(nil).SetRollupHourPartial), ctx, hour) +} + // TouchObserverBrokers mocks base method. func (m *MockQuerier) TouchObserverBrokers(ctx context.Context, arg db.TouchObserverBrokersParams) error { m.ctrl.T.Helper() @@ -1296,6 +2290,35 @@ func (mr *MockQuerierMockRecorder) TouchPackets(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TouchPackets", reflect.TypeOf((*MockQuerier)(nil).TouchPackets), ctx, arg) } +// TryRollupLock mocks base method. +func (m *MockQuerier) TryRollupLock(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TryRollupLock", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TryRollupLock indicates an expected call of TryRollupLock. +func (mr *MockQuerierMockRecorder) TryRollupLock(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryRollupLock", reflect.TypeOf((*MockQuerier)(nil).TryRollupLock), ctx) +} + +// UpdateObserverRegionScope mocks base method. +func (m *MockQuerier) UpdateObserverRegionScope(ctx context.Context, arg db.UpdateObserverRegionScopeParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateObserverRegionScope", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateObserverRegionScope indicates an expected call of UpdateObserverRegionScope. +func (mr *MockQuerierMockRecorder) UpdateObserverRegionScope(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateObserverRegionScope", reflect.TypeOf((*MockQuerier)(nil).UpdateObserverRegionScope), ctx, arg) +} + // UpdateObserverStatus mocks base method. func (m *MockQuerier) UpdateObserverStatus(ctx context.Context, arg db.UpdateObserverStatusParams) (uuid.UUID, error) { m.ctrl.T.Helper() @@ -1397,6 +2420,21 @@ func (mr *MockQuerierMockRecorder) UpsertIATADetails(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertIATADetails", reflect.TypeOf((*MockQuerier)(nil).UpsertIATADetails), ctx, arg) } +// UpsertImportedRegion mocks base method. +func (m *MockQuerier) UpsertImportedRegion(ctx context.Context, arg db.UpsertImportedRegionParams) (int32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertImportedRegion", ctx, arg) + ret0, _ := ret[0].(int32) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpsertImportedRegion indicates an expected call of UpsertImportedRegion. +func (mr *MockQuerierMockRecorder) UpsertImportedRegion(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertImportedRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertImportedRegion), ctx, arg) +} + // UpsertKnownRoute mocks base method. func (m *MockQuerier) UpsertKnownRoute(ctx context.Context, arg db.UpsertKnownRouteParams) error { m.ctrl.T.Helper() @@ -1469,18 +2507,18 @@ func (mr *MockQuerierMockRecorder) UpsertNodeShortID(ctx, arg any) *gomock.Call } // UpsertObserver mocks base method. -func (m *MockQuerier) UpsertObserver(ctx context.Context, publicKey []byte) (db.Observer, error) { +func (m *MockQuerier) UpsertObserver(ctx context.Context, arg db.UpsertObserverParams) (db.Observer, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpsertObserver", ctx, publicKey) + ret := m.ctrl.Call(m, "UpsertObserver", ctx, arg) ret0, _ := ret[0].(db.Observer) ret1, _ := ret[1].(error) return ret0, ret1 } // UpsertObserver indicates an expected call of UpsertObserver. -func (mr *MockQuerierMockRecorder) UpsertObserver(ctx, publicKey any) *gomock.Call { +func (mr *MockQuerierMockRecorder) UpsertObserver(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertObserver", reflect.TypeOf((*MockQuerier)(nil).UpsertObserver), ctx, publicKey) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertObserver", reflect.TypeOf((*MockQuerier)(nil).UpsertObserver), ctx, arg) } // UpsertObserverBroker mocks base method. @@ -1541,34 +2579,6 @@ func (mr *MockQuerierMockRecorder) UpsertRegion(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertRegion), ctx, arg) } -// UpsertRegionIATA mocks base method. -func (m *MockQuerier) UpsertRegionIATA(ctx context.Context, arg db.UpsertRegionIATAParams) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpsertRegionIATA", ctx, arg) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpsertRegionIATA indicates an expected call of UpsertRegionIATA. -func (mr *MockQuerierMockRecorder) UpsertRegionIATA(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegionIATA", reflect.TypeOf((*MockQuerier)(nil).UpsertRegionIATA), ctx, arg) -} - -// UpsertTraceIATA mocks base method. -func (m *MockQuerier) UpsertTraceIATA(ctx context.Context, arg db.UpsertTraceIATAParams) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpsertTraceIATA", ctx, arg) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpsertTraceIATA indicates an expected call of UpsertTraceIATA. -func (mr *MockQuerierMockRecorder) UpsertTraceIATA(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertTraceIATA", reflect.TypeOf((*MockQuerier)(nil).UpsertTraceIATA), ctx, arg) -} - // UpsertTransportScope mocks base method. func (m *MockQuerier) UpsertTransportScope(ctx context.Context, arg db.UpsertTransportScopeParams) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index a91f5451..78e14301 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -9,6 +9,148 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +type Account struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + DeactivatedAt pgtype.Timestamptz `json:"deactivated_at"` +} + +type AnalyticsDirtyHour struct { + Hour pgtype.Timestamptz `json:"hour"` + EnqueuedAt pgtype.Timestamptz `json:"enqueued_at"` +} + +type AnalyticsHourlyAdvertHearing struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + OriginPubkey []byte `json:"origin_pubkey"` + Observations int64 `json:"observations"` + LastHeard pgtype.Timestamptz `json:"last_heard"` + Name *string `json:"name"` + NodeType *int16 `json:"node_type"` +} + +type AnalyticsHourlyAdvertSet struct { + Hour pgtype.Timestamptz `json:"hour"` + OriginPubkey []byte `json:"origin_pubkey"` + Iatas []string `json:"iatas"` + AdvertPackets int64 `json:"advert_packets"` + FloodPackets int64 `json:"flood_packets"` + DirectPackets int64 `json:"direct_packets"` +} + +type AnalyticsHourlyIataObservation struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + ObservationCount int64 `json:"observation_count"` +} + +type AnalyticsHourlyObserverActivity struct { + Hour pgtype.Timestamptz `json:"hour"` + ObserverID uuid.UUID `json:"observer_id"` + PayloadType int16 `json:"payload_type"` + Observations int64 `json:"observations"` + AirtimeMs *float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum *float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin *float32 `json:"snr_min"` + RssiSum *int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + +type AnalyticsHourlyObserverIdentity struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + ObservationCount int64 `json:"observation_count"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` +} + +type AnalyticsHourlyPacketSet struct { + Hour pgtype.Timestamptz `json:"hour"` + Iatas []string `json:"iatas"` + Packets int64 `json:"packets"` +} + +type AnalyticsHourlyPath struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +type AnalyticsHourlyPayloadBreakdown struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + PayloadType int16 `json:"payload_type"` + Count int64 `json:"count"` +} + +type AnalyticsHourlyScopeNode struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + ScopeID int32 `json:"scope_id"` + OriginPubkey []byte `json:"origin_pubkey"` +} + +type AnalyticsHourlyScopeObserver struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + ScopeID int32 `json:"scope_id"` + ObserverID uuid.UUID `json:"observer_id"` +} + +type AnalyticsHourlyScopeSet struct { + Hour pgtype.Timestamptz `json:"hour"` + Iatas []string `json:"iatas"` + ScopeID int32 `json:"scope_id"` + Packets int64 `json:"packets"` +} + +type AnalyticsHourlySignal struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrSum float64 `json:"snr_sum"` + RssiSamples int64 `json:"rssi_samples"` + RssiSum float64 `json:"rssi_sum"` +} + +type AnalyticsHourlyTalkerSet struct { + Hour pgtype.Timestamptz `json:"hour"` + SenderName string `json:"sender_name"` + Iatas []string `json:"iatas"` + Messages int64 `json:"messages"` + LastSent pgtype.Timestamptz `json:"last_sent"` +} + +type AnalyticsRawState struct { + Singleton bool `json:"singleton"` + RawDeletedBefore pgtype.Timestamptz `json:"raw_deleted_before"` + Coverage int64 `json:"coverage"` +} + +type AnalyticsRollupHour struct { + Hour pgtype.Timestamptz `json:"hour"` + Status string `json:"status"` + RolledAt pgtype.Timestamptz `json:"rolled_at"` + ContentHash *string `json:"content_hash"` +} + +type AnalyticsState struct { + Singleton bool `json:"singleton"` + Revision int64 `json:"revision"` +} + type Channel struct { ID int32 `json:"id"` ChannelHash []byte `json:"channel_hash"` @@ -20,7 +162,12 @@ type Channel struct { KeyKnown *bool `json:"key_known"` FirstSeen pgtype.Timestamptz `json:"first_seen"` LastSeen pgtype.Timestamptz `json:"last_seen"` - MessageCount *int64 `json:"message_count"` + MessageCount int64 `json:"message_count"` +} + +type ChannelConfigScope struct { + KeyFingerprint []byte `json:"key_fingerprint"` + RegionSlug *string `json:"region_slug"` } type ChannelIata struct { @@ -57,72 +204,71 @@ type IataCode struct { } type KnownRoute struct { - ID int64 `json:"id"` - NodeIds []uuid.UUID `json:"node_ids"` - HashPrefix [][]byte `json:"hash_prefix"` - Iata string `json:"iata"` - HopCount int32 `json:"hop_count"` - FirstSeen pgtype.Timestamptz `json:"first_seen"` - LastSeen pgtype.Timestamptz `json:"last_seen"` - ObservationCount int64 `json:"observation_count"` -} - -type MvHourlyIataStat struct { - Iata string `json:"iata"` - Hour pgtype.Timestamptz `json:"hour"` - ObservationCount int64 `json:"observation_count"` - UniquePackets int64 `json:"unique_packets"` - ActiveObservers int64 `json:"active_observers"` + ID int64 `json:"id"` + PathKey []byte `json:"path_key"` + NodeIds []uuid.UUID `json:"node_ids"` + HashPrefix [][]byte `json:"hash_prefix"` + Iata string `json:"iata"` + HopCount int32 `json:"hop_count"` + FirstSeen pgtype.Timestamptz `json:"first_seen"` + LastSeen pgtype.Timestamptz `json:"last_seen"` + ObservationCount int64 `json:"observation_count"` + LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"` } -type MvPayloadBreakdownByIatum struct { +type MeshmapperChannelCatalogue struct { Iata string `json:"iata"` - PayloadType *int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - Count int64 `json:"count"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` } -type MvRadioPreset struct { - Preset string `json:"preset"` - Iata string `json:"iata"` - SourceType string `json:"source_type"` - Count int64 `json:"count"` +type MeshmapperChannelMember struct { + Iata string `json:"iata"` + KeyFingerprint []byte `json:"key_fingerprint"` } -type MvTopAdvertisersByIatum struct { +type MeshmapperScopeCatalogue struct { Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - AdvertCount int64 `json:"advert_count"` - LastHeard interface{} `json:"last_heard"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` } -type MvTopNodesByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - ObservationCount *int64 `json:"observation_count"` - LastHeard pgtype.Timestamptz `json:"last_heard"` +type MeshmapperZoneBoundary struct { + Iata string `json:"iata"` + Url string `json:"url"` + Feature []byte `json:"feature"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` } -type MvTopObserversByIatum struct { - Iata string `json:"iata"` - ObserverID uuid.UUID `json:"observer_id"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - ObservationCount int64 `json:"observation_count"` +type MeshmapperZoneList struct { + Country string `json:"country"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + FetchedAt pgtype.Timestamptz `json:"fetched_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` } -type MvTopTalkersByIatum struct { - Iata string `json:"iata"` - SenderName *string `json:"sender_name"` - Bucket pgtype.Timestamptz `json:"bucket"` - MessageCount int64 `json:"message_count"` - LastSent interface{} `json:"last_sent"` +type MvRadioPreset struct { + Preset string `json:"preset"` + Iata *string `json:"iata"` + SourceType string `json:"source_type"` + Count int64 `json:"count"` } type Node struct { @@ -163,6 +309,7 @@ type NodeNeighbor struct { LastSeen pgtype.Timestamptz `json:"last_seen"` ObservationCount int64 `json:"observation_count"` Snr *float32 `json:"snr"` + RegionScope *string `json:"region_scope"` } type NodeShortID struct { @@ -195,6 +342,9 @@ type Observer struct { LastSeen pgtype.Timestamptz `json:"last_seen"` ObservationCount *int64 `json:"observation_count"` Metadata []byte `json:"metadata"` + RegionScope *string `json:"region_scope"` + LastIata *string `json:"last_iata"` + LastIataAt pgtype.Timestamptz `json:"last_iata_at"` } type ObserverBroker struct { @@ -238,8 +388,8 @@ type ObserverTelemetry struct { ObserverID uuid.UUID `json:"observer_id"` ReportedAt pgtype.Timestamptz `json:"reported_at"` BatteryVoltageMv *int32 `json:"battery_voltage_mv"` - AirtimeTxPct *float32 `json:"airtime_tx_pct"` - AirtimeRxPct *float32 `json:"airtime_rx_pct"` + AirtimeTxSecs *float32 `json:"airtime_tx_secs"` + AirtimeRxSecs *float32 `json:"airtime_rx_secs"` NoiseFloorDb *float32 `json:"noise_floor_db"` UptimeSeconds *int64 `json:"uptime_seconds"` QueueLength *int32 `json:"queue_length"` @@ -265,6 +415,7 @@ type Packet struct { TraceTag []byte `json:"trace_tag"` FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` } type PacketObservation struct { @@ -286,6 +437,7 @@ type PacketObservation struct { CodingRate *int16 `json:"coding_rate"` SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` + AirtimeMs *float32 `json:"airtime_ms"` } type Region struct { @@ -299,6 +451,7 @@ type Region struct { ZoomLevel *int32 `json:"zoom_level"` CreatedAt pgtype.Timestamptz `json:"created_at"` UpdatedAt pgtype.Timestamptz `json:"updated_at"` + Imported bool `json:"imported"` } type RegionIata struct { @@ -307,12 +460,42 @@ type RegionIata struct { AddedAt pgtype.Timestamptz `json:"added_at"` } +type RollupOb struct { + PacketHash []byte `json:"packet_hash"` + ObserverID uuid.UUID `json:"observer_id"` + Iata string `json:"iata"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + PathLengthByte int16 `json:"path_length_byte"` + HashSize int16 `json:"hash_size"` + HopCount int16 `json:"hop_count"` + PathLen int32 `json:"path_len"` + Rssi *int16 `json:"rssi"` + Snr *float32 `json:"snr"` + AirtimeMs *float32 `json:"airtime_ms"` + PayloadType *int16 `json:"payload_type"` + RouteType int16 `json:"route_type"` + OriginPubkey []byte `json:"origin_pubkey"` + ScopeID *int32 `json:"scope_id"` +} + type TraceIata struct { TraceTag []byte `json:"trace_tag"` Iata string `json:"iata"` LastHeard pgtype.Timestamptz `json:"last_heard"` } +type TraceTag struct { + TraceTag []byte `json:"trace_tag"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + PacketCount int64 `json:"packet_count"` + TraceType *string `json:"trace_type"` + ScopeID *int32 `json:"scope_id"` + BestPayload []byte `json:"best_payload"` + BestPathLen int32 `json:"best_path_len"` + Heard bool `json:"heard"` +} + type TransportScope struct { ID int32 `json:"id"` Name string `json:"name"` @@ -320,4 +503,5 @@ type TransportScope struct { TransportKey []byte `json:"transport_key"` KeyFingerprint []byte `json:"key_fingerprint"` CreatedAt pgtype.Timestamptz `json:"created_at"` + ImportedOnly bool `json:"imported_only"` } diff --git a/db/sqlc/observer_comparison.sql.go b/db/sqlc/observer_comparison.sql.go new file mode 100644 index 00000000..00c0769e --- /dev/null +++ b/db/sqlc/observer_comparison.sql.go @@ -0,0 +1,75 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: observer_comparison.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const getObserverComparison = `-- name: GetObserverComparison :one +WITH heard AS ( + SELECT po.packet_hash, + bool_or(po.observer_id = $1::uuid) AS seen_a, + bool_or(po.observer_id = $2::uuid) AS seen_b + FROM packet_observations po + JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.observer_id IN ($1::uuid, $2::uuid) + AND po.heard_at >= $3::timestamptz + AND po.heard_at < $4::timestamptz + AND p.route_type IN (0, 1) + AND (COALESCE(cardinality($5::text[]), 0) = 0 OR po.iata = ANY($5::text[])) + GROUP BY po.packet_hash +) +SELECT count(*) AS total_packets, + count(*) FILTER (WHERE seen_a AND NOT seen_b) AS only_a, + count(*) FILTER (WHERE seen_b AND NOT seen_a) AS only_b, + count(*) FILTER (WHERE seen_a AND seen_b) AS both, + EXISTS (SELECT 1 FROM observers WHERE id = $1::uuid) AS known_a, + EXISTS (SELECT 1 FROM observers WHERE id = $2::uuid) AS known_b +FROM heard +` + +type GetObserverComparisonParams struct { + ObserverA uuid.UUID `json:"observer_a"` + ObserverB uuid.UUID `json:"observer_b"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetObserverComparisonRow struct { + TotalPackets int64 `json:"total_packets"` + OnlyA int64 `json:"only_a"` + OnlyB int64 `json:"only_b"` + Both int64 `json:"both"` + KnownA bool `json:"known_a"` + KnownB bool `json:"known_b"` +} + +// Count packet identities, not reception rows: a broker repeat or another +// reception by the same observer must not inflate the comparison. +func (q *Queries) GetObserverComparison(ctx context.Context, arg GetObserverComparisonParams) (GetObserverComparisonRow, error) { + row := q.db.QueryRow(ctx, getObserverComparison, + arg.ObserverA, + arg.ObserverB, + arg.Since, + arg.Until, + arg.Iatas, + ) + var i GetObserverComparisonRow + err := row.Scan( + &i.TotalPackets, + &i.OnlyA, + &i.OnlyB, + &i.Both, + &i.KnownA, + &i.KnownB, + ) + return i, err +} diff --git a/db/sqlc/paths.sql.go b/db/sqlc/paths.sql.go new file mode 100644 index 00000000..5c2f3422 --- /dev/null +++ b/db/sqlc/paths.sql.go @@ -0,0 +1,78 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: paths.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getPathStats = `-- name: GetPathStats :many +WITH readings AS ( + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz + AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz + AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) +) +SELECT (grouping(hour) = 0)::boolean AS is_hourly, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + category, hash_bytes, COALESCE(entries, 0)::integer AS entries, + sum(receptions)::bigint AS receptions, + -- Non-routed categories store entries = 0, so an hour of only empty paths gives 0. + COALESCE(max(entries) FILTER (WHERE receptions > 0), 0)::integer AS max_entries +FROM readings +GROUP BY GROUPING SETS ((category, hash_bytes, entries), (hour, category, hash_bytes)) +ORDER BY is_hourly, hour, category, hash_bytes, entries +` + +type GetPathStatsParams struct { + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetPathStatsRow struct { + IsHourly bool `json:"is_hourly"` + Hour pgtype.Timestamptz `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` + MaxEntries int32 `json:"max_entries"` +} + +// All HTTP aggregates read the hourly rollup, never observations. +func (q *Queries) GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]GetPathStatsRow, error) { + rows, err := q.db.Query(ctx, getPathStats, arg.Since, arg.Until, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetPathStatsRow{} + for rows.Next() { + var i GetPathStatsRow + if err := rows.Scan( + &i.IsHourly, + &i.Hour, + &i.Category, + &i.HashBytes, + &i.Entries, + &i.Receptions, + &i.MaxEntries, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 03a6a840..f3c54d3c 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -6,12 +6,27 @@ package db import ( "context" + "encoding/json" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) type Querier interface { + // An empty region places the channel Beacon-wide. + AddChannelConfigScopes(ctx context.Context, arg AddChannelConfigScopesParams) error + AddChannelMembers(ctx context.Context, arg AddChannelMembersParams) error + AddIATAs(ctx context.Context, iatas []string) error + AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error + // Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. + AmbiguousPrefixes(ctx context.Context) ([]AmbiguousPrefixesRow, error) + CreateAccount(ctx context.Context, name string) (Account, error) + // Lock the current row before deciding the outcome, including when another + // deactivation commits while this statement is waiting for its row lock. + DeactivateAccount(ctx context.Context, id uuid.UUID) (DeactivateAccountRow, error) + DeleteAllChannelMembers(ctx context.Context) error + DeleteChannelConfigScopes(ctx context.Context) error + DeleteChannelMembersNotIn(ctx context.Context, arg DeleteChannelMembersNotInParams) error // Keeps the channel IATA filter in step with packet retention. DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error // Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- @@ -22,84 +37,149 @@ type Querier interface { // dangling in old routes there, but ReconfirmTask already prunes stale/ambiguous routes // periodically and will clean those up on its own schedule. DeleteOldNodes(ctx context.Context, lastSeen pgtype.Timestamptz) error - // Deletes packets and their observations older than the given cutoff. - // packet_observations cascade-delete via FK. - DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error + // Opt-in age-out: preserve retained history and manually recorded ownership. + // Bound deletions per cleanup tick and skip observers being updated by ingest. + DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) + // Deletes one bounded cohort (observations cascade). SKIP LOCKED leaves packets an in-flight + // observation insert holds. raw_deleted_before tells the rollup which hours lost raw rows. + DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) + // Bumps coverage when hours go, so cached responses that included them are not reused. + DeleteOldRollups(ctx context.Context, cutoff pgtype.Timestamptz) error + // One batch of routes past retention, or past grace with too few observations. + // GREATEST keeps the scan on idx_known_routes_last_seen. + DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) (int64, error) // Deletes telemetry rows older than the given cutoff. Called by the cleanup goroutine. DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error // Keeps the trace IATA filter in step with packet retention. DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error + DeleteOldTraceTags(ctx context.Context, lastHeardAt pgtype.Timestamptz) error + DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error + DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error + // Dirty hours that can no longer be re-rolled. + DeleteStaleDirtyHours(ctx context.Context) error + FillRollupObs(ctx context.Context, dollar_1 pgtype.Timestamptz) error + // Marks the hour complete and bumps the revision only when its content changed. + FinishRollupHour(ctx context.Context, arg FinishRollupHourParams) (bool, error) + GetAccount(ctx context.Context, id uuid.UUID) (Account, error) + // Both counters only grow, so their sum changes whenever rolled content or coverage does. + GetAnalyticsRevision(ctx context.Context) (int64, error) GetChannelByID(ctx context.Context, id int32) (Channel, error) // Returns neighbors of a node that are in a different IATA. GetCrossIATANeighbors(ctx context.Context, arg GetCrossIATANeighborsParams) ([]GetCrossIATANeighborsRow, error) - GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]MvHourlyIataStat, error) + GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) + // ============================================================ + // STATS + // ============================================================ + GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]GetHourlyStatsRow, error) GetIATA(ctx context.Context, iata string) (IataCode, error) - // border is NULL when the IATA exists but has no border configured; a - // missing row (unknown IATA) is sql.ErrNoRows, same not-found distinction - // GetIATA already makes. - GetIATABorder(ctx context.Context, iata string) ([]byte, error) - GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]KnownRoute, error) + // An imported MeshMapper boundary overrides the configured one. border is NULL + // when neither exists; a missing row (unknown IATA) is sql.ErrNoRows, same + // not-found distinction GetIATA already makes. + GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) + GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error) + GetLatestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) GetNodeByID(ctx context.Context, id uuid.UUID) (GetNodeByIDRow, error) GetNodeByPubkey(ctx context.Context, publicKey []byte) (uuid.UUID, error) // Returns the neighbors of a node with details, ordered by most recently seen. GetNodeNeighbors(ctx context.Context, nodeID uuid.UUID) ([]GetNodeNeighborsRow, error) GetNodesByIDs(ctx context.Context, dollar_1 []uuid.UUID) ([]GetNodesByIDsRow, error) + GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]GetNodesByPubkeysRow, error) + // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. + // Hours from @tail_start on aren't rolled yet and read raw rows. + GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) + // Same rollup/raw-tail split as GetObserverActivityHourly. + GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) + // Two indexed ranges, bounded to one observer; no legacy presence counters. + GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) + // Sub-hour activity buckets straight off idx_observations_observer; no join to packets. + // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as + // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. + GetObserverActivityRaw(ctx context.Context, arg GetObserverActivityRawParams) ([]GetObserverActivityRawRow, error) + GetObserverActivityRawPayloadTypes(ctx context.Context, arg GetObserverActivityRawPayloadTypesParams) ([]GetObserverActivityRawPayloadTypesRow, error) GetObserverBrokers(ctx context.Context, observerID uuid.UUID) ([]GetObserverBrokersRow, error) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, error) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Observer, error) - GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) + // Count packet identities, not reception rows: a broker repeat or another + // reception by the same observer must not inflate the comparison. + GetObserverComparison(ctx context.Context, arg GetObserverComparisonParams) (GetObserverComparisonRow, error) + GetObserverLastIATA(ctx context.Context, id uuid.UUID) (string, error) GetObserverRadio(ctx context.Context, id uuid.UUID) (GetObserverRadioRow, error) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) GetObserverTelemetry(ctx context.Context, arg GetObserverTelemetryParams) ([]GetObserverTelemetryRow, error) - GetObserverTelemetryBucketed(ctx context.Context, arg GetObserverTelemetryBucketedParams) ([]GetObserverTelemetryBucketedRow, error) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPacketByHashRow, error) - GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) - // Returns all packets for a given trace tag with observations. + // Return distinct observation IATAs in first-heard order for path resolution, + // without fetching full observations separately for every trace packet. GetPacketsByTraceTag(ctx context.Context, decode string) ([]GetPacketsByTraceTagRow, error) + // All HTTP aggregates read the hourly rollup, never observations. + GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]GetPathStatsRow, error) GetRadioPresets(ctx context.Context, arg GetRadioPresetsParams) ([]MvRadioPreset, error) GetRegion(ctx context.Context, id int32) (GetRegionRow, error) GetRegionBySlug(ctx context.Context, slug string) (GetRegionBySlugRow, error) GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) + GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) + // Packets and IATAs cover every retained rollup hour; IATAs are those that heard the + // scope's own packets. Observer and node counts are current memberships. GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) + GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) GetScopeNames(ctx context.Context) ([]string, error) - // Count each table on its own; the old cross-join blew up to millions of rows - // before COUNT(DISTINCT) (~10s). - GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) - GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) + // Packets since the given hour come from the IATA-set rollup (counted once per hour heard). + // Observer and node counts are current memberships; observers filter by their latest IATA. + GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([]GetScopeStatsRow, error) + // GetScopeStats packet counts split by hour (same window and IATA-set filter), with the + // distinct observers and advertising nodes active in each scope that hour. Empty hours omitted. + GetScopeStatsHourly(ctx context.Context, arg GetScopeStatsHourlyParams) ([]GetScopeStatsHourlyRow, error) + // Read the hourly rollup, never observations on an HTTP request. + // Weight averages by sample counts instead of averaging regional/hourly means. + GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) + // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds + // the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's + // latest measured drift, not an aggregate over a period. + // Select the filtered page before aggregating its IATA memberships. + GetStatsClockDrift(ctx context.Context, arg GetStatsClockDriftParams) ([]GetStatsClockDriftRow, error) // Returns node counts grouped by type, optionally filtered by IATA. GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]GetStatsNodeTypesRow, error) - // ============================================================ - // STATS - // ============================================================ - GetStatsOverview(ctx context.Context, dollar_1 []string) (GetStatsOverviewRow, error) - // Payload-type counts for the IATA within the window, summed from the - // precomputed hourly buckets. + // Payload-type observation counts since the given hour, from the hourly rollup. GetStatsPayloadBreakdown(ctx context.Context, arg GetStatsPayloadBreakdownParams) ([]GetStatsPayloadBreakdownRow, error) - // Top N advertisers in the window, summed from the hourly buckets. + // One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours. + // Counts sum; observers, IATAs and scopes are distinct across the window; packets count + // once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate. + GetStatsSeries(ctx context.Context, arg GetStatsSeriesParams) ([]GetStatsSeriesRow, error) + // Top N advertisers since the given hour. Each ADVERT packet counts once per hour heard, + // however many of the requested IATAs heard it (IATA-set rollup). Live names win. GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdvertisersParams) ([]GetStatsTopAdvertisersRow, error) - // Top N observers for the IATA within the window, summed from the precomputed - // hourly buckets. Counts sum across matched IATAs; iata is a representative one. + // Top N observers since the given hour. Counts sum across matched IATAs; iata is a + // representative one. Live names win over the rolled snapshot. GetStatsTopObservers(ctx context.Context, arg GetStatsTopObserversParams) ([]GetStatsTopObserversRow, error) - // Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. + // Top N senders since the given hour. Each message counts once per hour heard, however many + // of the requested IATAs heard it (IATA-set rollup). GetStatsTopTalkers(ctx context.Context, arg GetStatsTopTalkersParams) ([]GetStatsTopTalkersRow, error) - GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]MvTopNodesByIatum, error) + // Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; + // node_id is NULL once the node row is gone. iata is a representative one. + GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]GetTopNodesRow, error) GetTransportScopeByName(ctx context.Context, name string) (int32, error) GetTransportScopes(ctx context.Context) ([]GetTransportScopesRow, error) // ============================================================ // CHANNEL MESSAGES // ============================================================ - InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (int64, error) + // Read the immutable first-packet scope in the same statement as insertion. + // A later reception's transport code must not give live and historical messages different tags. + // message_count is a lifetime count, bumped only for a new message. A historical (backfilled) + // message queues its observation hours for a re-roll in the same statement, so the two can't diverge. + InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) // ============================================================ // PACKET OBSERVATIONS // ============================================================ - InsertObservation(ctx context.Context, arg InsertObservationParams) (PacketObservation, error) + // Bumps packets.observation_count only for a new row; a duplicate returns the current count. + InsertObservation(ctx context.Context, arg InsertObservationParams) (InsertObservationRow, error) // Inserts a telemetry snapshot for an observer. The reported_at timestamp should // be truncated to the configured resolution before calling to ensure deduplication. InsertObserverTelemetry(ctx context.Context, arg InsertObserverTelemetryParams) error + ListAccounts(ctx context.Context) ([]Account, error) // Returns all messages across all channels with optional time, IATA, scope and cursor filters. // Pass empty string for iata or scope to skip those filters. // Pass cursor=0 to start from the beginning. ListAllChannelMessages(ctx context.Context, arg ListAllChannelMessagesParams) ([]ListAllChannelMessagesRow, error) + ListChannelCatalogues(ctx context.Context) ([]MeshmapperChannelCatalogue, error) // Returns messages for a channel identified by integer ID. // Pass a zero/null timestamp for since to return all messages up to limit. // Pass empty string for iata to skip IATA filtering. @@ -110,20 +190,36 @@ type Querier interface { // Pass empty string for iata or scope to skip those filters. // Pass cursor=0 to start from the beginning. ListChannelMessagesByHash(ctx context.Context, arg ListChannelMessagesByHashParams) ([]ListChannelMessagesByHashRow, error) - // Channels ordered by last seen, optionally filtered by hash and/or IATAs - // (membership via channel_iatas). NULL hash / empty array skip those filters. + // Channels ordered by last seen, optionally filtered by hash and/or IATAs. + // A channel belongs to an IATA when MeshMapper lists it there or config scopes it + // to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters. // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) + // Keep the non-null tuple boundary separate from the legacy optional cursor so + // generic prepared plans can seek directly into the composite ordered index. + ListChannelsAfter(ctx context.Context, arg ListChannelsAfterParams) ([]Channel, error) + // Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours + // wait: their first roll consumes the entry only if the entry predates its snapshot. + ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) + ListHeardIATAs(ctx context.Context) ([]string, error) ListIATAs(ctx context.Context) ([]IataCode, error) - ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]KnownRoute, error) + // Only one branch runs. Keep the IATA range ordered by the composite index: + // generic plans can otherwise prefer scanning the global timestamp index. + // The text equality preserves exact input matching, including trailing spaces. + // Pages by (last_seen to the ms, id) so routes sharing the cursor's millisecond + // aren't skipped; cursor id 0 keeps the plain timestamp cursor. + ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) // Returns messages after the given message ID, ordered oldest first. // Used for WS reconnect backfill. ListMessagesAfterID(ctx context.Context, arg ListMessagesAfterIDParams) ([]ListMessagesAfterIDRow, error) + ListMissingRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) ListNodeObservations(ctx context.Context, arg ListNodeObservationsParams) ([]ListNodeObservationsRow, error) + // Limit the filtered node page before enriching IATA membership and neighbours. ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNodesRow, error) ListObservationsForPacket(ctx context.Context, packetHash []byte) ([]ListObservationsForPacketRow, error) // Returns advert packets (payload_type=4) heard by a specific observer. // Pass cursor=0 to start from the beginning, or the last seen id for pagination. + // Keep missing-origin adverts; the generated key field expects a string, not NULL. ListObserverAdverts(ctx context.Context, arg ListObserverAdvertsParams) ([]ListObserverAdvertsRow, error) // Pass cursor=0 to start from the beginning, or the last seen observer's rownum for pagination. // Note: observers use UUID PKs so we order by last_seen and use a keyset on last_seen+id. @@ -140,40 +236,74 @@ type Querier interface { // to fill a page for a quiet site; walking the site's own observation log is // proportional to the page size instead. Results are ordered by when the // requested sites heard the packet (site-local recency) and the cursor - // follows that ordering. scan_depth is a multiple of the page size to - // absorb per-observer duplicates; if duplication exceeds it across a - // page, pagination ends early (hasMore=false) rather than returning a - // short page, even though deeper matches exist. + // follows that ordering. scan_depth caps how deep each site's observation + // log is walked. A packet repeats once per observer that heard it, so a + // page can collapse to fewer distinct packets than were asked for without + // the site being exhausted. scan_saturated reports whether any site hit + // that cap and scan_floor the oldest heard_at they all cover, so a short + // page can keep paging instead of reading as the end of the data. + // A site that filled scan_depth still has unread history below its floor. + // The newest such floor is the point above which every site is covered. ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) + // Every region with its members, for reconciling imported MeshMapper groups. + ListRegionState(ctx context.Context) ([]ListRegionStateRow, error) // ============================================================ // REGIONS // ============================================================ ListRegions(ctx context.Context) ([]ListRegionsRow, error) + // Leading index equalities and the time/ID boundary bound both custom and generic plans. + // Full-byte equality is required even when the compact digest matches. TRACE path bytes + // carry readings; unclassified legacy observations cannot be safely called ordinary paths. + ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) + ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCatalogue, error) // ============================================================ // TRACES // ============================================================ - // Returns distinct trace tags with summary info, ordered by most recent first. - // IATA membership comes from trace_iatas (joining observations here spilled the - // hash join). Per-tag details filled in only for the returned page. + // Tags, most recent first. Filters match the tag summary (see RecordTrace). ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([]ListTraceTagsRow, error) // Returns GRP_TXT packets (payload_type=5) never successfully decrypted. Used at boot to // retry decryption against the current keystore for packets whose channel key was only added // to the config after they'd already been ingested -- see // internal/ingest.BackfillChannelMessages. ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error) + // Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key. + ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error) + ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) + ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) + // Returns the hour's status, or 'partial' if raw deletion has reached it since registration. + LockRollupHour(ctx context.Context, hour pgtype.Timestamptz) (string, error) + // Hours whose raw rows cleanup already started deleting can never be rolled completely. + // Readers report the new status, so the revision moves too. + MarkPartialRollupHours(ctx context.Context) error + OldestMissingRollupHour(ctx context.Context) (pgtype.Timestamptz, error) + PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) + // Drops imports for IATAs no longer configured, so their manual border returns. + PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) // Delete node_neighbors where the neighbor has departed from node_short_ids // for that IATA, or where its prefix_4 is now ambiguous. ReconfirmNeighbors(ctx context.Context) error - // Delete known_routes where any hop node has departed from node_short_ids for - // that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node). - ReconfirmRoutes(ctx context.Context) error - RefreshHourlyStats(ctx context.Context) error - RefreshPayloadBreakdown(ctx context.Context) error + // Checks one batch of least-recently-reconfirmed routes: deletes those with a departed + // hop node or a hop prefix now matching >1 node in that IATA (length-aware: + // 1/2/3/4-byte hop prefixes check prefix_1/2/3/4; ambiguity set supplied by AmbiguousPrefixes), + // and stamps the survivors. + ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams) (int64, error) + // One hearing of a TRACE packet: trace_iatas refreshes at most hourly and the tag's heard + // window widens (the first hearing replaces the packet's provisional times). Packet counts + // and payloads come from UpsertPacket. + RecordTrace(ctx context.Context, arg RecordTraceParams) error RefreshRadioPresets(ctx context.Context) error - RefreshTopAdvertisers(ctx context.Context) error - RefreshTopNodes(ctx context.Context) error - RefreshTopObservers(ctx context.Context) error - RefreshTopTalkers(ctx context.Context) error + // Adds a 'missing' row for every eligible hour since the earliest retained observation + // (and not before @since), zero-observation hours included. + // No observations yet means nothing to register (GREATEST would ignore the NULL MIN). + RegisterRollupHours(ctx context.Context, since pgtype.Timestamptz) error + // Batch form of ResolveEndpointHashes for a page of packets. Matches the cross + // product of IATAs and hashes; callers pick out the pairs they asked for. + ResolveEndpointHashPairs(ctx context.Context, arg ResolveEndpointHashPairsParams) ([]ResolveEndpointHashPairsRow, error) + // Logical endpoints can be any advertised role, unlike intermediate relay hops. + // Endpoint hashes are always one byte; use the existing (iata, prefix_1) index. + // LIMIT 1 keeps generic plans on a node PK lookup per candidate instead of + // flattening the join into a scan of all nodes. The PK already guarantees one row. + ResolveEndpointHashes(ctx context.Context, arg ResolveEndpointHashesParams) ([]ResolveEndpointHashesRow, error) // ============================================================ // HELPERS // ============================================================ @@ -184,18 +314,58 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error + // Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. + RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error + RollIATAObservations(ctx context.Context, hour pgtype.Timestamptz) error + RollObserverActivity(ctx context.Context, hour pgtype.Timestamptz) error + RollObserverIdentity(ctx context.Context, hour pgtype.Timestamptz) error + RollPacketSets(ctx context.Context, hour pgtype.Timestamptz) error + // category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). + RollPaths(ctx context.Context, hour pgtype.Timestamptz) error + RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error + RollScopeNodes(ctx context.Context, hour pgtype.Timestamptz) error + RollScopeObservers(ctx context.Context, hour pgtype.Timestamptz) error + RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error + // kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. + // rssi=0 AND snr=0 means "not reported". + RollSignal(ctx context.Context, hour pgtype.Timestamptz) error + RollTalkerSets(ctx context.Context, hour pgtype.Timestamptz) error + // Order-independent fingerprint of every family row for the hour. + RollupContentHash(ctx context.Context, hour pgtype.Timestamptz) (string, error) + RollupUnlock(ctx context.Context) error + // NULL payload/etag/checked_at retain the last good list after an error or 304. + SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error + // One statement commits the validated snapshot and its lookup identities together. + // Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data + // after an error or 304. Imported names never replace existing manual metadata. + SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error + // NULL feature/etag/checked_at retain the last good boundary after an error or 304. + SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryParams) error + // NULL payload/etag/fetched_at retain the last good list after an error or 304. + SaveZoneList(ctx context.Context, arg SaveZoneListParams) error // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. - SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]KnownRoute, error) + SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) SetNodeDefaultScope(ctx context.Context, arg SetNodeDefaultScopeParams) error SetNodeMultibytePaths(ctx context.Context, id uuid.UUID) error SetNodeMultibyteTraces(ctx context.Context, id uuid.UUID) error SetPacketDecrypted(ctx context.Context, packetHash []byte) error + SetRollupHourPartial(ctx context.Context, hour pgtype.Timestamptz) error TouchObserverBrokers(ctx context.Context, arg TouchObserverBrokersParams) error // Batched flush of coalesced presence bumps. GREATEST keeps a late flush // from regressing a newer write-through (e.g. a status update). TouchObservers(ctx context.Context, arg TouchObserversParams) error TouchPackets(ctx context.Context, arg TouchPacketsParams) error + // ============================================================ + // HOURLY ANALYTICS ROLLUPS + // An hour [H, H+1h) is eligible once now() >= H + 95 min (ingest clamps heard_at to ±30 min). + // ============================================================ + TryRollupLock(ctx context.Context) (bool, error) + // Records the observer's own OTA-reported region scope, from the "self" + // field of a /neighbors report. Always known (not queried OTA), so this + // unconditionally overwrites, unlike the neighbor-side region_scope. + UpdateObserverRegionScope(ctx context.Context, arg UpdateObserverRegionScopeParams) error UpdateObserverStatus(ctx context.Context, arg UpdateObserverStatusParams) (uuid.UUID, error) // ============================================================ // CHANNELS @@ -217,12 +387,14 @@ type Querier interface { // bbox already computed -- see internal/config/border.go. UpsertIATABorder(ctx context.Context, arg UpsertIATABorderParams) error UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsParams) error + // A hand-written region owns its slug: the WHERE turns a clash into no row. + UpsertImportedRegion(ctx context.Context, arg UpsertImportedRegionParams) (int32, error) // ============================================================ // ROUTES // ============================================================ - // Inserts or updates a known route (all hops resolved to high confidence). - // node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and - // their hash bytes. last_seen is bumped on conflict. + // Route identity is path_key, an md5 of node_ids computed by the caller. + // On conflict, observation_count and last_seen are bumped and hash_prefix follows the + // latest hearing, so evidence matches the hash width the route uses now. UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error // ============================================================ // NODES @@ -238,15 +410,18 @@ type Querier interface { // Records or updates a neighbor relationship between two nodes observed in the same IATA. // node_id is the advertising node, neighbor_id is the first-hop forwarder. // snr is optional; pass NULL when no signal reading is available (the - // common case). On conflict, snr is only overwritten when a new non-null - // value is supplied, so a later no-SNR observation doesn't erase an - // earlier real reading. + // common case). regionScope is optional too; pass NULL whenever the OTA + // scope query for this neighbor didn't succeed (status != "responded"), + // so a failed/timed-out query doesn't erase a previously known scope. + // On conflict, snr and region_scope are only overwritten when a new + // non-null value is supplied. UpsertNodeNeighbor(ctx context.Context, arg UpsertNodeNeighborParams) error UpsertNodeShortID(ctx context.Context, arg UpsertNodeShortIDParams) error // ============================================================ // OBSERVERS // ============================================================ - UpsertObserver(ctx context.Context, publicKey []byte) (Observer, error) + // Empty iata (status/neighbors) leaves last_iata alone; otherwise the newest iata_at wins. + UpsertObserver(ctx context.Context, arg UpsertObserverParams) (Observer, error) // ============================================================ // OBSERVER BROKERS // ============================================================ @@ -255,11 +430,10 @@ type Querier interface { // ============================================================ // PACKETS // ============================================================ + // A new TRACE packet adds itself to its tag summary in the same statement, so the summary + // can't miss a stored packet. 'TRACE' > 'PING'; the longest path keeps the best payload. UpsertPacket(ctx context.Context, arg UpsertPacketParams) (UpsertPacketRow, error) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int32, error) - UpsertRegionIATA(ctx context.Context, arg UpsertRegionIATAParams) error - // Refreshes at most hourly so repeat hears don't churn the row. - UpsertTraceIATA(ctx context.Context, arg UpsertTraceIATAParams) error // ============================================================ // TRANSPORT CODES // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 4156701d..46b13b50 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -7,11 +7,184 @@ package db import ( "context" + "encoding/json" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) +const addChannelConfigScopes = `-- name: AddChannelConfigScopes :exec +INSERT INTO channel_config_scopes (key_fingerprint, region_slug) +SELECT unnest($1::bytea[]), NULLIF(unnest($2::text[]), '') +` + +type AddChannelConfigScopesParams struct { + Fingerprints [][]byte `json:"fingerprints"` + Regions []string `json:"regions"` +} + +// An empty region places the channel Beacon-wide. +func (q *Queries) AddChannelConfigScopes(ctx context.Context, arg AddChannelConfigScopesParams) error { + _, err := q.db.Exec(ctx, addChannelConfigScopes, arg.Fingerprints, arg.Regions) + return err +} + +const addChannelMembers = `-- name: AddChannelMembers :exec +INSERT INTO meshmapper_channel_members (iata, key_fingerprint) +SELECT $1, unnest($2::bytea[]) +ON CONFLICT (iata, key_fingerprint) DO NOTHING +` + +type AddChannelMembersParams struct { + Iata string `json:"iata"` + Fingerprints [][]byte `json:"fingerprints"` +} + +func (q *Queries) AddChannelMembers(ctx context.Context, arg AddChannelMembersParams) error { + _, err := q.db.Exec(ctx, addChannelMembers, arg.Iata, arg.Fingerprints) + return err +} + +const addIATAs = `-- name: AddIATAs :exec +INSERT INTO iata_codes (iata) +SELECT unnest($1::bpchar[]) +ON CONFLICT (iata) DO NOTHING +` + +func (q *Queries) AddIATAs(ctx context.Context, iatas []string) error { + _, err := q.db.Exec(ctx, addIATAs, iatas) + return err +} + +const addRegionIATAs = `-- name: AddRegionIATAs :exec +INSERT INTO region_iatas (region_id, iata) +SELECT $1, unnest($2::bpchar[]) +ON CONFLICT (region_id, iata) DO NOTHING +` + +type AddRegionIATAsParams struct { + RegionID int32 `json:"region_id"` + Iatas []string `json:"iatas"` +} + +func (q *Queries) AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error { + _, err := q.db.Exec(ctx, addRegionIATAs, arg.RegionID, arg.Iatas) + return err +} + +const ambiguousPrefixes = `-- name: AmbiguousPrefixes :many +SELECT iata::text AS iata, 1::int AS len, prefix_1 AS prefix FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 2::int, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 3::int, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 4::int, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1 +` + +type AmbiguousPrefixesRow struct { + Iata string `json:"iata"` + Len int32 `json:"len"` + Prefix []byte `json:"prefix"` +} + +// Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. +func (q *Queries) AmbiguousPrefixes(ctx context.Context) ([]AmbiguousPrefixesRow, error) { + rows, err := q.db.Query(ctx, ambiguousPrefixes) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AmbiguousPrefixesRow{} + for rows.Next() { + var i AmbiguousPrefixesRow + if err := rows.Scan(&i.Iata, &i.Len, &i.Prefix); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const createAccount = `-- name: CreateAccount :one +INSERT INTO accounts (name) VALUES ($1) +ON CONFLICT (name) WHERE deactivated_at IS NULL DO NOTHING +RETURNING id, name, created_at, deactivated_at +` + +func (q *Queries) CreateAccount(ctx context.Context, name string) (Account, error) { + row := q.db.QueryRow(ctx, createAccount, name) + var i Account + err := row.Scan( + &i.ID, + &i.Name, + &i.CreatedAt, + &i.DeactivatedAt, + ) + return i, err +} + +const deactivateAccount = `-- name: DeactivateAccount :one +WITH target AS MATERIALIZED ( + SELECT a.id, a.deactivated_at FROM accounts a WHERE a.id = $1 FOR UPDATE +), changed AS ( + UPDATE accounts a SET deactivated_at = NOW() + FROM target t WHERE a.id = t.id AND t.deactivated_at IS NULL + RETURNING a.id +) +SELECT EXISTS(SELECT 1 FROM target) AS found, + EXISTS(SELECT 1 FROM changed) AS deactivated +` + +type DeactivateAccountRow struct { + Found bool `json:"found"` + Deactivated bool `json:"deactivated"` +} + +// Lock the current row before deciding the outcome, including when another +// deactivation commits while this statement is waiting for its row lock. +func (q *Queries) DeactivateAccount(ctx context.Context, id uuid.UUID) (DeactivateAccountRow, error) { + row := q.db.QueryRow(ctx, deactivateAccount, id) + var i DeactivateAccountRow + err := row.Scan(&i.Found, &i.Deactivated) + return i, err +} + +const deleteAllChannelMembers = `-- name: DeleteAllChannelMembers :exec +DELETE FROM meshmapper_channel_members +` + +func (q *Queries) DeleteAllChannelMembers(ctx context.Context) error { + _, err := q.db.Exec(ctx, deleteAllChannelMembers) + return err +} + +const deleteChannelConfigScopes = `-- name: DeleteChannelConfigScopes :exec +DELETE FROM channel_config_scopes +` + +func (q *Queries) DeleteChannelConfigScopes(ctx context.Context) error { + _, err := q.db.Exec(ctx, deleteChannelConfigScopes) + return err +} + +const deleteChannelMembersNotIn = `-- name: DeleteChannelMembersNotIn :exec +DELETE FROM meshmapper_channel_members WHERE iata = $1 AND NOT (key_fingerprint = ANY($2::bytea[])) +` + +type DeleteChannelMembersNotInParams struct { + Iata string `json:"iata"` + Keep [][]byte `json:"keep"` +} + +func (q *Queries) DeleteChannelMembersNotIn(ctx context.Context, arg DeleteChannelMembersNotInParams) error { + _, err := q.db.Exec(ctx, deleteChannelMembersNotIn, arg.Iata, arg.Keep) + return err +} + const deleteOldChannelIATAs = `-- name: DeleteOldChannelIATAs :exec DELETE FROM channel_iatas WHERE last_heard < $1 ` @@ -40,15 +213,112 @@ func (q *Queries) DeleteOldNodes(ctx context.Context, lastSeen pgtype.Timestampt return err } -const deleteOldPackets = `-- name: DeleteOldPackets :exec -DELETE FROM packets WHERE last_heard_at < $1 +const deleteOldObservers = `-- name: DeleteOldObservers :many +WITH expired AS ( + SELECT o.id + FROM observers o + WHERE o.last_seen < $1 + AND (o.last_status_at IS NULL OR o.last_status_at < $1) + AND NOT EXISTS (SELECT 1 FROM packet_observations po WHERE po.observer_id = o.id) + AND NOT EXISTS (SELECT 1 FROM observer_telemetry ot WHERE ot.observer_id = o.id) + AND NOT EXISTS (SELECT 1 FROM observer_owners oo WHERE oo.observer_id = o.id) + ORDER BY o.last_seen, o.id + LIMIT 1000 + FOR UPDATE OF o SKIP LOCKED +) +DELETE FROM observers o USING expired e +WHERE o.id = e.id +RETURNING o.id ` -// Deletes packets and their observations older than the given cutoff. -// packet_observations cascade-delete via FK. -func (q *Queries) DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { - _, err := q.db.Exec(ctx, deleteOldPackets, lastHeardAt) - return err +// Opt-in age-out: preserve retained history and manually recorded ownership. +// Bound deletions per cleanup tick and skip observers being updated by ingest. +func (q *Queries) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) { + rows, err := q.db.Query(ctx, deleteOldObservers, lastSeen) + if err != nil { + return nil, err + } + defer rows.Close() + items := []uuid.UUID{} + for rows.Next() { + var id uuid.UUID + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const deleteOldPackets = `-- name: DeleteOldPackets :one +WITH victims AS MATERIALIZED ( + SELECT vp.packet_hash, vp.last_heard_at FROM packets vp + WHERE vp.last_heard_at < $1::timestamptz + ORDER BY vp.last_heard_at, vp.packet_hash + LIMIT $2::integer + FOR UPDATE OF vp SKIP LOCKED +), deleted AS ( + DELETE FROM packets dp USING victims v WHERE dp.packet_hash = v.packet_hash + RETURNING dp.last_heard_at +), mark AS ( + UPDATE analytics_raw_state + SET raw_deleted_before = GREATEST(raw_deleted_before, (SELECT max(last_heard_at) FROM deleted)) + WHERE EXISTS (SELECT 1 FROM deleted) +) +SELECT count(*) FROM deleted +` + +type DeleteOldPacketsParams struct { + Cutoff pgtype.Timestamptz `json:"cutoff"` + BatchSize int32 `json:"batch_size"` +} + +// Deletes one bounded cohort (observations cascade). SKIP LOCKED leaves packets an in-flight +// observation insert holds. raw_deleted_before tells the rollup which hours lost raw rows. +func (q *Queries) DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) { + row := q.db.QueryRow(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) + var count int64 + err := row.Scan(&count) + return count, err +} + +const deleteOldRoutes = `-- name: DeleteOldRoutes :execrows +WITH expired AS ( + SELECT r.iata, r.path_key + FROM known_routes r + WHERE r.last_seen < GREATEST($1::timestamptz, $2::timestamptz) + AND (r.last_seen < $1 OR + (r.observation_count < $3 AND r.last_seen < $2)) + LIMIT $4 + FOR UPDATE OF r SKIP LOCKED +) +DELETE FROM known_routes kr USING expired e +WHERE kr.iata = e.iata AND kr.path_key = e.path_key +` + +type DeleteOldRoutesParams struct { + RetentionCutoff pgtype.Timestamptz `json:"retention_cutoff"` + GraceCutoff pgtype.Timestamptz `json:"grace_cutoff"` + MinObservations int64 `json:"min_observations"` + BatchSize int32 `json:"batch_size"` +} + +// One batch of routes past retention, or past grace with too few observations. +// GREATEST keeps the scan on idx_known_routes_last_seen. +func (q *Queries) DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) (int64, error) { + result, err := q.db.Exec(ctx, deleteOldRoutes, + arg.RetentionCutoff, + arg.GraceCutoff, + arg.MinObservations, + arg.BatchSize, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil } const deleteOldTelemetry = `-- name: DeleteOldTelemetry :exec @@ -71,6 +341,45 @@ func (q *Queries) DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Time return err } +const deleteOldTraceTags = `-- name: DeleteOldTraceTags :exec +DELETE FROM trace_tags WHERE last_heard_at < $1 +` + +func (q *Queries) DeleteOldTraceTags(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, deleteOldTraceTags, lastHeardAt) + return err +} + +const deleteRegionIATAsNotIn = `-- name: DeleteRegionIATAsNotIn :exec +DELETE FROM region_iatas WHERE region_id = $1 AND NOT (iata = ANY($2::bpchar[])) +` + +type DeleteRegionIATAsNotInParams struct { + RegionID int32 `json:"region_id"` + Keep []string `json:"keep"` +} + +func (q *Queries) DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error { + _, err := q.db.Exec(ctx, deleteRegionIATAsNotIn, arg.RegionID, arg.Keep) + return err +} + +const getAccount = `-- name: GetAccount :one +SELECT id, name, created_at, deactivated_at FROM accounts WHERE id = $1 +` + +func (q *Queries) GetAccount(ctx context.Context, id uuid.UUID) (Account, error) { + row := q.db.QueryRow(ctx, getAccount, id) + var i Account + err := row.Scan( + &i.ID, + &i.Name, + &i.CreatedAt, + &i.DeactivatedAt, + ) + return i, err +} + const getChannelByID = `-- name: GetChannelByID :one SELECT id, channel_hash, key_fingerprint, name, hashtag, is_hashtag, is_public, key_known, first_seen, last_seen, message_count FROM channels WHERE id = $1 ` @@ -154,34 +463,38 @@ func (q *Queries) GetCrossIATANeighbors(ctx context.Context, arg GetCrossIATANei } const getHourlyStats = `-- name: GetHourlyStats :many -SELECT iata, hour, observation_count, unique_packets, active_observers -FROM mv_hourly_iata_stats + +SELECT iata, hour, observation_count +FROM analytics_hourly_iata_observations WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND hour >= NOW() - $2::interval + AND hour >= $2::timestamptz ORDER BY iata, hour ` type GetHourlyStatsParams struct { - Column1 []string `json:"column_1"` - Column2 pgtype.Interval `json:"column_2"` + Iatas []string `json:"iatas"` + Since pgtype.Timestamptz `json:"since"` +} + +type GetHourlyStatsRow struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount int64 `json:"observation_count"` } -func (q *Queries) GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]MvHourlyIataStat, error) { - rows, err := q.db.Query(ctx, getHourlyStats, arg.Column1, arg.Column2) +// ============================================================ +// STATS +// ============================================================ +func (q *Queries) GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]GetHourlyStatsRow, error) { + rows, err := q.db.Query(ctx, getHourlyStats, arg.Iatas, arg.Since) if err != nil { return nil, err } defer rows.Close() - items := []MvHourlyIataStat{} + items := []GetHourlyStatsRow{} for rows.Next() { - var i MvHourlyIataStat - if err := rows.Scan( - &i.Iata, - &i.Hour, - &i.ObservationCount, - &i.UniquePackets, - &i.ActiveObservers, - ); err != nil { + var i GetHourlyStatsRow + if err := rows.Scan(&i.Iata, &i.Hour, &i.ObservationCount); err != nil { return nil, err } items = append(items, i) @@ -211,15 +524,18 @@ func (q *Queries) GetIATA(ctx context.Context, iata string) (IataCode, error) { } const getIATABorder = `-- name: GetIATABorder :one -SELECT border FROM iata_codes WHERE iata = $1 +SELECT COALESCE(z.feature, i.border)::jsonb AS border +FROM iata_codes i +LEFT JOIN meshmapper_zone_boundaries z ON z.iata = i.iata +WHERE i.iata = $1 ` -// border is NULL when the IATA exists but has no border configured; a -// missing row (unknown IATA) is sql.ErrNoRows, same not-found distinction -// GetIATA already makes. -func (q *Queries) GetIATABorder(ctx context.Context, iata string) ([]byte, error) { +// An imported MeshMapper boundary overrides the configured one. border is NULL +// when neither exists; a missing row (unknown IATA) is sql.ErrNoRows, same +// not-found distinction GetIATA already makes. +func (q *Queries) GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) { row := q.db.QueryRow(ctx, getIATABorder, iata) - var border []byte + var border json.RawMessage err := row.Scan(&border) return border, err } @@ -237,15 +553,26 @@ type GetKnownRoutesByNodeParams struct { Column2 uuid.UUID `json:"column_2"` } -func (q *Queries) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]KnownRoute, error) { +type GetKnownRoutesByNodeRow struct { + ID int64 `json:"id"` + NodeIds []uuid.UUID `json:"node_ids"` + HashPrefix [][]byte `json:"hash_prefix"` + Iata string `json:"iata"` + HopCount int32 `json:"hop_count"` + FirstSeen pgtype.Timestamptz `json:"first_seen"` + LastSeen pgtype.Timestamptz `json:"last_seen"` + ObservationCount int64 `json:"observation_count"` +} + +func (q *Queries) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error) { rows, err := q.db.Query(ctx, getKnownRoutesByNode, arg.Iata, arg.Column2) if err != nil { return nil, err } defer rows.Close() - items := []KnownRoute{} + items := []GetKnownRoutesByNodeRow{} for rows.Next() { - var i KnownRoute + var i GetKnownRoutesByNodeRow if err := rows.Scan( &i.ID, &i.NodeIds, @@ -445,6 +772,338 @@ func (q *Queries) GetNodesByIDs(ctx context.Context, dollar_1 []uuid.UUID) ([]Ge return items, nil } +const getNodesByPubkeys = `-- name: GetNodesByPubkeys :many +SELECT id, public_key, name, latitude, longitude +FROM nodes +WHERE public_key = ANY($1::bytea[]) +` + +type GetNodesByPubkeysRow struct { + ID uuid.UUID `json:"id"` + PublicKey []byte `json:"public_key"` + Name *string `json:"name"` + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` +} + +func (q *Queries) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]GetNodesByPubkeysRow, error) { + rows, err := q.db.Query(ctx, getNodesByPubkeys, pubkeys) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetNodesByPubkeysRow{} + for rows.Next() { + var i GetNodesByPubkeysRow + if err := rows.Scan( + &i.ID, + &i.PublicKey, + &i.Name, + &i.Latitude, + &i.Longitude, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getObserverActivityHourly = `-- name: GetObserverActivityHourly :many +WITH src AS ( + SELECT a.hour AS t, a.observations, a.airtime_ms, a.airtime_n, a.snr_sum, a.snr_n, a.snr_min, a.rssi_sum, a.rssi_n + FROM analytics_hourly_observer_activity a + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST($4::timestamptz, $5::timestamptz) + UNION ALL + SELECT o.heard_at, 1::bigint, o.airtime_ms, (o.airtime_ms IS NOT NULL)::int::bigint, + s.snr, (s.snr IS NOT NULL)::int::bigint, s.snr, s.rssi::bigint, (s.rssi IS NOT NULL)::int::bigint + FROM packet_observations o CROSS JOIN + LATERAL (SELECT CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.snr END AS snr, + CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.rssi END AS rssi) s + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, $5::timestamptz) AND o.heard_at < $4::timestamptz +) +SELECT + date_bin($3::interval, t, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + SUM(observations)::bigint AS observations, + COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, + SUM(airtime_n)::bigint AS airtime_n, + COALESCE(SUM(snr_sum), 0)::real AS snr_sum, + SUM(snr_n)::bigint AS snr_n, + COALESCE(MIN(snr_min), 0)::real AS snr_min, + COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, + SUM(rssi_n)::bigint AS rssi_n +FROM src +GROUP BY 1 +ORDER BY 1 +` + +type GetObserverActivityHourlyParams struct { + ObserverID uuid.UUID `json:"observer_id"` + Column2 pgtype.Timestamptz `json:"column_2"` + Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` + TailStart pgtype.Timestamptz `json:"tail_start"` +} + +type GetObserverActivityHourlyRow struct { + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin float32 `json:"snr_min"` + RssiSum int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + +// Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. +// Hours from @tail_start on aren't rolled yet and read raw rows. +func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) { + rows, err := q.db.Query(ctx, getObserverActivityHourly, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + arg.TailStart, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetObserverActivityHourlyRow{} + for rows.Next() { + var i GetObserverActivityHourlyRow + if err := rows.Scan( + &i.Bucket, + &i.Observations, + &i.AirtimeMs, + &i.AirtimeN, + &i.SnrSum, + &i.SnrN, + &i.SnrMin, + &i.RssiSum, + &i.RssiN, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many +WITH src AS ( + SELECT a.payload_type, a.observations AS n + FROM analytics_hourly_observer_activity a + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST($3::timestamptz, $4::timestamptz) + UNION ALL + SELECT COALESCE(o.payload_type, -1)::smallint, 1::bigint + FROM packet_observations o + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, $4::timestamptz) AND o.heard_at < $3::timestamptz +) +SELECT payload_type, SUM(n)::bigint AS count +FROM src +GROUP BY payload_type +ORDER BY count DESC +` + +type GetObserverActivityHourlyPayloadTypesParams struct { + ObserverID uuid.UUID `json:"observer_id"` + Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` + TailStart pgtype.Timestamptz `json:"tail_start"` +} + +type GetObserverActivityHourlyPayloadTypesRow struct { + PayloadType int16 `json:"payload_type"` + Count int64 `json:"count"` +} + +// Same rollup/raw-tail split as GetObserverActivityHourly. +func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) { + rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, + arg.ObserverID, + arg.Column2, + arg.Until, + arg.TailStart, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetObserverActivityHourlyPayloadTypesRow{} + for rows.Next() { + var i GetObserverActivityHourlyPayloadTypesRow + if err := rows.Scan(&i.PayloadType, &i.Count); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = $1::uuid + AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly +` + +type GetObserverActivityLiveSummaryParams struct { + ObserverID uuid.UUID `json:"observer_id"` + GeneratedAt pgtype.Timestamptz `json:"generated_at"` + HourStart pgtype.Timestamptz `json:"hour_start"` + HourEnd pgtype.Timestamptz `json:"hour_end"` +} + +type GetObserverActivityLiveSummaryRow struct { + LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"` + LastCompleteHour int64 `json:"last_complete_hour"` +} + +// Two indexed ranges, bounded to one observer; no legacy presence counters. +func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) { + row := q.db.QueryRow(ctx, getObserverActivityLiveSummary, + arg.ObserverID, + arg.GeneratedAt, + arg.HourStart, + arg.HourEnd, + ) + var i GetObserverActivityLiveSummaryRow + err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour) + return i, err +} + +const getObserverActivityRaw = `-- name: GetObserverActivityRaw :many +SELECT + date_bin($3::interval, heard_at, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + COALESCE(AVG(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS snr_avg, + COALESCE(MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS snr_min, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $4::timestamptz +GROUP BY bucket +ORDER BY bucket +` + +type GetObserverActivityRawParams struct { + ObserverID uuid.UUID `json:"observer_id"` + Column2 pgtype.Timestamptz `json:"column_2"` + Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` +} + +type GetObserverActivityRawRow struct { + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrAvg float32 `json:"snr_avg"` + SnrMin float32 `json:"snr_min"` + SnrN int64 `json:"snr_n"` + RssiAvg float32 `json:"rssi_avg"` + RssiN int64 `json:"rssi_n"` +} + +// Sub-hour activity buckets straight off idx_observations_observer; no join to packets. +// Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as +// NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. +func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverActivityRawParams) ([]GetObserverActivityRawRow, error) { + rows, err := q.db.Query(ctx, getObserverActivityRaw, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetObserverActivityRawRow{} + for rows.Next() { + var i GetObserverActivityRawRow + if err := rows.Scan( + &i.Bucket, + &i.Observations, + &i.AirtimeMs, + &i.AirtimeN, + &i.SnrAvg, + &i.SnrMin, + &i.SnrN, + &i.RssiAvg, + &i.RssiN, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getObserverActivityRawPayloadTypes = `-- name: GetObserverActivityRawPayloadTypes :many +SELECT payload_type, COUNT(*)::bigint AS count +FROM packet_observations +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $3::timestamptz +GROUP BY payload_type +ORDER BY count DESC +` + +type GetObserverActivityRawPayloadTypesParams struct { + ObserverID uuid.UUID `json:"observer_id"` + Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` +} + +type GetObserverActivityRawPayloadTypesRow struct { + PayloadType *int16 `json:"payload_type"` + Count int64 `json:"count"` +} + +func (q *Queries) GetObserverActivityRawPayloadTypes(ctx context.Context, arg GetObserverActivityRawPayloadTypesParams) ([]GetObserverActivityRawPayloadTypesRow, error) { + rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetObserverActivityRawPayloadTypesRow{} + for rows.Next() { + var i GetObserverActivityRawPayloadTypesRow + if err := rows.Scan(&i.PayloadType, &i.Count); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getObserverBrokers = `-- name: GetObserverBrokers :many SELECT broker_name, last_seen, last_packet_at FROM observer_brokers @@ -479,7 +1138,7 @@ func (q *Queries) GetObserverBrokers(ctx context.Context, observerID uuid.UUID) } const getObserverByID = `-- name: GetObserverByID :one -SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata FROM observers WHERE id = $1 +SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope, last_iata, last_iata_at FROM observers WHERE id = $1 ` func (q *Queries) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, error) { @@ -506,12 +1165,15 @@ func (q *Queries) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, &i.LastSeen, &i.ObservationCount, &i.Metadata, + &i.RegionScope, + &i.LastIata, + &i.LastIataAt, ) return i, err } const getObserverByPubkey = `-- name: GetObserverByPubkey :one -SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata FROM observers WHERE public_key = $1 +SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope, last_iata, last_iata_at FROM observers WHERE public_key = $1 ` func (q *Queries) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Observer, error) { @@ -538,22 +1200,22 @@ func (q *Queries) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Ob &i.LastSeen, &i.ObservationCount, &i.Metadata, + &i.RegionScope, + &i.LastIata, + &i.LastIataAt, ) return i, err } const getObserverLastIATA = `-- name: GetObserverLastIATA :one -SELECT iata FROM packet_observations -WHERE observer_id = $1 -ORDER BY heard_at DESC -LIMIT 1 +SELECT COALESCE(last_iata, '')::text FROM observers WHERE id = $1 ` -func (q *Queries) GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) { - row := q.db.QueryRow(ctx, getObserverLastIATA, observerID) - var iata string - err := row.Scan(&iata) - return iata, err +func (q *Queries) GetObserverLastIATA(ctx context.Context, id uuid.UUID) (string, error) { + row := q.db.QueryRow(ctx, getObserverLastIATA, id) + var column_1 string + err := row.Scan(&column_1) + return column_1, err } const getObserverRadio = `-- name: GetObserverRadio :one @@ -609,7 +1271,7 @@ func (q *Queries) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ( } const getObserverTelemetry = `-- name: GetObserverTelemetry :many -SELECT id, reported_at, battery_voltage_mv, airtime_tx_pct, airtime_rx_pct, +SELECT id, reported_at, battery_voltage_mv, airtime_tx_secs, airtime_rx_secs, noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors FROM observer_telemetry WHERE observer_id = $1 @@ -630,8 +1292,8 @@ type GetObserverTelemetryRow struct { ID int64 `json:"id"` ReportedAt pgtype.Timestamptz `json:"reported_at"` BatteryVoltageMv *int32 `json:"battery_voltage_mv"` - AirtimeTxPct *float32 `json:"airtime_tx_pct"` - AirtimeRxPct *float32 `json:"airtime_rx_pct"` + AirtimeTxSecs *float32 `json:"airtime_tx_secs"` + AirtimeRxSecs *float32 `json:"airtime_rx_secs"` NoiseFloorDb *float32 `json:"noise_floor_db"` UptimeSeconds *int64 `json:"uptime_seconds"` QueueLength *int32 `json:"queue_length"` @@ -654,86 +1316,15 @@ func (q *Queries) GetObserverTelemetry(ctx context.Context, arg GetObserverTelem for rows.Next() { var i GetObserverTelemetryRow if err := rows.Scan( - &i.ID, - &i.ReportedAt, - &i.BatteryVoltageMv, - &i.AirtimeTxPct, - &i.AirtimeRxPct, - &i.NoiseFloorDb, - &i.UptimeSeconds, - &i.QueueLength, - &i.DebugFlags, - &i.ReceiveErrors, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - -const getObserverTelemetryBucketed = `-- name: GetObserverTelemetryBucketed :many -SELECT - (date_trunc('day', reported_at) + - (EXTRACT(HOUR FROM reported_at)::int / $4::int) * ($4::int * interval '1 hour'))::timestamptz AS bucket, - AVG(battery_voltage_mv)::int AS battery_voltage_mv, - GREATEST(MAX(airtime_tx_pct) - MIN(airtime_tx_pct), 0)::real AS airtime_tx_pct, - GREATEST(MAX(airtime_rx_pct) - MIN(airtime_rx_pct), 0)::real AS airtime_rx_pct, - AVG(noise_floor_db)::real AS noise_floor_db, - MAX(uptime_seconds)::bigint AS uptime_seconds, - AVG(queue_length)::int AS queue_length, - GREATEST(MAX(receive_errors) - MIN(receive_errors), 0)::int AS receive_errors -FROM observer_telemetry -WHERE observer_id = $1 - AND ($2::timestamptz IS NULL OR reported_at >= $2) - AND ($3::timestamptz IS NULL OR reported_at <= $3) -GROUP BY bucket -ORDER BY bucket ASC -` - -type GetObserverTelemetryBucketedParams struct { - ObserverID uuid.UUID `json:"observer_id"` - Column2 pgtype.Timestamptz `json:"column_2"` - Column3 pgtype.Timestamptz `json:"column_3"` - Column4 int32 `json:"column_4"` -} - -type GetObserverTelemetryBucketedRow struct { - Bucket pgtype.Timestamptz `json:"bucket"` - BatteryVoltageMv int32 `json:"battery_voltage_mv"` - AirtimeTxPct float32 `json:"airtime_tx_pct"` - AirtimeRxPct float32 `json:"airtime_rx_pct"` - NoiseFloorDb float32 `json:"noise_floor_db"` - UptimeSeconds int64 `json:"uptime_seconds"` - QueueLength int32 `json:"queue_length"` - ReceiveErrors int32 `json:"receive_errors"` -} - -func (q *Queries) GetObserverTelemetryBucketed(ctx context.Context, arg GetObserverTelemetryBucketedParams) ([]GetObserverTelemetryBucketedRow, error) { - rows, err := q.db.Query(ctx, getObserverTelemetryBucketed, - arg.ObserverID, - arg.Column2, - arg.Column3, - arg.Column4, - ) - if err != nil { - return nil, err - } - defer rows.Close() - items := []GetObserverTelemetryBucketedRow{} - for rows.Next() { - var i GetObserverTelemetryBucketedRow - if err := rows.Scan( - &i.Bucket, + &i.ID, + &i.ReportedAt, &i.BatteryVoltageMv, - &i.AirtimeTxPct, - &i.AirtimeRxPct, + &i.AirtimeTxSecs, + &i.AirtimeRxSecs, &i.NoiseFloorDb, &i.UptimeSeconds, &i.QueueLength, + &i.DebugFlags, &i.ReceiveErrors, ); err != nil { return nil, err @@ -747,7 +1338,7 @@ func (q *Queries) GetObserverTelemetryBucketed(ctx context.Context, arg GetObser } const getPacketByHash = `-- name: GetPacketByHash :one -SELECT p.packet_hash, p.payload_type, p.payload_version, p.route_type, p.transport_codes_present, p.region_code, p.sub_region_code, p.scope_id, p.origin_pubkey, p.raw_payload, p.raw_header, p.parsed_payload, p.decrypted, p.channel_hash, p.trace_tag, p.first_heard_at, p.last_heard_at, ts.name AS scope_name, +SELECT p.packet_hash, p.payload_type, p.payload_version, p.route_type, p.transport_codes_present, p.region_code, p.sub_region_code, p.scope_id, p.origin_pubkey, p.raw_payload, p.raw_header, p.parsed_payload, p.decrypted, p.channel_hash, p.trace_tag, p.first_heard_at, p.last_heard_at, p.observation_count, ts.name AS scope_name, cm.sender_name AS cm_sender_name, cm.content AS cm_content, cm.sent_at AS cm_sent_at @@ -775,6 +1366,7 @@ type GetPacketByHashRow struct { TraceTag []byte `json:"trace_tag"` FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` ScopeName *string `json:"scope_name"` CmSenderName *string `json:"cm_sender_name"` CmContent *string `json:"cm_content"` @@ -802,6 +1394,7 @@ func (q *Queries) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPa &i.TraceTag, &i.FirstHeardAt, &i.LastHeardAt, + &i.ObservationCount, &i.ScopeName, &i.CmSenderName, &i.CmContent, @@ -810,17 +1403,6 @@ func (q *Queries) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPa return i, err } -const getPacketObservationCount = `-- name: GetPacketObservationCount :one -SELECT COUNT(*) FROM packet_observations WHERE packet_hash = $1 -` - -func (q *Queries) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) { - row := q.db.QueryRow(ctx, getPacketObservationCount, packetHash) - var count int64 - err := row.Scan(&count) - return count, err -} - const getPacketsByTraceTag = `-- name: GetPacketsByTraceTag :many SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, p.route_type, @@ -828,7 +1410,14 @@ SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, p.last_heard_at, p.parsed_payload, p.scope_id, - ts.name AS scope_name + ts.name AS scope_name, + ARRAY( + SELECT po.iata + FROM packet_observations po + WHERE po.packet_hash = p.packet_hash + GROUP BY po.iata + ORDER BY MIN(po.heard_at), po.iata + )::bpchar[] AS iatas FROM packets p LEFT JOIN transport_scopes ts ON ts.id = p.scope_id WHERE p.trace_tag = decode($1, 'hex') @@ -843,9 +1432,11 @@ type GetPacketsByTraceTagRow struct { ParsedPayload []byte `json:"parsed_payload"` ScopeID *int32 `json:"scope_id"` ScopeName *string `json:"scope_name"` + Iatas []string `json:"iatas"` } -// Returns all packets for a given trace tag with observations. +// Return distinct observation IATAs in first-heard order for path resolution, +// without fetching full observations separately for every trace packet. func (q *Queries) GetPacketsByTraceTag(ctx context.Context, decode string) ([]GetPacketsByTraceTagRow, error) { rows, err := q.db.Query(ctx, getPacketsByTraceTag, decode) if err != nil { @@ -863,6 +1454,7 @@ func (q *Queries) GetPacketsByTraceTag(ctx context.Context, decode string) ([]Ge &i.ParsedPayload, &i.ScopeID, &i.ScopeName, + &i.Iatas, ); err != nil { return nil, err } @@ -1003,19 +1595,18 @@ func (q *Queries) GetRegionIATAs(ctx context.Context, regionID int32) ([]string, const getScopeByName = `-- name: GetScopeByName :one SELECT ts.name, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count, - array_remove(array_agg(DISTINCT po.iata ORDER BY po.iata), NULL)::text[] AS iatas + COALESCE((SELECT SUM(s.packets) FROM analytics_hourly_scope_sets s WHERE s.scope_id = ts.id), 0)::bigint AS packet_count, + (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id)::bigint AS observer_count, + (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id)::bigint AS node_count, + cardinality(i.iatas)::bigint AS iata_count, + i.iatas FROM transport_scopes ts -LEFT JOIN packets p ON p.scope_id = ts.id -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id +CROSS JOIN LATERAL ( + SELECT COALESCE(array_agg(DISTINCT x.iata::text ORDER BY x.iata::text), '{}')::text[] AS iatas + FROM analytics_hourly_scope_sets s CROSS JOIN LATERAL unnest(s.iatas) AS x(iata) + WHERE s.scope_id = ts.id +) i WHERE ts.name = $1 -GROUP BY ts.name ` type GetScopeByNameRow struct { @@ -1027,6 +1618,8 @@ type GetScopeByNameRow struct { Iatas []string `json:"iatas"` } +// Packets and IATAs cover every retained rollup hour; IATAs are those that heard the +// scope's own packets. Observer and node counts are current memberships. func (q *Queries) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) { row := q.db.QueryRow(ctx, getScopeByName, name) var i GetScopeByNameRow @@ -1041,6 +1634,31 @@ func (q *Queries) GetScopeByName(ctx context.Context, name string) (GetScopeByNa return i, err } +const getScopeCatalogue = `-- name: GetScopeCatalogue :one +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2 +` + +type GetScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` +} + +func (q *Queries) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) { + row := q.db.QueryRow(ctx, getScopeCatalogue, arg.Iata, arg.Url) + var i MeshmapperScopeCatalogue + err := row.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ) + return i, err +} + const getScopeNames = `-- name: GetScopeNames :many SELECT name FROM transport_scopes ORDER BY name ` @@ -1066,15 +1684,42 @@ func (q *Queries) GetScopeNames(ctx context.Context) ([]string, error) { } const getScopeStats = `-- name: GetScopeStats :many +WITH packet_counts AS ( + SELECT s.scope_id, SUM(s.packets)::bigint AS packet_count + FROM analytics_hourly_scope_sets s + WHERE s.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR s.iatas && $2::bpchar[]) + GROUP BY s.scope_id +), observer_counts AS ( + SELECT os.scope_id, COUNT(*)::bigint AS observer_count + FROM observer_scopes os JOIN observers o ON o.id = os.observer_id + WHERE COALESCE(cardinality($2::bpchar[]), 0) = 0 OR o.last_iata = ANY($2::bpchar[]) + GROUP BY os.scope_id +), node_counts AS ( + SELECT n.default_scope_id AS scope_id, COUNT(*)::bigint AS node_count + FROM nodes n + WHERE n.default_scope_id IS NOT NULL + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 + OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) + GROUP BY n.default_scope_id +) SELECT ts.name, - (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count, - (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count, - (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count + COALESCE(pc.packet_count, 0)::bigint AS packet_count, + COALESCE(oc.observer_count, 0)::bigint AS observer_count, + COALESCE(nc.node_count, 0)::bigint AS node_count FROM transport_scopes ts +LEFT JOIN packet_counts pc ON pc.scope_id = ts.id +LEFT JOIN observer_counts oc ON oc.scope_id = ts.id +LEFT JOIN node_counts nc ON nc.scope_id = ts.id ORDER BY ts.name ` +type GetScopeStatsParams struct { + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` +} + type GetScopeStatsRow struct { Name string `json:"name"` PacketCount int64 `json:"packet_count"` @@ -1082,10 +1727,10 @@ type GetScopeStatsRow struct { NodeCount int64 `json:"node_count"` } -// Count each table on its own; the old cross-join blew up to millions of rows -// before COUNT(DISTINCT) (~10s). -func (q *Queries) GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) { - rows, err := q.db.Query(ctx, getScopeStats) +// Packets since the given hour come from the IATA-set rollup (counted once per hour heard). +// Observer and node counts are current memberships; observers filter by their latest IATA. +func (q *Queries) GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([]GetScopeStatsRow, error) { + rows, err := q.db.Query(ctx, getScopeStats, arg.Since, arg.Iatas) if err != nil { return nil, err } @@ -1109,43 +1754,140 @@ func (q *Queries) GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) return items, nil } -const getScopesByIATAs = `-- name: GetScopesByIATAs :many +const getScopeStatsHourly = `-- name: GetScopeStatsHourly :many +WITH pk AS ( + SELECT s.scope_id, s.hour, SUM(s.packets)::bigint AS n + FROM analytics_hourly_scope_sets s + WHERE s.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR s.iatas && $2::bpchar[]) + GROUP BY s.scope_id, s.hour +), ob AS ( + SELECT o.scope_id, o.hour, COUNT(DISTINCT o.observer_id)::bigint AS n + FROM analytics_hourly_scope_observers o + WHERE o.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR o.iata = ANY($2::bpchar[])) + GROUP BY o.scope_id, o.hour +), nd AS ( + SELECT d.scope_id, d.hour, COUNT(DISTINCT d.origin_pubkey)::bigint AS n + FROM analytics_hourly_scope_nodes d + WHERE d.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR d.iata = ANY($2::bpchar[])) + GROUP BY d.scope_id, d.hour +), k AS ( + SELECT scope_id, hour FROM pk UNION SELECT scope_id, hour FROM ob UNION SELECT scope_id, hour FROM nd +) +SELECT ts.name, k.hour, + COALESCE(pk.n, 0)::bigint AS packets, + COALESCE(ob.n, 0)::bigint AS observers, + COALESCE(nd.n, 0)::bigint AS nodes +FROM k +JOIN transport_scopes ts ON ts.id = k.scope_id +LEFT JOIN pk ON pk.scope_id = k.scope_id AND pk.hour = k.hour +LEFT JOIN ob ON ob.scope_id = k.scope_id AND ob.hour = k.hour +LEFT JOIN nd ON nd.scope_id = k.scope_id AND nd.hour = k.hour +WHERE COALESCE(pk.n, 0) > 0 OR COALESCE(ob.n, 0) > 0 OR COALESCE(nd.n, 0) > 0 +ORDER BY ts.name, k.hour +` + +type GetScopeStatsHourlyParams struct { + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` +} + +type GetScopeStatsHourlyRow struct { + Name string `json:"name"` + Hour pgtype.Timestamptz `json:"hour"` + Packets int64 `json:"packets"` + Observers int64 `json:"observers"` + Nodes int64 `json:"nodes"` +} + +// GetScopeStats packet counts split by hour (same window and IATA-set filter), with the +// distinct observers and advertising nodes active in each scope that hour. Empty hours omitted. +func (q *Queries) GetScopeStatsHourly(ctx context.Context, arg GetScopeStatsHourlyParams) ([]GetScopeStatsHourlyRow, error) { + rows, err := q.db.Query(ctx, getScopeStatsHourly, arg.Since, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetScopeStatsHourlyRow{} + for rows.Next() { + var i GetScopeStatsHourlyRow + if err := rows.Scan( + &i.Name, + &i.Hour, + &i.Packets, + &i.Observers, + &i.Nodes, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getStatsClockDrift = `-- name: GetStatsClockDrift :many +WITH page AS ( SELECT - ts.name, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count -FROM transport_scopes ts -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])) -GROUP BY ts.name -ORDER BY ts.name + n.id, + n.name, + n.node_type, + n.device_clock_drift_seconds, + n.last_advert_at +FROM nodes n +WHERE n.node_type IN (2, 3) + AND n.device_clock_drift_seconds IS NOT NULL + AND ABS(n.device_clock_drift_seconds) > $1::int + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) +ORDER BY ABS(n.device_clock_drift_seconds) DESC +LIMIT $3 +) +SELECT n.id, n.name, n.node_type, n.device_clock_drift_seconds, n.last_advert_at, + (SELECT json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) + FROM node_iatas ni WHERE ni.node_id = n.id) AS iatas +FROM page n +ORDER BY ABS(n.device_clock_drift_seconds) DESC ` -type GetScopesByIATAsRow struct { - Name string `json:"name"` - ObserverCount int64 `json:"observer_count"` - NodeCount int64 `json:"node_count"` - IataCount int64 `json:"iata_count"` +type GetStatsClockDriftParams struct { + Column1 int32 `json:"column_1"` + Column2 []string `json:"column_2"` + Limit int32 `json:"limit"` +} + +type GetStatsClockDriftRow struct { + ID uuid.UUID `json:"id"` + Name *string `json:"name"` + NodeType int16 `json:"node_type"` + DeviceClockDriftSeconds *int32 `json:"device_clock_drift_seconds"` + LastAdvertAt pgtype.Timestamptz `json:"last_advert_at"` + Iatas []byte `json:"iatas"` } -func (q *Queries) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) { - rows, err := q.db.Query(ctx, getScopesByIATAs, dollar_1) +// Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds +// the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's +// latest measured drift, not an aggregate over a period. +// Select the filtered page before aggregating its IATA memberships. +func (q *Queries) GetStatsClockDrift(ctx context.Context, arg GetStatsClockDriftParams) ([]GetStatsClockDriftRow, error) { + rows, err := q.db.Query(ctx, getStatsClockDrift, arg.Column1, arg.Column2, arg.Limit) if err != nil { return nil, err } defer rows.Close() - items := []GetScopesByIATAsRow{} + items := []GetStatsClockDriftRow{} for rows.Next() { - var i GetScopesByIATAsRow + var i GetStatsClockDriftRow if err := rows.Scan( + &i.ID, &i.Name, - &i.ObserverCount, - &i.NodeCount, - &i.IataCount, + &i.NodeType, + &i.DeviceClockDriftSeconds, + &i.LastAdvertAt, + &i.Iatas, ); err != nil { return nil, err } @@ -1194,65 +1936,30 @@ func (q *Queries) GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]G return items, nil } -const getStatsOverview = `-- name: GetStatsOverview :one - -SELECT - COUNT(DISTINCT po.packet_hash) AS total_packets, - COUNT(*) AS total_observations, - COUNT(DISTINCT po.observer_id) AS active_observers, - COUNT(DISTINCT po.iata) AS active_iatas -FROM packet_observations po -WHERE po.heard_at > NOW() - INTERVAL '24 hours' - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])) -` - -type GetStatsOverviewRow struct { - TotalPackets int64 `json:"total_packets"` - TotalObservations int64 `json:"total_observations"` - ActiveObservers int64 `json:"active_observers"` - ActiveIatas int64 `json:"active_iatas"` -} - -// ============================================================ -// STATS -// ============================================================ -func (q *Queries) GetStatsOverview(ctx context.Context, dollar_1 []string) (GetStatsOverviewRow, error) { - row := q.db.QueryRow(ctx, getStatsOverview, dollar_1) - var i GetStatsOverviewRow - err := row.Scan( - &i.TotalPackets, - &i.TotalObservations, - &i.ActiveObservers, - &i.ActiveIatas, - ) - return i, err -} - const getStatsPayloadBreakdown = `-- name: GetStatsPayloadBreakdown :many SELECT payload_type, SUM(count)::bigint AS count -FROM mv_payload_breakdown_by_iata +FROM analytics_hourly_payload_breakdown WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND bucket >= NOW() - $2::interval + AND hour >= $2::timestamptz GROUP BY payload_type ORDER BY count DESC ` type GetStatsPayloadBreakdownParams struct { - Column1 []string `json:"column_1"` - Column2 pgtype.Interval `json:"column_2"` + Iatas []string `json:"iatas"` + Since pgtype.Timestamptz `json:"since"` } type GetStatsPayloadBreakdownRow struct { - PayloadType *int16 `json:"payload_type"` - Count int64 `json:"count"` + PayloadType int16 `json:"payload_type"` + Count int64 `json:"count"` } -// Payload-type counts for the IATA within the window, summed from the -// precomputed hourly buckets. +// Payload-type observation counts since the given hour, from the hourly rollup. func (q *Queries) GetStatsPayloadBreakdown(ctx context.Context, arg GetStatsPayloadBreakdownParams) ([]GetStatsPayloadBreakdownRow, error) { - rows, err := q.db.Query(ctx, getStatsPayloadBreakdown, arg.Column1, arg.Column2) + rows, err := q.db.Query(ctx, getStatsPayloadBreakdown, arg.Iatas, arg.Since) if err != nil { return nil, err } @@ -1272,39 +1979,55 @@ func (q *Queries) GetStatsPayloadBreakdown(ctx context.Context, arg GetStatsPayl } const getStatsTopAdvertisers = `-- name: GetStatsTopAdvertisers :many -SELECT - node_id AS id, - name, - node_type, - COALESCE(SUM(advert_count), 0)::bigint AS advert_count, - MAX(last_heard)::timestamptz AS last_heard, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_advertisers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY node_id, name, node_type -ORDER BY advert_count DESC -LIMIT $3 +WITH ranked AS ( + SELECT origin_pubkey, SUM(advert_packets)::bigint AS advert_count, + SUM(flood_packets)::bigint AS flood_advert_count, SUM(direct_packets)::bigint AS direct_advert_count + FROM analytics_hourly_advert_sets + WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iatas && $2::bpchar[]) + GROUP BY origin_pubkey + ORDER BY advert_count DESC, origin_pubkey + LIMIT $3 +), heard AS ( + SELECT h.origin_pubkey, MAX(h.last_heard) AS last_heard, MAX(h.iata) AS iata, + MAX(h.name) AS name, MAX(h.node_type) AS node_type + FROM analytics_hourly_advert_hearings h + WHERE h.hour >= $1::timestamptz AND h.origin_pubkey IN (SELECT origin_pubkey FROM ranked) + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR h.iata = ANY($2::bpchar[])) + GROUP BY h.origin_pubkey +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, h.name, '')::text AS name, COALESCE(n.node_type, h.node_type, 0)::smallint AS node_type, + r.advert_count, r.flood_advert_count, r.direct_advert_count, + h.last_heard::timestamptz AS last_heard, COALESCE(h.iata, '')::bpchar AS iata +FROM ranked r +LEFT JOIN heard h ON h.origin_pubkey = r.origin_pubkey +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.advert_count DESC, r.origin_pubkey ` type GetStatsTopAdvertisersParams struct { - Column1 pgtype.Interval `json:"column_1"` - Column2 []string `json:"column_2"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } type GetStatsTopAdvertisersRow struct { - ID uuid.UUID `json:"id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - AdvertCount int64 `json:"advert_count"` - LastHeard pgtype.Timestamptz `json:"last_heard"` - Iata string `json:"iata"` + NodeID pgtype.UUID `json:"node_id"` + PublicKey string `json:"public_key"` + Name string `json:"name"` + NodeType int16 `json:"node_type"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` + LastHeard pgtype.Timestamptz `json:"last_heard"` + Iata string `json:"iata"` } -// Top N advertisers in the window, summed from the hourly buckets. +// Top N advertisers since the given hour. Each ADVERT packet counts once per hour heard, +// however many of the requested IATAs heard it (IATA-set rollup). Live names win. func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdvertisersParams) ([]GetStatsTopAdvertisersRow, error) { - rows, err := q.db.Query(ctx, getStatsTopAdvertisers, arg.Column1, arg.Column2, arg.Limit) + rows, err := q.db.Query(ctx, getStatsTopAdvertisers, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } @@ -1313,10 +2036,13 @@ func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdv for rows.Next() { var i GetStatsTopAdvertisersRow if err := rows.Scan( - &i.ID, + &i.NodeID, + &i.PublicKey, &i.Name, &i.NodeType, &i.AdvertCount, + &i.FloodAdvertCount, + &i.DirectAdvertCount, &i.LastHeard, &i.Iata, ); err != nil { @@ -1331,38 +2057,42 @@ func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdv } const getStatsTopObservers = `-- name: GetStatsTopObservers :many -SELECT - observer_id AS id, - display_name, - observer_type, - COALESCE(SUM(observation_count), 0)::bigint AS observation_count, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_observers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY observer_id, display_name, observer_type -ORDER BY observation_count DESC -LIMIT $3 +WITH ranked AS ( + SELECT observer_id, SUM(observation_count)::bigint AS observation_count, MAX(iata) AS iata, + MAX(display_name) AS display_name, MAX(observer_type) AS observer_type + FROM analytics_hourly_observer_identity + WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) + GROUP BY observer_id + ORDER BY observation_count DESC, observer_id + LIMIT $3 +) +SELECT r.observer_id AS id, COALESCE(o.display_name, r.display_name, '')::text AS display_name, + COALESCE(o.observer_type, r.observer_type, '')::text AS observer_type, + r.observation_count, r.iata::bpchar AS iata +FROM ranked r +LEFT JOIN observers o ON o.id = r.observer_id +ORDER BY r.observation_count DESC, r.observer_id ` type GetStatsTopObserversParams struct { - Column1 pgtype.Interval `json:"column_1"` - Column2 []string `json:"column_2"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } type GetStatsTopObserversRow struct { ID uuid.UUID `json:"id"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` + DisplayName string `json:"display_name"` + ObserverType string `json:"observer_type"` ObservationCount int64 `json:"observation_count"` Iata string `json:"iata"` } -// Top N observers for the IATA within the window, summed from the precomputed -// hourly buckets. Counts sum across matched IATAs; iata is a representative one. +// Top N observers since the given hour. Counts sum across matched IATAs; iata is a +// representative one. Live names win over the rolled snapshot. func (q *Queries) GetStatsTopObservers(ctx context.Context, arg GetStatsTopObserversParams) ([]GetStatsTopObserversRow, error) { - rows, err := q.db.Query(ctx, getStatsTopObservers, arg.Column1, arg.Column2, arg.Limit) + rows, err := q.db.Query(ctx, getStatsTopObservers, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } @@ -1390,31 +2120,32 @@ func (q *Queries) GetStatsTopObservers(ctx context.Context, arg GetStatsTopObser const getStatsTopTalkers = `-- name: GetStatsTopTalkers :many SELECT sender_name, - COALESCE(SUM(message_count), 0)::bigint AS message_count, + SUM(messages)::bigint AS message_count, MAX(last_sent)::timestamptz AS last_sent -FROM mv_top_talkers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) +FROM analytics_hourly_talker_sets +WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iatas && $2::bpchar[]) GROUP BY sender_name -ORDER BY message_count DESC +ORDER BY message_count DESC, sender_name LIMIT $3 ` type GetStatsTopTalkersParams struct { - Column1 pgtype.Interval `json:"column_1"` - Column2 []string `json:"column_2"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } type GetStatsTopTalkersRow struct { - SenderName *string `json:"sender_name"` + SenderName string `json:"sender_name"` MessageCount int64 `json:"message_count"` LastSent pgtype.Timestamptz `json:"last_sent"` } -// Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. +// Top N senders since the given hour. Each message counts once per hour heard, however many +// of the requested IATAs heard it (IATA-set rollup). func (q *Queries) GetStatsTopTalkers(ctx context.Context, arg GetStatsTopTalkersParams) ([]GetStatsTopTalkersRow, error) { - rows, err := q.db.Query(ctx, getStatsTopTalkers, arg.Column1, arg.Column2, arg.Limit) + rows, err := q.db.Query(ctx, getStatsTopTalkers, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } @@ -1434,31 +2165,57 @@ func (q *Queries) GetStatsTopTalkers(ctx context.Context, arg GetStatsTopTalkers } const getTopNodes = `-- name: GetTopNodes :many -SELECT iata, node_id, name, node_type, observation_count, last_heard FROM mv_top_nodes_by_iata -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) -ORDER BY observation_count DESC -LIMIT $2 +WITH ranked AS ( + SELECT origin_pubkey, SUM(observations)::bigint AS observation_count, MAX(last_heard) AS last_heard, + MAX(iata) AS iata, MAX(name) AS name, MAX(node_type) AS node_type + FROM analytics_hourly_advert_hearings + WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) + GROUP BY origin_pubkey + ORDER BY observation_count DESC, origin_pubkey + LIMIT $3 +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, r.name, '')::text AS name, COALESCE(n.node_type, r.node_type, 0)::smallint AS node_type, + r.iata::bpchar AS iata, r.observation_count, r.last_heard::timestamptz AS last_heard +FROM ranked r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.observation_count DESC, r.origin_pubkey ` type GetTopNodesParams struct { - Column1 []string `json:"column_1"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` +} + +type GetTopNodesRow struct { + NodeID pgtype.UUID `json:"node_id"` + PublicKey string `json:"public_key"` + Name string `json:"name"` + NodeType int16 `json:"node_type"` + Iata string `json:"iata"` + ObservationCount int64 `json:"observation_count"` + LastHeard pgtype.Timestamptz `json:"last_heard"` } -func (q *Queries) GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]MvTopNodesByIatum, error) { - rows, err := q.db.Query(ctx, getTopNodes, arg.Column1, arg.Limit) +// Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; +// node_id is NULL once the node row is gone. iata is a representative one. +func (q *Queries) GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]GetTopNodesRow, error) { + rows, err := q.db.Query(ctx, getTopNodes, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } defer rows.Close() - items := []MvTopNodesByIatum{} + items := []GetTopNodesRow{} for rows.Next() { - var i MvTopNodesByIatum + var i GetTopNodesRow if err := rows.Scan( - &i.Iata, &i.NodeID, + &i.PublicKey, &i.Name, &i.NodeType, + &i.Iata, &i.ObservationCount, &i.LastHeard, ); err != nil { @@ -1484,7 +2241,7 @@ func (q *Queries) GetTransportScopeByName(ctx context.Context, name string) (int } const getTransportScopes = `-- name: GetTransportScopes :many -SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name +SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name ` type GetTransportScopesRow struct { @@ -1515,10 +2272,25 @@ func (q *Queries) GetTransportScopes(ctx context.Context) ([]GetTransportScopesR const insertChannelMessage = `-- name: InsertChannelMessage :one -INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) -VALUES ($1, $2, $3, $4, $5) -ON CONFLICT (packet_hash) DO NOTHING -RETURNING id +WITH inserted AS ( + INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (packet_hash) DO NOTHING + RETURNING id, packet_hash, channel_id +), bump AS ( + UPDATE channels c SET message_count = c.message_count + 1 + FROM inserted WHERE c.id = inserted.channel_id +), dirty AS ( + INSERT INTO analytics_dirty_hours (hour) + SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC') + FROM packet_observations po JOIN inserted i ON i.packet_hash = po.packet_hash + WHERE $6::boolean + ON CONFLICT (hour) DO UPDATE SET enqueued_at = now() +) +SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present +FROM inserted +JOIN packets p ON p.packet_hash = inserted.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id ` type InsertChannelMessageParams struct { @@ -1527,26 +2299,39 @@ type InsertChannelMessageParams struct { SenderName *string `json:"sender_name"` Content *string `json:"content"` SentAt pgtype.Timestamptz `json:"sent_at"` + Column6 bool `json:"column_6"` +} + +type InsertChannelMessageRow struct { + ID int64 `json:"id"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` } // ============================================================ // CHANNEL MESSAGES // ============================================================ -func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (int64, error) { +// Read the immutable first-packet scope in the same statement as insertion. +// A later reception's transport code must not give live and historical messages different tags. +// message_count is a lifetime count, bumped only for a new message. A historical (backfilled) +// message queues its observation hours for a re-roll in the same statement, so the two can't diverge. +func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) { row := q.db.QueryRow(ctx, insertChannelMessage, arg.ChannelID, arg.PacketHash, arg.SenderName, arg.Content, arg.SentAt, + arg.Column6, ) - var id int64 - err := row.Scan(&id) - return id, err + var i InsertChannelMessageRow + err := row.Scan(&i.ID, &i.ScopeName, &i.TransportCodesPresent) + return i, err } const insertObservation = `-- name: InsertObservation :one +WITH ins AS ( INSERT INTO packet_observations ( packet_hash, observer_id, @@ -1564,12 +2349,21 @@ INSERT INTO packet_observations ( bandwidth_khz, coding_rate, source_broker, - payload_type + payload_type, + airtime_ms ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING -RETURNING id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type +RETURNING packet_hash +), bump AS ( + UPDATE packets p SET observation_count = p.observation_count + 1 + FROM ins WHERE p.packet_hash = ins.packet_hash + RETURNING p.observation_count +) +SELECT EXISTS (SELECT 1 FROM ins) AS inserted, + COALESCE((SELECT b.observation_count FROM bump b), + (SELECT pk.observation_count FROM packets pk WHERE pk.packet_hash = $1), 0)::bigint AS observation_count ` type InsertObservationParams struct { @@ -1590,12 +2384,19 @@ type InsertObservationParams struct { CodingRate *int16 `json:"coding_rate"` SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` + AirtimeMs *float32 `json:"airtime_ms"` +} + +type InsertObservationRow struct { + Inserted bool `json:"inserted"` + ObservationCount int64 `json:"observation_count"` } // ============================================================ // PACKET OBSERVATIONS // ============================================================ -func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationParams) (PacketObservation, error) { +// Bumps packets.observation_count only for a new row; a duplicate returns the current count. +func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationParams) (InsertObservationRow, error) { row := q.db.QueryRow(ctx, insertObservation, arg.PacketHash, arg.ObserverID, @@ -1614,35 +2415,17 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa arg.CodingRate, arg.SourceBroker, arg.PayloadType, + arg.AirtimeMs, ) - var i PacketObservation - err := row.Scan( - &i.ID, - &i.PacketHash, - &i.ObserverID, - &i.Iata, - &i.HeardAt, - &i.PathLengthByte, - &i.HashSize, - &i.HopCount, - &i.PathBytes, - &i.Rssi, - &i.Snr, - &i.PropagationTimeMs, - &i.RadioFreqMhz, - &i.SpreadFactor, - &i.BandwidthKhz, - &i.CodingRate, - &i.SourceBroker, - &i.PayloadType, - ) + var i InsertObservationRow + err := row.Scan(&i.Inserted, &i.ObservationCount) return i, err } const insertObserverTelemetry = `-- name: InsertObserverTelemetry :exec INSERT INTO observer_telemetry ( - observer_id, reported_at, battery_voltage_mv, airtime_tx_pct, - airtime_rx_pct, noise_floor_db, uptime_seconds, queue_length, + observer_id, reported_at, battery_voltage_mv, airtime_tx_secs, + airtime_rx_secs, noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (observer_id, reported_at) DO NOTHING @@ -1652,8 +2435,8 @@ type InsertObserverTelemetryParams struct { ObserverID uuid.UUID `json:"observer_id"` ReportedAt pgtype.Timestamptz `json:"reported_at"` BatteryVoltageMv *int32 `json:"battery_voltage_mv"` - AirtimeTxPct *float32 `json:"airtime_tx_pct"` - AirtimeRxPct *float32 `json:"airtime_rx_pct"` + AirtimeTxSecs *float32 `json:"airtime_tx_secs"` + AirtimeRxSecs *float32 `json:"airtime_rx_secs"` NoiseFloorDb *float32 `json:"noise_floor_db"` UptimeSeconds *int64 `json:"uptime_seconds"` QueueLength *int32 `json:"queue_length"` @@ -1668,8 +2451,8 @@ func (q *Queries) InsertObserverTelemetry(ctx context.Context, arg InsertObserve arg.ObserverID, arg.ReportedAt, arg.BatteryVoltageMv, - arg.AirtimeTxPct, - arg.AirtimeRxPct, + arg.AirtimeTxSecs, + arg.AirtimeRxSecs, arg.NoiseFloorDb, arg.UptimeSeconds, arg.QueueLength, @@ -1679,9 +2462,39 @@ func (q *Queries) InsertObserverTelemetry(ctx context.Context, arg InsertObserve return err } +const listAccounts = `-- name: ListAccounts :many +SELECT id, name, created_at, deactivated_at FROM accounts +ORDER BY created_at DESC, id DESC +` + +func (q *Queries) ListAccounts(ctx context.Context) ([]Account, error) { + rows, err := q.db.Query(ctx, listAccounts) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Account{} + for rows.Next() { + var i Account + if err := rows.Scan( + &i.ID, + &i.Name, + &i.CreatedAt, + &i.DeactivatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listAllChannelMessages = `-- name: ListAllChannelMessages :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -1704,16 +2517,18 @@ type ListAllChannelMessagesParams struct { } type ListAllChannelMessagesRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - PacketHashHex string `json:"packet_hash_hex"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + PacketHashHex string `json:"packet_hash_hex"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns all messages across all channels with optional time, IATA, scope and cursor filters. @@ -1744,6 +2559,8 @@ func (q *Queries) ListAllChannelMessages(ctx context.Context, arg ListAllChannel &i.SentAt, &i.PacketHashHex, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -1756,9 +2573,42 @@ func (q *Queries) ListAllChannelMessages(ctx context.Context, arg ListAllChannel return items, nil } +const listChannelCatalogues = `-- name: ListChannelCatalogues :many +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_channel_catalogues ORDER BY iata, attempted_at +` + +func (q *Queries) ListChannelCatalogues(ctx context.Context) ([]MeshmapperChannelCatalogue, error) { + rows, err := q.db.Query(ctx, listChannelCatalogues) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperChannelCatalogue{} + for rows.Next() { + var i MeshmapperChannelCatalogue + if err := rows.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listChannelMessages = `-- name: ListChannelMessages :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -1783,16 +2633,18 @@ type ListChannelMessagesParams struct { } type ListChannelMessagesRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - PacketHashHex string `json:"packet_hash_hex"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + PacketHashHex string `json:"packet_hash_hex"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns messages for a channel identified by integer ID. @@ -1825,6 +2677,8 @@ func (q *Queries) ListChannelMessages(ctx context.Context, arg ListChannelMessag &i.SentAt, &i.PacketHashHex, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -1838,8 +2692,8 @@ func (q *Queries) ListChannelMessages(ctx context.Context, arg ListChannelMessag } const listChannelMessagesByHash = `-- name: ListChannelMessagesByHash :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, c.channel_hash, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, + p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -1864,15 +2718,17 @@ type ListChannelMessagesByHashParams struct { } type ListChannelMessagesByHashRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns messages for all channels matching a hash byte. @@ -1904,6 +2760,8 @@ func (q *Queries) ListChannelMessagesByHash(ctx context.Context, arg ListChannel &i.Content, &i.SentAt, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -1919,30 +2777,103 @@ func (q *Queries) ListChannelMessagesByHash(ctx context.Context, arg ListChannel const listChannels = `-- name: ListChannels :many SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c WHERE ($1::bytea IS NULL OR c.channel_hash = $1) - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci - WHERE ci.iata = ANY($2::bpchar[]) - )) - AND ($3::timestamptz IS NULL OR c.last_seen < $3) -ORDER BY c.last_seen DESC -LIMIT $4 + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY($2::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY($2::bpchar[]))))) + AND ($3::boolean IS NULL OR COALESCE(c.key_known, false) = $3) + AND ($4::timestamptz IS NULL OR c.last_seen < $4) +ORDER BY c.last_seen DESC, c.id DESC +LIMIT $5 +` + +type ListChannelsParams struct { + ChannelHash []byte `json:"channel_hash"` + Iatas []string `json:"iatas"` + KeyKnown *bool `json:"key_known"` + CursorTs pgtype.Timestamptz `json:"cursor_ts"` + PageLimit int32 `json:"page_limit"` +} + +// Channels ordered by last seen, optionally filtered by hash and/or IATAs. +// A channel belongs to an IATA when MeshMapper lists it there or config scopes it +// to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters. +// Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). +func (q *Queries) ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) { + rows, err := q.db.Query(ctx, listChannels, + arg.ChannelHash, + arg.Iatas, + arg.KeyKnown, + arg.CursorTs, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Channel{} + for rows.Next() { + var i Channel + if err := rows.Scan( + &i.ID, + &i.ChannelHash, + &i.KeyFingerprint, + &i.Name, + &i.Hashtag, + &i.IsHashtag, + &i.IsPublic, + &i.KeyKnown, + &i.FirstSeen, + &i.LastSeen, + &i.MessageCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listChannelsAfter = `-- name: ListChannelsAfter :many +SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c +WHERE (c.last_seen, c.id) < ($1::timestamptz, $2::integer) + AND ($3::bytea IS NULL OR c.channel_hash = $3) + AND (COALESCE(cardinality($4::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY($4::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY($4::bpchar[]))))) + AND ($5::boolean IS NULL OR COALESCE(c.key_known, false) = $5) +ORDER BY c.last_seen DESC, c.id DESC +LIMIT $6 ` -type ListChannelsParams struct { +type ListChannelsAfterParams struct { + CursorTs pgtype.Timestamptz `json:"cursor_ts"` + CursorID int32 `json:"cursor_id"` ChannelHash []byte `json:"channel_hash"` Iatas []string `json:"iatas"` - CursorTs pgtype.Timestamptz `json:"cursor_ts"` + KeyKnown *bool `json:"key_known"` PageLimit int32 `json:"page_limit"` } -// Channels ordered by last seen, optionally filtered by hash and/or IATAs -// (membership via channel_iatas). NULL hash / empty array skip those filters. -// Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). -func (q *Queries) ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) { - rows, err := q.db.Query(ctx, listChannels, +// Keep the non-null tuple boundary separate from the legacy optional cursor so +// generic prepared plans can seek directly into the composite ordered index. +func (q *Queries) ListChannelsAfter(ctx context.Context, arg ListChannelsAfterParams) ([]Channel, error) { + rows, err := q.db.Query(ctx, listChannelsAfter, + arg.CursorTs, + arg.CursorID, arg.ChannelHash, arg.Iatas, - arg.CursorTs, + arg.KeyKnown, arg.PageLimit, ) if err != nil { @@ -1975,6 +2906,30 @@ func (q *Queries) ListChannels(ctx context.Context, arg ListChannelsParams) ([]C return items, nil } +const listHeardIATAs = `-- name: ListHeardIATAs :many +SELECT DISTINCT last_iata::text AS iata FROM observers WHERE last_iata IS NOT NULL ORDER BY 1 +` + +func (q *Queries) ListHeardIATAs(ctx context.Context) ([]string, error) { + rows, err := q.db.Query(ctx, listHeardIATAs) + if err != nil { + return nil, err + } + defer rows.Close() + items := []string{} + for rows.Next() { + var iata string + if err := rows.Scan(&iata); err != nil { + return nil, err + } + items = append(items, iata) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listIATAs = `-- name: ListIATAs :many SELECT iata, display_name, approx_lat, approx_lng, added_at, border FROM iata_codes ORDER BY iata ` @@ -2007,12 +2962,32 @@ func (q *Queries) ListIATAs(ctx context.Context) ([]IataCode, error) { } const listKnownRoutes = `-- name: ListKnownRoutes :many +SELECT r.id, r.node_ids, r.hash_prefix, r.iata, r.hop_count, r.first_seen, r.last_seen, r.observation_count +FROM (( SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes -WHERE ($1 = '' OR iata = $1) +WHERE $1 = '' AND ($2 = 0 OR hop_count = $2) - AND ($3::timestamptz IS NULL OR last_seen < $3) -ORDER BY last_seen DESC + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC +LIMIT $4 +) +UNION ALL +( +SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count +FROM known_routes +WHERE $1 <> '' AND iata::text = $1 + AND iata >= $1::bpchar AND iata <= $1::bpchar + AND ($2 = 0 OR hop_count = $2) + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC +LIMIT $4 +)) r +ORDER BY date_trunc('milliseconds', r.last_seen, 'UTC') DESC, r.id DESC LIMIT $4 ` @@ -2021,22 +2996,40 @@ type ListKnownRoutesParams struct { Column2 interface{} `json:"column_2"` Column3 pgtype.Timestamptz `json:"column_3"` Limit int32 `json:"limit"` + Column5 int64 `json:"column_5"` +} + +type ListKnownRoutesRow struct { + ID int64 `json:"id"` + NodeIds []uuid.UUID `json:"node_ids"` + HashPrefix [][]byte `json:"hash_prefix"` + Iata string `json:"iata"` + HopCount int32 `json:"hop_count"` + FirstSeen pgtype.Timestamptz `json:"first_seen"` + LastSeen pgtype.Timestamptz `json:"last_seen"` + ObservationCount int64 `json:"observation_count"` } -func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]KnownRoute, error) { +// Only one branch runs. Keep the IATA range ordered by the composite index: +// generic plans can otherwise prefer scanning the global timestamp index. +// The text equality preserves exact input matching, including trailing spaces. +// Pages by (last_seen to the ms, id) so routes sharing the cursor's millisecond +// aren't skipped; cursor id 0 keeps the plain timestamp cursor. +func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) { rows, err := q.db.Query(ctx, listKnownRoutes, arg.Column1, arg.Column2, arg.Column3, arg.Limit, + arg.Column5, ) if err != nil { return nil, err } defer rows.Close() - items := []KnownRoute{} + items := []ListKnownRoutesRow{} for rows.Next() { - var i KnownRoute + var i ListKnownRoutesRow if err := rows.Scan( &i.ID, &i.NodeIds, @@ -2058,8 +3051,8 @@ func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams } const listMessagesAfterID = `-- name: ListMessagesAfterID :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -2080,16 +3073,18 @@ type ListMessagesAfterIDParams struct { } type ListMessagesAfterIDRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - PacketHashHex string `json:"packet_hash_hex"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + PacketHashHex string `json:"packet_hash_hex"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns messages after the given message ID, ordered oldest first. @@ -2118,6 +3113,8 @@ func (q *Queries) ListMessagesAfterID(ctx context.Context, arg ListMessagesAfter &i.SentAt, &i.PacketHashHex, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -2189,20 +3186,10 @@ func (q *Queries) ListNodeObservations(ctx context.Context, arg ListNodeObservat } const listNodes = `-- name: ListNodes :many +WITH page AS ( SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, - n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, - ts.name AS default_scope_name, - json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) FILTER (WHERE ni.iata IS NOT NULL) AS iatas, - EXISTS (SELECT 1 FROM observers o WHERE o.public_key = n.public_key) AS is_observer, - (SELECT o.id FROM observers o WHERE o.public_key = n.public_key LIMIT 1) AS observer_id, - (SELECT COUNT(DISTINCT nn.neighbor_id) FROM node_neighbors nn WHERE nn.node_id = n.id)::bigint AS known_neighbor_count, - -- CASE short-circuits: the array_agg subquery only runs when $10 is true, - -- so requests that don't ask for neighbor IDs don't pay for it. -(CASE WHEN $10::bool THEN - (SELECT COALESCE(array_agg(DISTINCT nn.neighbor_id), '{}'::uuid[]) FROM node_neighbors nn WHERE nn.node_id = n.id) - ELSE NULL END)::uuid[] AS neighbor_ids + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, ts.name AS default_scope_name FROM nodes n -LEFT JOIN node_iatas ni ON ni.node_id = n.id LEFT JOIN transport_scopes ts ON ts.id = n.default_scope_id WHERE ($1 = 0 OR n.node_type = $1) @@ -2222,9 +3209,21 @@ WHERE AND ($7::timestamptz IS NULL OR n.last_seen < $7) AND ($9::text = '' OR ts.name = $9::text) AND ($11::text = '' OR encode(n.public_key, 'hex') ILIKE $11 || '%') -GROUP BY n.id, ts.name ORDER BY n.last_seen DESC LIMIT $8 +) +SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, n.default_scope_name, + (SELECT json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) + FROM node_iatas ni WHERE ni.node_id = n.id) AS iatas, + EXISTS (SELECT 1 FROM observers o WHERE o.public_key = n.public_key) AS is_observer, + (SELECT o.id FROM observers o WHERE o.public_key = n.public_key LIMIT 1) AS observer_id, + (SELECT COUNT(DISTINCT nn.neighbor_id) FROM node_neighbors nn WHERE nn.node_id = n.id)::bigint AS known_neighbor_count, + (CASE WHEN $10::bool THEN + (SELECT COALESCE(array_agg(DISTINCT nn.neighbor_id), '{}'::uuid[]) FROM node_neighbors nn WHERE nn.node_id = n.id) + ELSE NULL END)::uuid[] AS neighbor_ids +FROM page n +ORDER BY n.last_seen DESC ` type ListNodesParams struct { @@ -2260,6 +3259,7 @@ type ListNodesRow struct { NeighborIds []uuid.UUID `json:"neighbor_ids"` } +// Limit the filtered node page before enriching IATA membership and neighbours. func (q *Queries) ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNodesRow, error) { rows, err := q.db.Query(ctx, listNodes, arg.Column1, @@ -2310,7 +3310,7 @@ func (q *Queries) ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNod } const listObservationsForPacket = `-- name: ListObservationsForPacket :many -SELECT po.id, po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, po.path_bytes, po.rssi, po.snr, po.propagation_time_ms, po.radio_freq_mhz, po.spread_factor, po.bandwidth_khz, po.coding_rate, po.source_broker, po.payload_type, o.display_name AS observer_name +SELECT po.id, po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, po.path_bytes, po.rssi, po.snr, po.propagation_time_ms, po.radio_freq_mhz, po.spread_factor, po.bandwidth_khz, po.coding_rate, po.source_broker, po.payload_type, po.airtime_ms, o.display_name AS observer_name FROM packet_observations po LEFT JOIN observers o ON o.id = po.observer_id WHERE po.packet_hash = $1 @@ -2336,6 +3336,7 @@ type ListObservationsForPacketRow struct { CodingRate *int16 `json:"coding_rate"` SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` + AirtimeMs *float32 `json:"airtime_ms"` ObserverName *string `json:"observer_name"` } @@ -2367,6 +3368,7 @@ func (q *Queries) ListObservationsForPacket(ctx context.Context, packetHash []by &i.CodingRate, &i.SourceBroker, &i.PayloadType, + &i.AirtimeMs, &i.ObserverName, ); err != nil { return nil, err @@ -2390,7 +3392,7 @@ SELECT po.snr, po.hop_count, n.name AS node_name, - encode(p.origin_pubkey, 'hex') AS node_public_key + COALESCE(encode(p.origin_pubkey, 'hex'), '')::text AS node_public_key FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash LEFT JOIN nodes n ON n.public_key = p.origin_pubkey @@ -2422,6 +3424,7 @@ type ListObserverAdvertsRow struct { // Returns advert packets (payload_type=4) heard by a specific observer. // Pass cursor=0 to start from the beginning, or the last seen id for pagination. +// Keep missing-origin adverts; the generated key field expects a string, not NULL. func (q *Queries) ListObserverAdverts(ctx context.Context, arg ListObserverAdvertsParams) ([]ListObserverAdvertsRow, error) { rows, err := q.db.Query(ctx, listObserverAdverts, arg.ObserverID, arg.Column2, arg.Limit) if err != nil { @@ -2467,23 +3470,13 @@ COALESCE(CASE WHEN GREATEST(COALESCE(o.last_status_at, o.last_seen), o.last_seen) > NOW() - INTERVAL '5 minutes' THEN 'online' ELSE 'offline' END, 'offline')::text AS status, -COALESCE(( - SELECT po.iata - FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC - LIMIT 1 -), '')::text AS iata +COALESCE(o.last_iata, '')::text AS iata FROM observers o LEFT JOIN observer_brokers ob ON ob.observer_id = o.id LEFT JOIN observer_scopes os ON os.observer_id = o.id LEFT JOIN transport_scopes ts ON ts.id = os.scope_id WHERE - (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ( - SELECT po.iata FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC LIMIT 1 - ) = ANY($1::bpchar[])) + (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR o.last_iata = ANY($1::bpchar[])) AND ($2 = '' OR o.observer_type = $2) AND ($3 = '' OR ob.broker_name = $3) AND ($4 = '' OR CASE @@ -2572,18 +3565,37 @@ const listPackets = `-- name: ListPackets :many SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, p.scope_id, ts.name AS scope_name, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, + -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, + p.raw_payload, + p.origin_pubkey FROM packets p LEFT JOIN LATERAL ( - SELECT observer_id, iata + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -2613,17 +3625,24 @@ type ListPacketsParams struct { } type ListPacketsRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + Summary string `json:"summary"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + RawPayload []byte `json:"raw_payload"` + OriginPubkey []byte `json:"origin_pubkey"` } // Returns packets with the latest observation rolled in for display. @@ -2649,6 +3668,7 @@ func (q *Queries) ListPackets(ctx context.Context, arg ListPacketsParams) ([]Lis if err := rows.Scan( &i.PacketHash, &i.PayloadType, + &i.Summary, &i.RouteType, &i.FirstHeardAt, &i.LastHeardAt, @@ -2658,6 +3678,12 @@ func (q *Queries) ListPackets(ctx context.Context, arg ListPacketsParams) ([]Lis &i.LatestObserverID, &i.LatestObserverName, &i.LatestObserverIata, + &i.LatestObserverPathLengthByte, + &i.LatestObserverHashSize, + &i.LatestObserverHopCount, + &i.LatestObserverPathBytes, + &i.RawPayload, + &i.OriginPubkey, ); err != nil { return nil, err } @@ -2673,13 +3699,31 @@ const listPacketsAfterID = `-- name: ListPacketsAfterID :many SELECT p.packet_hash, p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, + p.raw_payload, + p.origin_pubkey, ts.name AS scope_name FROM packets p JOIN packet_observations po ON po.packet_hash = p.packet_hash @@ -2704,16 +3748,23 @@ type ListPacketsAfterIDParams struct { } type ListPacketsAfterIDRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` - ScopeName *string `json:"scope_name"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + Summary string `json:"summary"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + RawPayload []byte `json:"raw_payload"` + OriginPubkey []byte `json:"origin_pubkey"` + ScopeName *string `json:"scope_name"` } // Returns packets with observations after the given observation ID, ordered oldest first. @@ -2737,6 +3788,7 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID if err := rows.Scan( &i.PacketHash, &i.PayloadType, + &i.Summary, &i.RouteType, &i.FirstHeardAt, &i.LastHeardAt, @@ -2744,6 +3796,12 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID &i.LatestObserverID, &i.LatestObserverName, &i.LatestObserverIata, + &i.LatestObserverPathLengthByte, + &i.LatestObserverHashSize, + &i.LatestObserverHopCount, + &i.LatestObserverPathBytes, + &i.RawPayload, + &i.OriginPubkey, &i.ScopeName, ); err != nil { return nil, err @@ -2757,21 +3815,8 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID } const listPacketsByIATAs = `-- name: ListPacketsByIATAs :many -SELECT - p.packet_hash, - p.payload_type, - p.route_type, - p.first_heard_at, - p.last_heard_at, - p.scope_id, - ts.name AS scope_name, - sh.site_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, - po.observer_id AS latest_observer_id, - o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata -FROM ( - SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at +WITH scanned AS ( + SELECT req.iata AS req_iata, hits.packet_hash, hits.heard_at FROM unnest($1::bpchar[]) AS req(iata) CROSS JOIN LATERAL ( SELECT po3.packet_hash, po3.heard_at @@ -2789,18 +3834,69 @@ FROM ( ORDER BY po3.heard_at DESC LIMIT $8 ) hits - GROUP BY hits.packet_hash +), +saturation AS ( + SELECT + COUNT(*) > 0 AS scan_saturated, + MAX(floor_ts)::timestamptz AS scan_floor + FROM ( + SELECT MIN(heard_at) AS floor_ts + FROM scanned + GROUP BY req_iata + HAVING COUNT(*) >= $8 + ) filled +), +page AS ( + SELECT scanned.packet_hash, MAX(scanned.heard_at)::timestamptz AS site_heard_at + FROM scanned + GROUP BY scanned.packet_hash HAVING ($2::timestamptz IS NULL OR NOT EXISTS ( SELECT 1 FROM packet_observations px - WHERE px.packet_hash = hits.packet_hash + WHERE px.packet_hash = scanned.packet_hash AND px.iata = ANY($1::bpchar[]) AND px.heard_at >= $2)) ORDER BY site_heard_at DESC LIMIT $9 -) sh +) +SELECT + p.packet_hash, + p.payload_type, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, + p.route_type, + p.first_heard_at, + p.last_heard_at, + p.scope_id, + ts.name AS scope_name, + sh.site_heard_at, + sat.scan_saturated, + sat.scan_floor, + p.observation_count AS observation_count, + -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. + po.observer_id AS latest_observer_id, + o.display_name AS latest_observer_name, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, + p.raw_payload, + p.origin_pubkey +FROM page sh +CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( - SELECT observer_id, iata + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -2824,18 +3920,27 @@ type ListPacketsByIATAsParams struct { } type ListPacketsByIATAsRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + Summary string `json:"summary"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` + ScanSaturated bool `json:"scan_saturated"` + ScanFloor pgtype.Timestamptz `json:"scan_floor"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + RawPayload []byte `json:"raw_payload"` + OriginPubkey []byte `json:"origin_pubkey"` } // IATA-filtered packet list, driven from idx_observations_iata_heard. @@ -2843,10 +3948,14 @@ type ListPacketsByIATAsRow struct { // to fill a page for a quiet site; walking the site's own observation log is // proportional to the page size instead. Results are ordered by when the // requested sites heard the packet (site-local recency) and the cursor -// follows that ordering. scan_depth is a multiple of the page size to -// absorb per-observer duplicates; if duplication exceeds it across a -// page, pagination ends early (hasMore=false) rather than returning a -// short page, even though deeper matches exist. +// follows that ordering. scan_depth caps how deep each site's observation +// log is walked. A packet repeats once per observer that heard it, so a +// page can collapse to fewer distinct packets than were asked for without +// the site being exhausted. scan_saturated reports whether any site hit +// that cap and scan_floor the oldest heard_at they all cover, so a short +// page can keep paging instead of reading as the end of the data. +// A site that filled scan_depth still has unread history below its floor. +// The newest such floor is the point above which every site is covered. func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) { rows, err := q.db.Query(ctx, listPacketsByIATAs, arg.Iatas, @@ -2869,16 +3978,73 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs if err := rows.Scan( &i.PacketHash, &i.PayloadType, + &i.Summary, &i.RouteType, &i.FirstHeardAt, &i.LastHeardAt, &i.ScopeID, &i.ScopeName, &i.SiteHeardAt, + &i.ScanSaturated, + &i.ScanFloor, &i.ObservationCount, &i.LatestObserverID, &i.LatestObserverName, &i.LatestObserverIata, + &i.LatestObserverPathLengthByte, + &i.LatestObserverHashSize, + &i.LatestObserverHopCount, + &i.LatestObserverPathBytes, + &i.RawPayload, + &i.OriginPubkey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listRegionState = `-- name: ListRegionState :many +SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, r.center_lat, r.center_lng, + COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas +FROM regions r +LEFT JOIN region_iatas ri ON ri.region_id = r.id +GROUP BY r.id +ORDER BY r.slug +` + +type ListRegionStateRow struct { + Slug string `json:"slug"` + Name string `json:"name"` + DisplayOrder int32 `json:"display_order"` + Imported bool `json:"imported"` + CenterLat *float64 `json:"center_lat"` + CenterLng *float64 `json:"center_lng"` + Iatas []string `json:"iatas"` +} + +// Every region with its members, for reconciling imported MeshMapper groups. +func (q *Queries) ListRegionState(ctx context.Context) ([]ListRegionStateRow, error) { + rows, err := q.db.Query(ctx, listRegionState) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListRegionStateRow{} + for rows.Next() { + var i ListRegionStateRow + if err := rows.Scan( + &i.Slug, + &i.Name, + &i.DisplayOrder, + &i.Imported, + &i.CenterLat, + &i.CenterLng, + &i.Iatas, ); err != nil { return nil, err } @@ -2926,45 +4092,65 @@ func (q *Queries) ListRegions(ctx context.Context) ([]ListRegionsRow, error) { return items, nil } +const listScopeCatalogues = `-- name: ListScopeCatalogues :many +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues ORDER BY iata, attempted_at +` + +func (q *Queries) ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCatalogue, error) { + rows, err := q.db.Query(ctx, listScopeCatalogues) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperScopeCatalogue{} + for rows.Next() { + var i MeshmapperScopeCatalogue + if err := rows.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listTraceTags = `-- name: ListTraceTags :many -WITH tags AS ( - SELECT - p.trace_tag, - MIN(p.first_heard_at) AS first_heard_at, - MAX(p.last_heard_at) AS last_heard_at, - COUNT(*) AS packet_count, - MAX(p.parsed_payload->>'type') AS trace_type - FROM packets p - WHERE p.trace_tag IS NOT NULL - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR p.trace_tag IN ( - SELECT ti.trace_tag FROM trace_iatas ti WHERE ti.iata = ANY($1::bpchar[]))) - AND ($2::text = '' OR p.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) - AND ($3::timestamptz IS NULL OR p.first_heard_at >= $3) - AND ($4::timestamptz IS NULL OR p.first_heard_at <= $4) - AND ($5::timestamptz IS NULL OR p.last_heard_at < $5) - AND ($7::text = '' OR p.parsed_payload->>'type' = $7) - GROUP BY p.trace_tag - ORDER BY MAX(p.last_heard_at) DESC - LIMIT $6 -) SELECT encode(t.trace_tag, 'hex') AS trace_tag, - t.first_heard_at::timestamptz AS first_heard_at, - t.last_heard_at::timestamptz AS last_heard_at, + t.first_heard_at, + t.last_heard_at, t.packet_count, (SELECT COUNT(*) FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, - t.trace_type::text AS trace_type, - (SELECT p3.parsed_payload - FROM packets p3 - WHERE p3.trace_tag = t.trace_tag - ORDER BY jsonb_array_length(p3.parsed_payload->'pathHashes') DESC - LIMIT 1) AS best_payload -FROM tags t -ORDER BY t.last_heard_at DESC + COALESCE(t.trace_type, '')::text AS trace_type, + t.best_payload +FROM trace_tags t +WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( + SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) + AND ($2::text = '' OR t.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) + AND ($3::timestamptz IS NULL OR t.first_heard_at >= $3) + AND ($4::timestamptz IS NULL OR t.first_heard_at <= $4) + -- Keyset on the millisecond clients see, tie-broken by tag; matches idx_trace_tags_keyset. + AND ($5::timestamptz IS NULL + OR date_trunc('milliseconds', t.last_heard_at, 'UTC') < $5 + OR ($8::bytea IS NOT NULL AND date_trunc('milliseconds', t.last_heard_at, 'UTC') = $5 AND t.trace_tag < $8)) + AND ($7::text = '' OR t.trace_type = $7) +ORDER BY date_trunc('milliseconds', t.last_heard_at, 'UTC') DESC, t.trace_tag DESC +LIMIT $6 ` type ListTraceTagsParams struct { @@ -2975,6 +4161,7 @@ type ListTraceTagsParams struct { Column5 pgtype.Timestamptz `json:"column_5"` Limit int32 `json:"limit"` Column7 string `json:"column_7"` + Column8 []byte `json:"column_8"` } type ListTraceTagsRow struct { @@ -2987,37 +4174,166 @@ type ListTraceTagsRow struct { BestPayload []byte `json:"best_payload"` } -// ============================================================ -// TRACES -// ============================================================ -// Returns distinct trace tags with summary info, ordered by most recent first. -// IATA membership comes from trace_iatas (joining observations here spilled the -// hash join). Per-tag details filled in only for the returned page. -func (q *Queries) ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([]ListTraceTagsRow, error) { - rows, err := q.db.Query(ctx, listTraceTags, - arg.Column1, - arg.Column2, - arg.Column3, - arg.Column4, - arg.Column5, - arg.Limit, - arg.Column7, - ) +// ============================================================ +// TRACES +// ============================================================ +// Tags, most recent first. Filters match the tag summary (see RecordTrace). +func (q *Queries) ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([]ListTraceTagsRow, error) { + rows, err := q.db.Query(ctx, listTraceTags, + arg.Column1, + arg.Column2, + arg.Column3, + arg.Column4, + arg.Column5, + arg.Limit, + arg.Column7, + arg.Column8, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListTraceTagsRow{} + for rows.Next() { + var i ListTraceTagsRow + if err := rows.Scan( + &i.TraceTag, + &i.FirstHeardAt, + &i.LastHeardAt, + &i.PacketCount, + &i.IataCount, + &i.TraceType, + &i.BestPayload, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listUndecryptedGroupTextPackets = `-- name: ListUndecryptedGroupTextPackets :many +SELECT packet_hash, raw_payload FROM packets +WHERE payload_type = 5 AND decrypted IS NOT TRUE +` + +type ListUndecryptedGroupTextPacketsRow struct { + PacketHash []byte `json:"packet_hash"` + RawPayload []byte `json:"raw_payload"` +} + +// Returns GRP_TXT packets (payload_type=5) never successfully decrypted. Used at boot to +// retry decryption against the current keystore for packets whose channel key was only added +// to the config after they'd already been ingested -- see +// internal/ingest.BackfillChannelMessages. +func (q *Queries) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error) { + rows, err := q.db.Query(ctx, listUndecryptedGroupTextPackets) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListUndecryptedGroupTextPacketsRow{} + for rows.Next() { + var i ListUndecryptedGroupTextPacketsRow + if err := rows.Scan(&i.PacketHash, &i.RawPayload); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listUndecryptedGroupTextPacketsByHash = `-- name: ListUndecryptedGroupTextPacketsByHash :many +SELECT packet_hash, raw_payload FROM packets +WHERE payload_type = 5 AND decrypted IS NOT TRUE AND channel_hash = ANY($1::bytea[]) +` + +type ListUndecryptedGroupTextPacketsByHashRow struct { + PacketHash []byte `json:"packet_hash"` + RawPayload []byte `json:"raw_payload"` +} + +// Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key. +func (q *Queries) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error) { + rows, err := q.db.Query(ctx, listUndecryptedGroupTextPacketsByHash, hashes) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListUndecryptedGroupTextPacketsByHashRow{} + for rows.Next() { + var i ListUndecryptedGroupTextPacketsByHashRow + if err := rows.Scan(&i.PacketHash, &i.RawPayload); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listZoneBoundaries = `-- name: ListZoneBoundaries :many +SELECT iata, url, feature, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_boundaries ORDER BY iata +` + +func (q *Queries) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) { + rows, err := q.db.Query(ctx, listZoneBoundaries) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperZoneBoundary{} + for rows.Next() { + var i MeshmapperZoneBoundary + if err := rows.Scan( + &i.Iata, + &i.Url, + &i.Feature, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listZoneLists = `-- name: ListZoneLists :many +SELECT country, payload, etag, fetched_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_lists ORDER BY country +` + +func (q *Queries) ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) { + rows, err := q.db.Query(ctx, listZoneLists) if err != nil { return nil, err } defer rows.Close() - items := []ListTraceTagsRow{} + items := []MeshmapperZoneList{} for rows.Next() { - var i ListTraceTagsRow + var i MeshmapperZoneList if err := rows.Scan( - &i.TraceTag, - &i.FirstHeardAt, - &i.LastHeardAt, - &i.PacketCount, - &i.IataCount, - &i.TraceType, - &i.BestPayload, + &i.Country, + &i.Payload, + &i.Etag, + &i.FetchedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, ); err != nil { return nil, err } @@ -3029,33 +4345,48 @@ func (q *Queries) ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([ return items, nil } -const listUndecryptedGroupTextPackets = `-- name: ListUndecryptedGroupTextPackets :many -SELECT packet_hash, raw_payload FROM packets -WHERE payload_type = 5 AND decrypted IS NOT TRUE +const pruneImportedRegions = `-- name: PruneImportedRegions :many +DELETE FROM regions WHERE imported AND NOT (slug = ANY($1::text[])) RETURNING slug ` -type ListUndecryptedGroupTextPacketsRow struct { - PacketHash []byte `json:"packet_hash"` - RawPayload []byte `json:"raw_payload"` +func (q *Queries) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { + rows, err := q.db.Query(ctx, pruneImportedRegions, keep) + if err != nil { + return nil, err + } + defer rows.Close() + items := []string{} + for rows.Next() { + var slug string + if err := rows.Scan(&slug); err != nil { + return nil, err + } + items = append(items, slug) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil } -// Returns GRP_TXT packets (payload_type=5) never successfully decrypted. Used at boot to -// retry decryption against the current keystore for packets whose channel key was only added -// to the config after they'd already been ingested -- see -// internal/ingest.BackfillChannelMessages. -func (q *Queries) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error) { - rows, err := q.db.Query(ctx, listUndecryptedGroupTextPackets) +const pruneZoneBoundaries = `-- name: PruneZoneBoundaries :many +DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY($1::text[])) RETURNING iata +` + +// Drops imports for IATAs no longer configured, so their manual border returns. +func (q *Queries) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { + rows, err := q.db.Query(ctx, pruneZoneBoundaries, keep) if err != nil { return nil, err } defer rows.Close() - items := []ListUndecryptedGroupTextPacketsRow{} + items := []string{} for rows.Next() { - var i ListUndecryptedGroupTextPacketsRow - if err := rows.Scan(&i.PacketHash, &i.RawPayload); err != nil { + var iata string + if err := rows.Scan(&iata); err != nil { return nil, err } - items = append(items, i) + items = append(items, iata) } if err := rows.Err(); err != nil { return nil, err @@ -3088,50 +4419,109 @@ func (q *Queries) ReconfirmNeighbors(ctx context.Context) error { return err } -const reconfirmRoutes = `-- name: ReconfirmRoutes :exec -DELETE FROM known_routes kr -WHERE EXISTS ( - SELECT 1 - FROM unnest(kr.node_ids) AS hop_node_id - WHERE NOT EXISTS ( - SELECT 1 FROM node_short_ids ns - WHERE ns.node_id = hop_node_id - AND ns.iata = kr.iata +const reconfirmRoutes = `-- name: ReconfirmRoutes :one +WITH batch AS MATERIALIZED ( + SELECT r.iata, r.path_key, r.node_ids, r.hash_prefix + FROM known_routes r + WHERE r.last_reconfirmed_at < $1 + ORDER BY r.last_reconfirmed_at + LIMIT $2 + FOR UPDATE OF r SKIP LOCKED +), +amb AS ( + SELECT a.iata::char(3) AS iata, a.len, a.p + FROM ROWS FROM (unnest($3::text[]), unnest($4::int[]), unnest($5::bytea[])) AS a(iata, len, p) +), +dead AS ( + SELECT b.iata, b.path_key + FROM batch b + WHERE EXISTS ( + SELECT 1 + FROM unnest(b.node_ids) AS hop_node_id + WHERE NOT EXISTS ( + SELECT 1 FROM node_short_ids ns + WHERE ns.node_id = hop_node_id + AND ns.iata = b.iata + ) ) + UNION + SELECT DISTINCT b.iata, b.path_key + FROM batch b + CROSS JOIN LATERAL unnest(b.hash_prefix) AS hp + JOIN amb a ON a.iata = b.iata AND a.len = length(hp) AND a.p = hp +), +deleted AS ( + DELETE FROM known_routes kr + USING dead d + WHERE kr.iata = d.iata AND kr.path_key = d.path_key +), +updated AS ( + UPDATE known_routes kr + SET last_reconfirmed_at = GREATEST(NOW(), $1::timestamptz) + FROM batch b + WHERE kr.iata = b.iata AND kr.path_key = b.path_key + AND NOT EXISTS ( + SELECT 1 FROM dead d + WHERE d.iata = b.iata AND d.path_key = b.path_key + ) ) -OR EXISTS ( - SELECT 1 - FROM unnest(kr.hash_prefix) AS hop_prefix - WHERE ( - SELECT COUNT(*) FROM node_short_ids ns - WHERE ns.iata = kr.iata - AND ns.prefix_4 = hop_prefix - ) > 1 -) +SELECT count(*) FROM batch ` -// Delete known_routes where any hop node has departed from node_short_ids for -// that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node). -func (q *Queries) ReconfirmRoutes(ctx context.Context) error { - _, err := q.db.Exec(ctx, reconfirmRoutes) - return err +type ReconfirmRoutesParams struct { + Before pgtype.Timestamptz `json:"before"` + BatchSize int32 `json:"batch_size"` + AmbIata []string `json:"amb_iata"` + AmbLen []int32 `json:"amb_len"` + AmbPrefix [][]byte `json:"amb_prefix"` +} + +// Checks one batch of least-recently-reconfirmed routes: deletes those with a departed +// hop node or a hop prefix now matching >1 node in that IATA (length-aware: +// 1/2/3/4-byte hop prefixes check prefix_1/2/3/4; ambiguity set supplied by AmbiguousPrefixes), +// and stamps the survivors. +func (q *Queries) ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams) (int64, error) { + row := q.db.QueryRow(ctx, reconfirmRoutes, + arg.Before, + arg.BatchSize, + arg.AmbIata, + arg.AmbLen, + arg.AmbPrefix, + ) + var count int64 + err := row.Scan(&count) + return count, err } -const refreshHourlyStats = `-- name: RefreshHourlyStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hourly_iata_stats +const recordTrace = `-- name: RecordTrace :exec +WITH iata AS ( + INSERT INTO trace_iatas (trace_tag, iata, last_heard) + VALUES ($1, $3, $2) + ON CONFLICT (trace_tag, iata) DO UPDATE SET + last_heard = EXCLUDED.last_heard + WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour' +) +INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, heard) +VALUES ($1, $2, $2, true) +ON CONFLICT (trace_tag) DO UPDATE SET + first_heard_at = CASE WHEN trace_tags.heard THEN LEAST(trace_tags.first_heard_at, EXCLUDED.first_heard_at) + ELSE EXCLUDED.first_heard_at END, + last_heard_at = CASE WHEN trace_tags.heard THEN GREATEST(trace_tags.last_heard_at, EXCLUDED.last_heard_at) + ELSE EXCLUDED.last_heard_at END, + heard = true ` -func (q *Queries) RefreshHourlyStats(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshHourlyStats) - return err +type RecordTraceParams struct { + TraceTag []byte `json:"trace_tag"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + Iata string `json:"iata"` } -const refreshPayloadBreakdown = `-- name: RefreshPayloadBreakdown :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_payload_breakdown_by_iata -` - -func (q *Queries) RefreshPayloadBreakdown(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshPayloadBreakdown) +// One hearing of a TRACE packet: trace_iatas refreshes at most hourly and the tag's heard +// window widens (the first hearing replaces the packet's provisional times). Packet counts +// and payloads come from UpsertPacket. +func (q *Queries) RecordTrace(ctx context.Context, arg RecordTraceParams) error { + _, err := q.db.Exec(ctx, recordTrace, arg.TraceTag, arg.HeardAt, arg.Iata) return err } @@ -3144,40 +4534,118 @@ func (q *Queries) RefreshRadioPresets(ctx context.Context) error { return err } -const refreshTopAdvertisers = `-- name: RefreshTopAdvertisers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata +const resolveEndpointHashPairs = `-- name: ResolveEndpointHashPairs :many +SELECT ns.iata, ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM node_short_ids ns +CROSS JOIN LATERAL ( + SELECT id, name, latitude, longitude, public_key + FROM nodes WHERE id = ns.node_id + LIMIT 1 +) n +WHERE ns.iata = ANY($1::bpchar[]) + AND ns.prefix_1 = ANY($2::bytea[]) ` -func (q *Queries) RefreshTopAdvertisers(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopAdvertisers) - return err +type ResolveEndpointHashPairsParams struct { + Iatas []string `json:"iatas"` + Hashes [][]byte `json:"hashes"` } -const refreshTopNodes = `-- name: RefreshTopNodes :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_nodes_by_iata -` +type ResolveEndpointHashPairsRow struct { + Iata string `json:"iata"` + Hash []byte `json:"hash"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` + PublicKey []byte `json:"public_key"` +} -func (q *Queries) RefreshTopNodes(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopNodes) - return err +// Batch form of ResolveEndpointHashes for a page of packets. Matches the cross +// product of IATAs and hashes; callers pick out the pairs they asked for. +func (q *Queries) ResolveEndpointHashPairs(ctx context.Context, arg ResolveEndpointHashPairsParams) ([]ResolveEndpointHashPairsRow, error) { + rows, err := q.db.Query(ctx, resolveEndpointHashPairs, arg.Iatas, arg.Hashes) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ResolveEndpointHashPairsRow{} + for rows.Next() { + var i ResolveEndpointHashPairsRow + if err := rows.Scan( + &i.Iata, + &i.Hash, + &i.NodeID, + &i.Name, + &i.Latitude, + &i.Longitude, + &i.PublicKey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil } -const refreshTopObservers = `-- name: RefreshTopObservers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_observers_by_iata +const resolveEndpointHashes = `-- name: ResolveEndpointHashes :many +SELECT ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM node_short_ids ns +CROSS JOIN LATERAL ( + SELECT id, name, latitude, longitude, public_key + FROM nodes WHERE id = ns.node_id + LIMIT 1 +) n +WHERE ns.iata = $1 + AND ns.prefix_1 = ANY($2::bytea[]) ` -func (q *Queries) RefreshTopObservers(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopObservers) - return err +type ResolveEndpointHashesParams struct { + Iata string `json:"iata"` + Column2 [][]byte `json:"column_2"` } -const refreshTopTalkers = `-- name: RefreshTopTalkers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_talkers_by_iata -` +type ResolveEndpointHashesRow struct { + Hash []byte `json:"hash"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` + PublicKey []byte `json:"public_key"` +} -func (q *Queries) RefreshTopTalkers(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopTalkers) - return err +// Logical endpoints can be any advertised role, unlike intermediate relay hops. +// Endpoint hashes are always one byte; use the existing (iata, prefix_1) index. +// LIMIT 1 keeps generic plans on a node PK lookup per candidate instead of +// flattening the join into a scan of all nodes. The PK already guarantees one row. +func (q *Queries) ResolveEndpointHashes(ctx context.Context, arg ResolveEndpointHashesParams) ([]ResolveEndpointHashesRow, error) { + rows, err := q.db.Query(ctx, resolveEndpointHashes, arg.Iata, arg.Column2) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ResolveEndpointHashesRow{} + for rows.Next() { + var i ResolveEndpointHashesRow + if err := rows.Scan( + &i.Hash, + &i.NodeID, + &i.Name, + &i.Latitude, + &i.Longitude, + &i.PublicKey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil } const resolvePathHashesP1 = `-- name: ResolvePathHashesP1 :many @@ -3388,6 +4856,176 @@ func (q *Queries) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashes return items, nil } +const saveChannelCatalogue = `-- name: SaveChannelCatalogue :exec +INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_channel_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_channel_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_channel_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveChannelCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +// NULL payload/etag/checked_at retain the last good list after an error or 304. +func (q *Queries) SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error { + _, err := q.db.Exec(ctx, saveChannelCatalogue, + arg.Iata, + arg.Url, + arg.Payload, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + ) + return err +} + +const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest($9::text[]) AS name, unnest($10::bytea[]) AS key, + unnest($11::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` + Names []string `json:"names"` + Keys [][]byte `json:"keys"` + Fingerprints [][]byte `json:"fingerprints"` +} + +// One statement commits the validated snapshot and its lookup identities together. +// Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +// after an error or 304. Imported names never replace existing manual metadata. +func (q *Queries) SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error { + _, err := q.db.Exec(ctx, saveScopeCatalogue, + arg.Iata, + arg.Url, + arg.Payload, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + arg.Names, + arg.Keys, + arg.Fingerprints, + ) + return err +} + +const saveZoneBoundary = `-- name: SaveZoneBoundary :exec +INSERT INTO meshmapper_zone_boundaries (iata, url, feature, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata) DO UPDATE SET + url = EXCLUDED.url, + feature = COALESCE(EXCLUDED.feature, meshmapper_zone_boundaries.feature), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_boundaries.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_zone_boundaries.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveZoneBoundaryParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Feature []byte `json:"feature"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +// NULL feature/etag/checked_at retain the last good boundary after an error or 304. +func (q *Queries) SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryParams) error { + _, err := q.db.Exec(ctx, saveZoneBoundary, + arg.Iata, + arg.Url, + arg.Feature, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + ) + return err +} + +const saveZoneList = `-- name: SaveZoneList :exec +INSERT INTO meshmapper_zone_lists (country, payload, etag, fetched_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2::jsonb, $3::text, + $4::timestamptz, $5, $6, $7) +ON CONFLICT (country) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_zone_lists.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_lists.etag), + fetched_at = COALESCE(EXCLUDED.fetched_at, meshmapper_zone_lists.fetched_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveZoneListParams struct { + Country string `json:"country"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + FetchedAt pgtype.Timestamptz `json:"fetched_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +// NULL payload/etag/fetched_at retain the last good list after an error or 304. +func (q *Queries) SaveZoneList(ctx context.Context, arg SaveZoneListParams) error { + _, err := q.db.Exec(ctx, saveZoneList, + arg.Country, + arg.Payload, + arg.Etag, + arg.FetchedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + ) + return err +} + const searchKnownRoutes = `-- name: SearchKnownRoutes :many SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes @@ -3404,17 +5042,28 @@ type SearchKnownRoutesParams struct { Column3 []byte `json:"column_3"` } +type SearchKnownRoutesRow struct { + ID int64 `json:"id"` + NodeIds []uuid.UUID `json:"node_ids"` + HashPrefix [][]byte `json:"hash_prefix"` + Iata string `json:"iata"` + HopCount int32 `json:"hop_count"` + FirstSeen pgtype.Timestamptz `json:"first_seen"` + LastSeen pgtype.Timestamptz `json:"last_seen"` + ObservationCount int64 `json:"observation_count"` +} + // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. -func (q *Queries) SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]KnownRoute, error) { +func (q *Queries) SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) { rows, err := q.db.Query(ctx, searchKnownRoutes, arg.Iata, arg.Column2, arg.Column3) if err != nil { return nil, err } defer rows.Close() - items := []KnownRoute{} + items := []SearchKnownRoutesRow{} for rows.Next() { - var i KnownRoute + var i SearchKnownRoutesRow if err := rows.Scan( &i.ID, &i.NodeIds, @@ -3481,11 +5130,12 @@ func (q *Queries) SetPacketDecrypted(ctx context.Context, packetHash []byte) err const touchObserverBrokers = `-- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name ` @@ -3494,21 +5144,32 @@ type TouchObserverBrokersParams struct { Column1 []uuid.UUID `json:"column_1"` Column2 []string `json:"column_2"` Column3 []pgtype.Timestamptz `json:"column_3"` + Column4 []pgtype.Timestamptz `json:"column_4"` } func (q *Queries) TouchObserverBrokers(ctx context.Context, arg TouchObserverBrokersParams) error { - _, err := q.db.Exec(ctx, touchObserverBrokers, arg.Column1, arg.Column2, arg.Column3) + _, err := q.db.Exec(ctx, touchObserverBrokers, + arg.Column1, + arg.Column2, + arg.Column3, + arg.Column4, + ) return err } const touchObservers = `-- name: TouchObservers :exec UPDATE observers o SET last_seen = GREATEST(o.last_seen, v.seen), - observation_count = COALESCE(o.observation_count, 0) + v.delta + observation_count = COALESCE(o.observation_count, 0) + v.delta, + last_iata = CASE WHEN v.iata <> '' AND (o.last_iata_at IS NULL OR v.iata_at > o.last_iata_at) + THEN v.iata ELSE o.last_iata END, + last_iata_at = GREATEST(o.last_iata_at, v.iata_at) FROM ( SELECT unnest($1::uuid[]) AS id, unnest($2::timestamptz[]) AS seen, - unnest($3::int[]) AS delta + unnest($3::int[]) AS delta, + unnest($4::text[]) AS iata, + unnest($5::timestamptz[]) AS iata_at ) v WHERE o.id = v.id ` @@ -3517,12 +5178,20 @@ type TouchObserversParams struct { Column1 []uuid.UUID `json:"column_1"` Column2 []pgtype.Timestamptz `json:"column_2"` Column3 []int32 `json:"column_3"` + Column4 []string `json:"column_4"` + Column5 []pgtype.Timestamptz `json:"column_5"` } // Batched flush of coalesced presence bumps. GREATEST keeps a late flush // from regressing a newer write-through (e.g. a status update). func (q *Queries) TouchObservers(ctx context.Context, arg TouchObserversParams) error { - _, err := q.db.Exec(ctx, touchObservers, arg.Column1, arg.Column2, arg.Column3) + _, err := q.db.Exec(ctx, touchObservers, + arg.Column1, + arg.Column2, + arg.Column3, + arg.Column4, + arg.Column5, + ) return err } @@ -3546,6 +5215,23 @@ func (q *Queries) TouchPackets(ctx context.Context, arg TouchPacketsParams) erro return err } +const updateObserverRegionScope = `-- name: UpdateObserverRegionScope :exec +UPDATE observers SET region_scope = $2 WHERE id = $1 +` + +type UpdateObserverRegionScopeParams struct { + ID uuid.UUID `json:"id"` + RegionScope *string `json:"region_scope"` +} + +// Records the observer's own OTA-reported region scope, from the "self" +// field of a /neighbors report. Always known (not queried OTA), so this +// unconditionally overwrites, unlike the neighbor-side region_scope. +func (q *Queries) UpdateObserverRegionScope(ctx context.Context, arg UpdateObserverRegionScopeParams) error { + _, err := q.db.Exec(ctx, updateObserverRegionScope, arg.ID, arg.RegionScope) + return err +} + const updateObserverStatus = `-- name: UpdateObserverStatus :one UPDATE observers SET display_name = COALESCE(NULLIF($2, ''), display_name), @@ -3612,18 +5298,16 @@ INSERT INTO channels (channel_hash, key_fingerprint, name, hashtag, is_hashtag, VALUES ($1, $2::bytea, $3, $4, $5, ($2 IS NOT NULL), NOW()) ON CONFLICT (channel_hash, key_fingerprint) DO UPDATE SET last_seen = NOW(), - name = COALESCE(EXCLUDED.name, channels.name), - message_count = CASE WHEN $6 THEN channels.message_count + 1 ELSE channels.message_count END + name = COALESCE(EXCLUDED.name, channels.name) RETURNING id, channel_hash, key_fingerprint, name, hashtag, is_hashtag, is_public, key_known, first_seen, last_seen, message_count ` type UpsertChannelParams struct { - ChannelHash []byte `json:"channel_hash"` - Column2 []byte `json:"column_2"` - Name *string `json:"name"` - Hashtag *string `json:"hashtag"` - IsHashtag *bool `json:"is_hashtag"` - MessageCount *int64 `json:"message_count"` + ChannelHash []byte `json:"channel_hash"` + Column2 []byte `json:"column_2"` + Name *string `json:"name"` + Hashtag *string `json:"hashtag"` + IsHashtag *bool `json:"is_hashtag"` } // ============================================================ @@ -3638,7 +5322,6 @@ func (q *Queries) UpsertChannel(ctx context.Context, arg UpsertChannelParams) (C arg.Name, arg.Hashtag, arg.IsHashtag, - arg.MessageCount, ) var i Channel err := row.Scan( @@ -3756,16 +5439,53 @@ func (q *Queries) UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsPa return err } +const upsertImportedRegion = `-- name: UpsertImportedRegion :one +INSERT INTO regions (slug, name, display_order, center_lat, center_lng, zoom_level, imported, updated_at) +VALUES ($1, $2, $3, $4, $5, NULL, TRUE, NOW()) +ON CONFLICT (slug) DO UPDATE SET + name = EXCLUDED.name, + display_order = EXCLUDED.display_order, + center_lat = EXCLUDED.center_lat, + center_lng = EXCLUDED.center_lng, + updated_at = NOW() +WHERE regions.imported +RETURNING id +` + +type UpsertImportedRegionParams struct { + Slug string `json:"slug"` + Name string `json:"name"` + DisplayOrder *int32 `json:"display_order"` + CenterLat *float64 `json:"center_lat"` + CenterLng *float64 `json:"center_lng"` +} + +// A hand-written region owns its slug: the WHERE turns a clash into no row. +func (q *Queries) UpsertImportedRegion(ctx context.Context, arg UpsertImportedRegionParams) (int32, error) { + row := q.db.QueryRow(ctx, upsertImportedRegion, + arg.Slug, + arg.Name, + arg.DisplayOrder, + arg.CenterLat, + arg.CenterLng, + ) + var id int32 + err := row.Scan(&id) + return id, err +} + const upsertKnownRoute = `-- name: UpsertKnownRoute :exec -INSERT INTO known_routes (node_ids, hash_prefix, iata, hop_count) -VALUES ($1, $2, $3, $4) -ON CONFLICT (node_ids, iata) DO UPDATE SET +INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) +VALUES ($1, $2, $3, $4, $5) +ON CONFLICT (iata, path_key) DO UPDATE SET + hash_prefix = EXCLUDED.hash_prefix, last_seen = NOW(), observation_count = known_routes.observation_count + 1 ` type UpsertKnownRouteParams struct { + PathKey []byte `json:"path_key"` NodeIds []uuid.UUID `json:"node_ids"` HashPrefix [][]byte `json:"hash_prefix"` Iata string `json:"iata"` @@ -3775,11 +5495,12 @@ type UpsertKnownRouteParams struct { // ============================================================ // ROUTES // ============================================================ -// Inserts or updates a known route (all hops resolved to high confidence). -// node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and -// their hash bytes. last_seen is bumped on conflict. +// Route identity is path_key, an md5 of node_ids computed by the caller. +// On conflict, observation_count and last_seen are bumped and hash_prefix follows the +// latest hearing, so evidence matches the hash width the route uses now. func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error { _, err := q.db.Exec(ctx, upsertKnownRoute, + arg.PathKey, arg.NodeIds, arg.HashPrefix, arg.Iata, @@ -3791,13 +5512,13 @@ func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRoutePara const upsertNode = `-- name: UpsertNode :one INSERT INTO nodes (public_key, node_type, name, latitude, longitude, location_source, last_advert_at, last_seen, radio_freq_mhz, radio_sf, radio_bw_khz, device_clock_drift_seconds) -VALUES ($1, $2, $3, $4, $5, 'advert', NOW(), NOW(), $6, $7, $8, $9) +VALUES ($1, $2, $3, $4, $5, CASE WHEN $4::double precision IS NOT NULL THEN 'advert' END, NOW(), NOW(), $6, $7, $8, $9) ON CONFLICT (public_key) DO UPDATE SET node_type = EXCLUDED.node_type, name = COALESCE(EXCLUDED.name, nodes.name), - latitude = COALESCE(EXCLUDED.latitude, nodes.latitude), - longitude = COALESCE(EXCLUDED.longitude, nodes.longitude), - location_source = CASE WHEN EXCLUDED.latitude IS NOT NULL THEN 'advert' ELSE nodes.location_source END, + latitude = CASE WHEN $10::bool THEN NULL ELSE COALESCE(EXCLUDED.latitude, nodes.latitude) END, + longitude = CASE WHEN $10::bool THEN NULL ELSE COALESCE(EXCLUDED.longitude, nodes.longitude) END, + location_source = CASE WHEN $10::bool THEN NULL WHEN EXCLUDED.latitude IS NOT NULL THEN 'advert' ELSE nodes.location_source END, last_advert_at = NOW(), last_seen = NOW(), radio_freq_mhz = EXCLUDED.radio_freq_mhz, @@ -3817,6 +5538,7 @@ type UpsertNodeParams struct { RadioSf *int16 `json:"radio_sf"` RadioBwKhz *float32 `json:"radio_bw_khz"` DeviceClockDriftSeconds *int32 `json:"device_clock_drift_seconds"` + ClearLocation bool `json:"clear_location"` } // ============================================================ @@ -3833,6 +5555,7 @@ func (q *Queries) UpsertNode(ctx context.Context, arg UpsertNodeParams) (Node, e arg.RadioSf, arg.RadioBwKhz, arg.DeviceClockDriftSeconds, + arg.ClearLocation, ) var i Node err := row.Scan( @@ -3883,19 +5606,21 @@ func (q *Queries) UpsertNodeIATA(ctx context.Context, arg UpsertNodeIATAParams) const upsertNodeNeighbor = `-- name: UpsertNodeNeighbor :exec -INSERT INTO node_neighbors (node_id, neighbor_id, iata, observation_count, snr) -VALUES ($1, $2, $3, 1, $4) +INSERT INTO node_neighbors (node_id, neighbor_id, iata, observation_count, snr, region_scope) +VALUES ($1, $2, $3, 1, $4, $5) ON CONFLICT (node_id, neighbor_id, iata) DO UPDATE SET last_seen = NOW(), observation_count = node_neighbors.observation_count + 1, - snr = COALESCE(EXCLUDED.snr, node_neighbors.snr) + snr = COALESCE(EXCLUDED.snr, node_neighbors.snr), + region_scope = COALESCE(EXCLUDED.region_scope, node_neighbors.region_scope) ` type UpsertNodeNeighborParams struct { - NodeID uuid.UUID `json:"node_id"` - NeighborID uuid.UUID `json:"neighbor_id"` - Iata string `json:"iata"` - Snr *float32 `json:"snr"` + NodeID uuid.UUID `json:"node_id"` + NeighborID uuid.UUID `json:"neighbor_id"` + Iata string `json:"iata"` + Snr *float32 `json:"snr"` + RegionScope *string `json:"region_scope"` } // ============================================================ @@ -3904,15 +5629,18 @@ type UpsertNodeNeighborParams struct { // Records or updates a neighbor relationship between two nodes observed in the same IATA. // node_id is the advertising node, neighbor_id is the first-hop forwarder. // snr is optional; pass NULL when no signal reading is available (the -// common case). On conflict, snr is only overwritten when a new non-null -// value is supplied, so a later no-SNR observation doesn't erase an -// earlier real reading. +// common case). regionScope is optional too; pass NULL whenever the OTA +// scope query for this neighbor didn't succeed (status != "responded"), +// so a failed/timed-out query doesn't erase a previously known scope. +// On conflict, snr and region_scope are only overwritten when a new +// non-null value is supplied. func (q *Queries) UpsertNodeNeighbor(ctx context.Context, arg UpsertNodeNeighborParams) error { _, err := q.db.Exec(ctx, upsertNodeNeighbor, arg.NodeID, arg.NeighborID, arg.Iata, arg.Snr, + arg.RegionScope, ) return err } @@ -3936,19 +5664,31 @@ func (q *Queries) UpsertNodeShortID(ctx context.Context, arg UpsertNodeShortIDPa const upsertObserver = `-- name: UpsertObserver :one -INSERT INTO observers (public_key, observer_type, last_seen) -VALUES ($1, 'unknown', NOW()) +INSERT INTO observers (public_key, observer_type, last_seen, last_iata, last_iata_at) +VALUES ($1, 'unknown', NOW(), NULLIF($2::text, ''), + CASE WHEN $2::text <> '' THEN $3::timestamptz END) ON CONFLICT (public_key) DO UPDATE SET last_seen = NOW(), - observation_count = observers.observation_count + 1 -RETURNING id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata + observation_count = observers.observation_count + 1, + last_iata = CASE WHEN EXCLUDED.last_iata IS NOT NULL + AND (observers.last_iata_at IS NULL OR EXCLUDED.last_iata_at > observers.last_iata_at) + THEN EXCLUDED.last_iata ELSE observers.last_iata END, + last_iata_at = GREATEST(observers.last_iata_at, EXCLUDED.last_iata_at) +RETURNING id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope, last_iata, last_iata_at ` +type UpsertObserverParams struct { + PublicKey []byte `json:"public_key"` + Iata string `json:"iata"` + IataAt pgtype.Timestamptz `json:"iata_at"` +} + // ============================================================ // OBSERVERS // ============================================================ -func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observer, error) { - row := q.db.QueryRow(ctx, upsertObserver, publicKey) +// Empty iata (status/neighbors) leaves last_iata alone; otherwise the newest iata_at wins. +func (q *Queries) UpsertObserver(ctx context.Context, arg UpsertObserverParams) (Observer, error) { + row := q.db.QueryRow(ctx, upsertObserver, arg.PublicKey, arg.Iata, arg.IataAt) var i Observer err := row.Scan( &i.ID, @@ -3971,6 +5711,9 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe &i.LastSeen, &i.ObservationCount, &i.Metadata, + &i.RegionScope, + &i.LastIata, + &i.LastIataAt, ) return i, err } @@ -3978,22 +5721,23 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe const upsertObserverBroker = `-- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN $3::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW() + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at) ` type UpsertObserverBrokerParams struct { ObserverID uuid.UUID `json:"observer_id"` BrokerName string `json:"broker_name"` + IsPacket bool `json:"is_packet"` } // ============================================================ // OBSERVER BROKERS // ============================================================ func (q *Queries) UpsertObserverBroker(ctx context.Context, arg UpsertObserverBrokerParams) error { - _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName) + _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName, arg.IsPacket) return err } @@ -4016,6 +5760,7 @@ func (q *Queries) UpsertObserverScope(ctx context.Context, arg UpsertObserverSco const upsertPacket = `-- name: UpsertPacket :one +WITH up AS ( INSERT INTO packets ( packet_hash, payload_type, @@ -4038,8 +5783,25 @@ INSERT INTO packets ( ) ON CONFLICT (packet_hash) DO UPDATE SET last_heard_at = NOW() -RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, (xmax = 0) +RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, trace_tag, scope_id, (xmax = 0) AS inserted +), trace AS ( + INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, packet_count, trace_type, scope_id, best_payload, best_path_len) + SELECT u.trace_tag, u.first_heard_at, u.last_heard_at, 1, u.parsed_payload->>'type', u.scope_id, u.parsed_payload, + CASE WHEN jsonb_typeof(u.parsed_payload->'pathHashes') = 'array' + THEN jsonb_array_length(u.parsed_payload->'pathHashes') ELSE 0 END + FROM up u + WHERE u.inserted AND u.trace_tag IS NOT NULL + ON CONFLICT (trace_tag) DO UPDATE SET + packet_count = trace_tags.packet_count + 1, + trace_type = GREATEST(trace_tags.trace_type, EXCLUDED.trace_type), + scope_id = COALESCE(trace_tags.scope_id, EXCLUDED.scope_id), + best_payload = CASE WHEN EXCLUDED.best_path_len > trace_tags.best_path_len + THEN EXCLUDED.best_payload ELSE trace_tags.best_payload END, + best_path_len = GREATEST(trace_tags.best_path_len, EXCLUDED.best_path_len) +) +SELECT packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, inserted +FROM up ` type UpsertPacketParams struct { @@ -4081,6 +5843,8 @@ type UpsertPacketRow struct { // ============================================================ // PACKETS // ============================================================ +// A new TRACE packet adds itself to its tag summary in the same statement, so the summary +// can't miss a stored packet. 'TRACE' > 'PING'; the longest path keeps the best payload. func (q *Queries) UpsertPacket(ctx context.Context, arg UpsertPacketParams) (UpsertPacketRow, error) { row := q.db.QueryRow(ctx, upsertPacket, arg.PacketHash, @@ -4130,6 +5894,7 @@ ON CONFLICT (slug) DO UPDATE SET center_lat = EXCLUDED.center_lat, center_lng = EXCLUDED.center_lng, zoom_level = EXCLUDED.zoom_level, + imported = FALSE, -- config owns the slug from now on updated_at = NOW() RETURNING id ` @@ -4159,42 +5924,6 @@ func (q *Queries) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int return id, err } -const upsertRegionIATA = `-- name: UpsertRegionIATA :exec -INSERT INTO region_iatas (region_id, iata) -VALUES ($1, $2) -ON CONFLICT (region_id, iata) DO NOTHING -` - -type UpsertRegionIATAParams struct { - RegionID int32 `json:"region_id"` - Iata string `json:"iata"` -} - -func (q *Queries) UpsertRegionIATA(ctx context.Context, arg UpsertRegionIATAParams) error { - _, err := q.db.Exec(ctx, upsertRegionIATA, arg.RegionID, arg.Iata) - return err -} - -const upsertTraceIATA = `-- name: UpsertTraceIATA :exec -INSERT INTO trace_iatas (trace_tag, iata, last_heard) -VALUES ($1, $2, $3) -ON CONFLICT (trace_tag, iata) DO UPDATE SET - last_heard = EXCLUDED.last_heard -WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour' -` - -type UpsertTraceIATAParams struct { - TraceTag []byte `json:"trace_tag"` - Iata string `json:"iata"` - LastHeard pgtype.Timestamptz `json:"last_heard"` -} - -// Refreshes at most hourly so repeat hears don't churn the row. -func (q *Queries) UpsertTraceIATA(ctx context.Context, arg UpsertTraceIATAParams) error { - _, err := q.db.Exec(ctx, upsertTraceIATA, arg.TraceTag, arg.Iata, arg.LastHeard) - return err -} - const upsertTransportScope = `-- name: UpsertTransportScope :exec INSERT INTO transport_scopes (name, display_name, transport_key, key_fingerprint) @@ -4202,7 +5931,8 @@ VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO UPDATE SET display_name = EXCLUDED.display_name, transport_key = EXCLUDED.transport_key, - key_fingerprint = EXCLUDED.key_fingerprint + key_fingerprint = EXCLUDED.key_fingerprint, + imported_only = FALSE ` type UpsertTransportScopeParams struct { diff --git a/db/sqlc/rollup.sql.go b/db/sqlc/rollup.sql.go new file mode 100644 index 00000000..02f7fd08 --- /dev/null +++ b/db/sqlc/rollup.sql.go @@ -0,0 +1,529 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: rollup.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const deleteOldRollups = `-- name: DeleteOldRollups :exec +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour < $1::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour < $1::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour < $1::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour < $1::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour < $1::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour < $1::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour < $1::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour < $1::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour < $1::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour < $1::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour < $1::timestamptz), + l AS (DELETE FROM analytics_dirty_hours tl WHERE tl.hour < $1::timestamptz), + n AS (DELETE FROM analytics_hourly_scope_observers tn WHERE tn.hour < $1::timestamptz), + o AS (DELETE FROM analytics_hourly_scope_nodes tob WHERE tob.hour < $1::timestamptz), + m AS (DELETE FROM analytics_rollup_hours tm WHERE tm.hour < $1::timestamptz RETURNING 1) +UPDATE analytics_raw_state SET coverage = coverage + 1 WHERE EXISTS (SELECT 1 FROM m) +` + +// Bumps coverage when hours go, so cached responses that included them are not reused. +func (q *Queries) DeleteOldRollups(ctx context.Context, cutoff pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, deleteOldRollups, cutoff) + return err +} + +const deleteRollupHour = `-- name: DeleteRollupHour :exec +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour = $1::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour = $1::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour = $1::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour = $1::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour = $1::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour = $1::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour = $1::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour = $1::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour = $1::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = $1::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = $1::timestamptz), + l AS (DELETE FROM analytics_hourly_scope_observers tl WHERE tl.hour = $1::timestamptz) +DELETE FROM analytics_hourly_scope_nodes tm WHERE tm.hour = $1::timestamptz +` + +func (q *Queries) DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, deleteRollupHour, hour) + return err +} + +const deleteStaleDirtyHours = `-- name: DeleteStaleDirtyHours :exec +DELETE FROM analytics_dirty_hours d +WHERE EXISTS (SELECT 1 FROM analytics_rollup_hours h WHERE h.hour = d.hour AND h.status = 'partial') + OR d.hour <= (SELECT raw_deleted_before + INTERVAL '30 minutes' FROM analytics_raw_state) +` + +// Dirty hours that can no longer be re-rolled. +func (q *Queries) DeleteStaleDirtyHours(ctx context.Context) error { + _, err := q.db.Exec(ctx, deleteStaleDirtyHours) + return err +} + +const fillRollupObs = `-- name: FillRollupObs :exec +INSERT INTO rollup_obs (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, + path_len, rssi, snr, airtime_ms, payload_type, route_type, origin_pubkey, scope_id) +SELECT po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, + COALESCE(octet_length(po.path_bytes), 0), po.rssi, po.snr, po.airtime_ms, po.payload_type, + p.route_type, p.origin_pubkey, p.scope_id +FROM packet_observations po +JOIN packets p ON p.packet_hash = po.packet_hash +WHERE po.heard_at >= $1::timestamptz AND po.heard_at < $1::timestamptz + INTERVAL '1 hour' +` + +func (q *Queries) FillRollupObs(ctx context.Context, dollar_1 pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, fillRollupObs, dollar_1) + return err +} + +const finishRollupHour = `-- name: FinishRollupHour :one +WITH prev AS ( + SELECT ph.content_hash FROM analytics_rollup_hours ph WHERE ph.hour = $1::timestamptz +), upd AS ( + UPDATE analytics_rollup_hours uh + SET status = 'complete', rolled_at = now(), content_hash = $2::text + WHERE uh.hour = $1::timestamptz +), undirty AS ( + DELETE FROM analytics_dirty_hours dh WHERE dh.hour = $1::timestamptz +), rev AS ( + UPDATE analytics_state SET revision = revision + 1 + WHERE (SELECT content_hash FROM prev) IS DISTINCT FROM $2::text + RETURNING revision +) +SELECT EXISTS (SELECT 1 FROM rev)::boolean AS changed +` + +type FinishRollupHourParams struct { + Hour pgtype.Timestamptz `json:"hour"` + ContentHash string `json:"content_hash"` +} + +// Marks the hour complete and bumps the revision only when its content changed. +func (q *Queries) FinishRollupHour(ctx context.Context, arg FinishRollupHourParams) (bool, error) { + row := q.db.QueryRow(ctx, finishRollupHour, arg.Hour, arg.ContentHash) + var changed bool + err := row.Scan(&changed) + return changed, err +} + +const getAnalyticsRevision = `-- name: GetAnalyticsRevision :one +SELECT (s.revision + r.coverage)::bigint FROM analytics_state s CROSS JOIN analytics_raw_state r +` + +// Both counters only grow, so their sum changes whenever rolled content or coverage does. +func (q *Queries) GetAnalyticsRevision(ctx context.Context) (int64, error) { + row := q.db.QueryRow(ctx, getAnalyticsRevision) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err +} + +const listDirtyRollupHours = `-- name: ListDirtyRollupHours :many +SELECT d.hour FROM analytics_dirty_hours d +JOIN analytics_rollup_hours h ON h.hour = d.hour AND h.status = 'complete' +CROSS JOIN analytics_raw_state r +WHERE r.raw_deleted_before IS NULL OR d.hour > r.raw_deleted_before + INTERVAL '30 minutes' +ORDER BY d.hour +LIMIT $1 +` + +// Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours +// wait: their first roll consumes the entry only if the entry predates its snapshot. +func (q *Queries) ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + rows, err := q.db.Query(ctx, listDirtyRollupHours, limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.Timestamptz{} + for rows.Next() { + var hour pgtype.Timestamptz + if err := rows.Scan(&hour); err != nil { + return nil, err + } + items = append(items, hour) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listMissingRollupHours = `-- name: ListMissingRollupHours :many +SELECT hour FROM analytics_rollup_hours +WHERE status = 'missing' +ORDER BY hour +LIMIT $1 +` + +func (q *Queries) ListMissingRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + rows, err := q.db.Query(ctx, listMissingRollupHours, limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.Timestamptz{} + for rows.Next() { + var hour pgtype.Timestamptz + if err := rows.Scan(&hour); err != nil { + return nil, err + } + items = append(items, hour) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const lockRollupHour = `-- name: LockRollupHour :one +SELECT CASE + WHEN r.raw_deleted_before IS NOT NULL AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + THEN 'partial' + ELSE h.status + END::text AS status +FROM analytics_rollup_hours h CROSS JOIN analytics_raw_state r +WHERE h.hour = $1 +FOR UPDATE OF h +` + +// Returns the hour's status, or 'partial' if raw deletion has reached it since registration. +func (q *Queries) LockRollupHour(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + row := q.db.QueryRow(ctx, lockRollupHour, hour) + var status string + err := row.Scan(&status) + return status, err +} + +const markPartialRollupHours = `-- name: MarkPartialRollupHours :exec +WITH marked AS ( + UPDATE analytics_rollup_hours h SET status = 'partial' + FROM analytics_raw_state r + WHERE h.status = 'missing' + AND r.raw_deleted_before IS NOT NULL + AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked) +` + +// Hours whose raw rows cleanup already started deleting can never be rolled completely. +// Readers report the new status, so the revision moves too. +func (q *Queries) MarkPartialRollupHours(ctx context.Context) error { + _, err := q.db.Exec(ctx, markPartialRollupHours) + return err +} + +const oldestMissingRollupHour = `-- name: OldestMissingRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'missing' +` + +func (q *Queries) OldestMissingRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + row := q.db.QueryRow(ctx, oldestMissingRollupHour) + var column_1 pgtype.Timestamptz + err := row.Scan(&column_1) + return column_1, err +} + +const registerRollupHours = `-- name: RegisterRollupHours :exec +INSERT INTO analytics_rollup_hours (hour, status) +SELECT h, 'missing' +FROM (SELECT CASE WHEN MIN(heard_at) IS NOT NULL + THEN date_trunc('hour', GREATEST(MIN(heard_at), $1::timestamptz), 'UTC') END AS first_hour + FROM packet_observations) f, + generate_series(f.first_hour, date_trunc('hour', now() - INTERVAL '95 minutes', 'UTC'), INTERVAL '1 hour') h +WHERE f.first_hour IS NOT NULL +ON CONFLICT (hour) DO NOTHING +` + +// Adds a 'missing' row for every eligible hour since the earliest retained observation +// (and not before @since), zero-observation hours included. +// No observations yet means nothing to register (GREATEST would ignore the NULL MIN). +func (q *Queries) RegisterRollupHours(ctx context.Context, since pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, registerRollupHours, since) + return err +} + +const rollAdvertHearings = `-- name: RollAdvertHearings :exec +INSERT INTO analytics_hourly_advert_hearings (hour, iata, origin_pubkey, observations, last_heard, name, node_type) +SELECT $1::timestamptz, r.iata, r.origin_pubkey, r.n, r.last_heard, n.name, n.node_type +FROM (SELECT iata, origin_pubkey, count(*) AS n, max(heard_at) AS last_heard + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY iata, origin_pubkey) r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +` + +func (q *Queries) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollAdvertHearings, hour) + return err +} + +const rollAdvertSets = `-- name: RollAdvertSets :exec +INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) +SELECT $1::timestamptz, origin_pubkey, iatas, count(*), + count(*) FILTER (WHERE route_type IN (0, 1)), + count(*) FILTER (WHERE route_type IN (2, 3)) +FROM (SELECT origin_pubkey, route_type, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY packet_hash, origin_pubkey, route_type) p +GROUP BY origin_pubkey, iatas +` + +// Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. +func (q *Queries) RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollAdvertSets, hour) + return err +} + +const rollIATAObservations = `-- name: RollIATAObservations :exec +INSERT INTO analytics_hourly_iata_observations (hour, iata, observation_count) +SELECT $1::timestamptz, iata, count(*) FROM rollup_obs GROUP BY iata +` + +func (q *Queries) RollIATAObservations(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollIATAObservations, hour) + return err +} + +const rollObserverActivity = `-- name: RollObserverActivity :exec +INSERT INTO analytics_hourly_observer_activity (hour, observer_id, payload_type, observations, airtime_ms, airtime_n, + snr_sum, snr_n, snr_min, rssi_sum, rssi_n) +SELECT $1::timestamptz, observer_id, COALESCE(payload_type, -1)::smallint, + count(*), sum(airtime_ms)::real, count(airtime_ms), + sum(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + count(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), + min(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + sum(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + count(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)) +FROM rollup_obs +GROUP BY observer_id, COALESCE(payload_type, -1) +` + +func (q *Queries) RollObserverActivity(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollObserverActivity, hour) + return err +} + +const rollObserverIdentity = `-- name: RollObserverIdentity :exec +INSERT INTO analytics_hourly_observer_identity (hour, iata, observer_id, observation_count, display_name, observer_type) +SELECT $1::timestamptz, r.iata, r.observer_id, r.n, o.display_name, o.observer_type +FROM (SELECT iata, observer_id, count(*) AS n FROM rollup_obs GROUP BY iata, observer_id) r +LEFT JOIN observers o ON o.id = r.observer_id +` + +func (q *Queries) RollObserverIdentity(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollObserverIdentity, hour) + return err +} + +const rollPacketSets = `-- name: RollPacketSets :exec +INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) +SELECT $1::timestamptz, iatas, count(*) +FROM (SELECT array_agg(DISTINCT iata ORDER BY iata) AS iatas FROM rollup_obs GROUP BY packet_hash) p +GROUP BY iatas +` + +func (q *Queries) RollPacketSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollPacketSets, hour) + return err +} + +const rollPaths = `-- name: RollPaths :exec +INSERT INTO analytics_hourly_paths (hour, iata, category, hash_bytes, entries, receptions) +SELECT $1::timestamptz, c.iata, c.category, c.hash_bytes, c.entries, count(*) +FROM ( + SELECT k.iata, k.category, + CASE WHEN k.category = 0 THEN k.hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN k.category = 0 THEN k.hop_count ELSE 0 END::integer AS entries + FROM ( + SELECT iata, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND path_len = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM rollup_obs + ) k +) c +GROUP BY c.iata, c.category, c.hash_bytes, c.entries +` + +// category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). +func (q *Queries) RollPaths(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollPaths, hour) + return err +} + +const rollPayloadBreakdown = `-- name: RollPayloadBreakdown :exec +INSERT INTO analytics_hourly_payload_breakdown (hour, iata, payload_type, count) +SELECT $1::timestamptz, iata, payload_type, count(*) +FROM rollup_obs WHERE payload_type IS NOT NULL +GROUP BY iata, payload_type +` + +func (q *Queries) RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollPayloadBreakdown, hour) + return err +} + +const rollScopeNodes = `-- name: RollScopeNodes :exec +INSERT INTO analytics_hourly_scope_nodes (hour, iata, scope_id, origin_pubkey) +SELECT DISTINCT $1::timestamptz, iata, scope_id, origin_pubkey +FROM rollup_obs WHERE payload_type = 4 AND origin_pubkey IS NOT NULL AND scope_id IS NOT NULL +` + +func (q *Queries) RollScopeNodes(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollScopeNodes, hour) + return err +} + +const rollScopeObservers = `-- name: RollScopeObservers :exec +INSERT INTO analytics_hourly_scope_observers (hour, iata, scope_id, observer_id) +SELECT DISTINCT $1::timestamptz, iata, scope_id, observer_id +FROM rollup_obs WHERE scope_id IS NOT NULL +` + +func (q *Queries) RollScopeObservers(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollScopeObservers, hour) + return err +} + +const rollScopeSets = `-- name: RollScopeSets :exec +INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) +SELECT $1::timestamptz, iatas, scope_id, count(*) +FROM (SELECT scope_id, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs WHERE scope_id IS NOT NULL + GROUP BY packet_hash, scope_id) p +GROUP BY iatas, scope_id +` + +func (q *Queries) RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollScopeSets, hour) + return err +} + +const rollSignal = `-- name: RollSignal :exec +INSERT INTO analytics_hourly_signal (hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) +SELECT $1::timestamptz, b.iata, grouping(b.snr_bin, b.rssi_bin)::integer, + COALESCE(b.snr_bin, -1)::integer, COALESCE(b.rssi_bin, -1)::integer, + count(*), count(b.snr), COALESCE(sum(b.snr), 0)::double precision, + count(b.rssi), COALESCE(sum(b.rssi), 0)::double precision +FROM ( + SELECT s.iata, s.snr, s.rssi, + width_bucket(s.snr, -30, 30, 12) AS snr_bin, + width_bucket(s.rssi, -140, 0, 14) AS rssi_bin + FROM ( + SELECT ro.iata, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + AND ro.snr > '-Infinity'::real AND ro.snr < 'Infinity'::real + THEN ro.snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + THEN ro.rssi::double precision END AS rssi + FROM rollup_obs ro + ) s +) b +GROUP BY GROUPING SETS ((b.iata), (b.iata, b.snr_bin), (b.iata, b.rssi_bin)) +` + +// kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. +// rssi=0 AND snr=0 means "not reported". +func (q *Queries) RollSignal(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollSignal, hour) + return err +} + +const rollTalkerSets = `-- name: RollTalkerSets :exec +INSERT INTO analytics_hourly_talker_sets (hour, sender_name, iatas, messages, last_sent) +SELECT $1::timestamptz, sender_name, iatas, count(*), max(sent_at) +FROM (SELECT cm.sender_name, max(cm.sent_at) AS sent_at, array_agg(DISTINCT ro.iata ORDER BY ro.iata) AS iatas + FROM rollup_obs ro + JOIN channel_messages cm ON cm.packet_hash = ro.packet_hash + WHERE cm.sender_name IS NOT NULL + GROUP BY cm.id, cm.sender_name) m +GROUP BY sender_name, iatas +` + +func (q *Queries) RollTalkerSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollTalkerSets, hour) + return err +} + +const rollupContentHash = `-- name: RollupContentHash :one +SELECT md5(concat_ws('|', + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_iata_observations t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_payload_breakdown t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_signal t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_paths t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_activity t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_identity t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_hearings t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_talker_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_packet_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_observers t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_nodes t WHERE t.hour = $1::timestamptz) +))::text AS content_hash +` + +// Order-independent fingerprint of every family row for the hour. +func (q *Queries) RollupContentHash(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + row := q.db.QueryRow(ctx, rollupContentHash, hour) + var content_hash string + err := row.Scan(&content_hash) + return content_hash, err +} + +const rollupUnlock = `-- name: RollupUnlock :exec +SELECT pg_advisory_unlock(25189, 1) +` + +func (q *Queries) RollupUnlock(ctx context.Context) error { + _, err := q.db.Exec(ctx, rollupUnlock) + return err +} + +const setRollupHourPartial = `-- name: SetRollupHourPartial :exec +WITH marked AS ( + UPDATE analytics_rollup_hours SET status = 'partial' WHERE hour = $1 AND status = 'missing' RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked) +` + +func (q *Queries) SetRollupHourPartial(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, setRollupHourPartial, hour) + return err +} + +const tryRollupLock = `-- name: TryRollupLock :one + +SELECT pg_try_advisory_lock(25189, 1) +` + +// ============================================================ +// HOURLY ANALYTICS ROLLUPS +// An hour [H, H+1h) is eligible once now() >= H + 95 min (ingest clamps heard_at to ±30 min). +// ============================================================ +func (q *Queries) TryRollupLock(ctx context.Context) (bool, error) { + row := q.db.QueryRow(ctx, tryRollupLock) + var pg_try_advisory_lock bool + err := row.Scan(&pg_try_advisory_lock) + return pg_try_advisory_lock, err +} diff --git a/db/sqlc/route_evidence.sql.go b/db/sqlc/route_evidence.sql.go new file mode 100644 index 00000000..0099c47b --- /dev/null +++ b/db/sqlc/route_evidence.sql.go @@ -0,0 +1,126 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: route_evidence.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const getRouteEvidenceRoute = `-- name: GetRouteEvidenceRoute :one +SELECT id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count, last_reconfirmed_at FROM known_routes +WHERE iata = $1::bpchar AND path_key = $2::bytea +` + +type GetRouteEvidenceRouteParams struct { + Iata string `json:"iata"` + PathKey []byte `json:"path_key"` +} + +func (q *Queries) GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) { + row := q.db.QueryRow(ctx, getRouteEvidenceRoute, arg.Iata, arg.PathKey) + var i KnownRoute + err := row.Scan( + &i.ID, + &i.PathKey, + &i.NodeIds, + &i.HashPrefix, + &i.Iata, + &i.HopCount, + &i.FirstSeen, + &i.LastSeen, + &i.ObservationCount, + &i.LastReconfirmedAt, + ) + return i, err +} + +const listRouteEvidence = `-- name: ListRouteEvidence :many +SELECT po.id, po.packet_hash, po.observer_id, o.display_name AS observer_name, + po.heard_at, po.payload_type, po.rssi, po.snr +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.iata = $1::bpchar + AND po.hash_size = $2::smallint + AND decode(md5(po.path_bytes), 'hex') = $3::bytea + AND po.path_bytes = $4::bytea + AND po.hop_count >= 2 + AND po.hop_count = $5::smallint + AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 + AND po.heard_at >= $6::timestamptz AND po.heard_at < $7::timestamptz + AND (po.heard_at, po.id) < ($8::timestamptz, $9::bigint) +ORDER BY po.heard_at DESC, po.id DESC +LIMIT $10::integer +` + +type ListRouteEvidenceParams struct { + Iata string `json:"iata"` + HashSize int16 `json:"hash_size"` + PathDigest []byte `json:"path_digest"` + PathBytes []byte `json:"path_bytes"` + HopCount int16 `json:"hop_count"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + BeforeAt pgtype.Timestamptz `json:"before_at"` + BeforeID int64 `json:"before_id"` + PageLimit int32 `json:"page_limit"` +} + +type ListRouteEvidenceRow struct { + ID int64 `json:"id"` + PacketHash []byte `json:"packet_hash"` + ObserverID uuid.UUID `json:"observer_id"` + ObserverName *string `json:"observer_name"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + PayloadType *int16 `json:"payload_type"` + Rssi *int16 `json:"rssi"` + Snr *float32 `json:"snr"` +} + +// Leading index equalities and the time/ID boundary bound both custom and generic plans. +// Full-byte equality is required even when the compact digest matches. TRACE path bytes +// carry readings; unclassified legacy observations cannot be safely called ordinary paths. +func (q *Queries) ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) { + rows, err := q.db.Query(ctx, listRouteEvidence, + arg.Iata, + arg.HashSize, + arg.PathDigest, + arg.PathBytes, + arg.HopCount, + arg.Since, + arg.Until, + arg.BeforeAt, + arg.BeforeID, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListRouteEvidenceRow{} + for rows.Next() { + var i ListRouteEvidenceRow + if err := rows.Scan( + &i.ID, + &i.PacketHash, + &i.ObserverID, + &i.ObserverName, + &i.HeardAt, + &i.PayloadType, + &i.Rssi, + &i.Snr, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/db/sqlc/series.sql.go b/db/sqlc/series.sql.go new file mode 100644 index 00000000..8bd7875c --- /dev/null +++ b/db/sqlc/series.sql.go @@ -0,0 +1,180 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: series.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getEarliestCompleteRollupHour = `-- name: GetEarliestCompleteRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete' +` + +func (q *Queries) GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + row := q.db.QueryRow(ctx, getEarliestCompleteRollupHour) + var column_1 pgtype.Timestamptz + err := row.Scan(&column_1) + return column_1, err +} + +const getLatestCompleteRollupHour = `-- name: GetLatestCompleteRollupHour :one +SELECT max(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete' +` + +func (q *Queries) GetLatestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + row := q.db.QueryRow(ctx, getLatestCompleteRollupHour) + var column_1 pgtype.Timestamptz + err := row.Scan(&column_1) + return column_1, err +} + +const getStatsSeries = `-- name: GetStatsSeries :many +WITH hours AS ( + SELECT g.h::timestamptz AS hour, COALESCE(r.status, 'missing')::text AS status + FROM generate_series($1::timestamptz, $2::timestamptz, INTERVAL '1 hour') g(h) + LEFT JOIN analytics_rollup_hours r ON r.hour = g.h + WHERE g.h < $2::timestamptz +), complete AS ( + SELECT hour FROM hours WHERE status = 'complete' +), obs AS ( + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE cardinality($3::bpchar[]) > 0 AND o.iata = ANY($3::bpchar[]) +), obs_agg AS ( + SELECT hour, sum(observation_count)::bigint AS observations, count(DISTINCT iata)::bigint AS active_iatas + FROM obs GROUP BY GROUPING SETS ((), (hour)) +), pkt AS ( + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[] +), pkt_agg AS ( + SELECT hour, sum(packets)::bigint AS unique_packets FROM pkt GROUP BY GROUPING SETS ((), (hour)) +), obsr AS ( + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE cardinality($3::bpchar[]) > 0 AND i.iata = ANY($3::bpchar[]) +), obsr_agg AS ( + SELECT hour, count(DISTINCT observer_id)::bigint AS active_observers FROM obsr GROUP BY GROUPING SETS ((), (hour)) +), scp AS ( + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[] +), scp_agg AS ( + SELECT hour, sum(packets)::bigint AS scoped_packets, count(DISTINCT scope_id)::bigint AS active_scopes + FROM scp GROUP BY GROUPING SETS ((), (hour)) +), pth AS ( + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND cardinality($3::bpchar[]) > 0 AND p.iata = ANY($3::bpchar[]) +), pth_agg AS ( + SELECT hour, max(entries)::integer AS max_path_entries FROM pth GROUP BY GROUPING SETS ((), (hour)) +), sig AS ( + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND cardinality($3::bpchar[]) > 0 AND s.iata = ANY($3::bpchar[]) +), sig_agg AS ( + SELECT hour, sum(snr_sum)::double precision AS snr_sum, sum(snr_samples)::bigint AS snr_samples, + sum(rssi_sum)::double precision AS rssi_sum, sum(rssi_samples)::bigint AS rssi_samples + FROM sig GROUP BY GROUPING SETS ((), (hour)) +), keys AS ( + SELECT hour, status FROM hours + UNION ALL + SELECT NULL::timestamptz, 'summary'::text +) +SELECT k.hour, k.status, + COALESCE(o.observations, 0)::bigint AS observations, + COALESCE(p.unique_packets, 0)::bigint AS unique_packets, + COALESCE(ob.active_observers, 0)::bigint AS active_observers, + COALESCE(o.active_iatas, 0)::bigint AS active_iatas, + COALESCE(s.scoped_packets, 0)::bigint AS scoped_packets, + COALESCE(s.active_scopes, 0)::bigint AS active_scopes, + COALESCE(pa.max_path_entries, 0)::integer AS max_path_entries, + COALESCE(sg.snr_sum, 0)::double precision AS snr_sum, + COALESCE(sg.snr_samples, 0)::bigint AS snr_samples, + COALESCE(sg.rssi_sum, 0)::double precision AS rssi_sum, + COALESCE(sg.rssi_samples, 0)::bigint AS rssi_samples +FROM keys k +LEFT JOIN obs_agg o ON o.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pkt_agg p ON p.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN obsr_agg ob ON ob.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN scp_agg s ON s.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pth_agg pa ON pa.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN sig_agg sg ON sg.hour IS NOT DISTINCT FROM k.hour +ORDER BY k.hour NULLS LAST +` + +type GetStatsSeriesParams struct { + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetStatsSeriesRow struct { + Hour pgtype.Timestamptz `json:"hour"` + Status string `json:"status"` + Observations int64 `json:"observations"` + UniquePackets int64 `json:"unique_packets"` + ActiveObservers int64 `json:"active_observers"` + ActiveIatas int64 `json:"active_iatas"` + ScopedPackets int64 `json:"scoped_packets"` + ActiveScopes int64 `json:"active_scopes"` + MaxPathEntries int32 `json:"max_path_entries"` + SnrSum float64 `json:"snr_sum"` + SnrSamples int64 `json:"snr_samples"` + RssiSum float64 `json:"rssi_sum"` + RssiSamples int64 `json:"rssi_samples"` +} + +// One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours. +// Counts sum; observers, IATAs and scopes are distinct across the window; packets count +// once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate. +func (q *Queries) GetStatsSeries(ctx context.Context, arg GetStatsSeriesParams) ([]GetStatsSeriesRow, error) { + rows, err := q.db.Query(ctx, getStatsSeries, arg.Since, arg.Until, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetStatsSeriesRow{} + for rows.Next() { + var i GetStatsSeriesRow + if err := rows.Scan( + &i.Hour, + &i.Status, + &i.Observations, + &i.UniquePackets, + &i.ActiveObservers, + &i.ActiveIatas, + &i.ScopedPackets, + &i.ActiveScopes, + &i.MaxPathEntries, + &i.SnrSum, + &i.SnrSamples, + &i.RssiSum, + &i.RssiSamples, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/db/sqlc/signal.sql.go b/db/sqlc/signal.sql.go new file mode 100644 index 00000000..abca971a --- /dev/null +++ b/db/sqlc/signal.sql.go @@ -0,0 +1,90 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: signal.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getSignalStats = `-- name: GetSignalStats :many +WITH readings AS ( + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz + AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz + AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) +) +SELECT (3 + 4 * grouping(hour))::integer AS kind, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + -1::integer AS snr_bin, -1::integer AS rssi_bin, + COALESCE(sum(receptions), 0)::bigint AS receptions, + COALESCE(sum(snr_samples), 0)::bigint AS snr_samples, + COALESCE(sum(snr_sum) / NULLIF(sum(snr_samples), 0), 0)::double precision AS snr_average, + COALESCE(sum(rssi_samples), 0)::bigint AS rssi_samples, + COALESCE(sum(rssi_sum) / NULLIF(sum(rssi_samples), 0), 0)::double precision AS rssi_average +FROM readings WHERE kind = 3 +GROUP BY GROUPING SETS ((), (hour)) +UNION ALL +SELECT (kind + 4)::integer, 'epoch'::timestamptz, snr_bin, rssi_bin, + sum(receptions)::bigint, 0::bigint, 0::double precision, 0::bigint, 0::double precision +FROM readings WHERE kind IN (1, 2) +GROUP BY kind, snr_bin, rssi_bin +ORDER BY kind, hour, snr_bin, rssi_bin +` + +type GetSignalStatsParams struct { + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetSignalStatsRow struct { + Kind int32 `json:"kind"` + Hour pgtype.Timestamptz `json:"hour"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrAverage float64 `json:"snr_average"` + RssiSamples int64 `json:"rssi_samples"` + RssiAverage float64 `json:"rssi_average"` +} + +// Read the hourly rollup, never observations on an HTTP request. +// Weight averages by sample counts instead of averaging regional/hourly means. +func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) { + rows, err := q.db.Query(ctx, getSignalStats, arg.Since, arg.Until, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetSignalStatsRow{} + for rows.Next() { + var i GetSignalStatsRow + if err := rows.Scan( + &i.Kind, + &i.Hour, + &i.SnrBin, + &i.RssiBin, + &i.Receptions, + &i.SnrSamples, + &i.SnrAverage, + &i.RssiSamples, + &i.RssiAverage, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/db/sqlc_scratch/rollup_obs.sql b/db/sqlc_scratch/rollup_obs.sql new file mode 100644 index 00000000..660e325b --- /dev/null +++ b/db/sqlc_scratch/rollup_obs.sql @@ -0,0 +1,9 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- sqlc type-check stub only; db/rollup.go creates this as a TEMP table per rolled hour. +CREATE TABLE rollup_obs ( + packet_hash bytea NOT NULL, observer_id uuid NOT NULL, iata character(3) NOT NULL, + heard_at timestamptz NOT NULL, path_length_byte smallint NOT NULL, hash_size smallint NOT NULL, + hop_count smallint NOT NULL, path_len integer NOT NULL, rssi smallint, snr real, airtime_ms real, + payload_type smallint, route_type smallint NOT NULL, origin_pubkey bytea, scope_id integer); diff --git a/db/stats.go b/db/stats.go index 78c79848..a0907b63 100644 --- a/db/stats.go +++ b/db/stats.go @@ -5,36 +5,62 @@ package db import ( "context" + "encoding/json" + "log/slog" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) +// rollupSince snaps a window start to its UTC hour; zero means def before now. +func rollupSince(since time.Time, def time.Duration) pgtype.Timestamptz { + if since.IsZero() { + since = time.Now().Add(-def) + } + return ts(since.UTC().Truncate(time.Hour)) +} + +// optional maps the ” the rollup queries use for a missing name back to nil. +func optional(s string) *string { + if s == "" { + return nil + } + return &s +} + +func pgUUID(id pgtype.UUID) *uuid.UUID { + if !id.Valid { + return nil + } + u := uuid.UUID(id.Bytes) + return &u +} + +// GetStatsOverview summarises the 24 most recent hours that can have been rolled; +// WindowHours counts the complete ones the totals actually cover. func (s *Store) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { - row, err := s.q.GetStatsOverview(ctx, iatas) + until := time.Now().UTC().Add(-rollupDelay).Truncate(time.Hour).Add(time.Hour) + since := until.Add(-24 * time.Hour) + series, err := s.GetStatsSeries(ctx, since, until, iatas) if err != nil { return nil, err } return &api.StatsOverview{ - TotalPackets: row.TotalPackets, - TotalObservations: row.TotalObservations, - ActiveObservers: row.ActiveObservers, - ActiveIATAs: row.ActiveIatas, - WindowHours: 24, + TotalPackets: series.Summary.UniquePackets, + TotalObservations: series.Summary.Observations, + ActiveObservers: series.Summary.ActiveObservers, + ActiveIATAs: series.Summary.ActiveIATAs, + WindowHours: series.CompleteHours, + Since: since.UnixMilli(), + Until: until.UnixMilli(), }, nil } func (s *Store) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { - if since.IsZero() { - since = time.Now().Add(-7 * 24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetHourlyStats(ctx, sqlc.GetHourlyStatsParams{ - Column1: iatas, - Column2: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - }) + rows, err := s.q.GetHourlyStats(ctx, sqlc.GetHourlyStatsParams{Iatas: iatas, Since: rollupSince(since, 7*24*time.Hour)}) if err != nil { return nil, err } @@ -44,60 +70,42 @@ func (s *Store) GetStatsObservations(ctx context.Context, iatas []string, since Hour: v.Hour.Time.UnixMilli(), IATA: v.Iata, ObservationCount: v.ObservationCount, - UniquePackets: v.UniquePackets, - ActiveObservers: v.ActiveObservers, }) } return points, nil } func (s *Store) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsPayloadBreakdown(ctx, sqlc.GetStatsPayloadBreakdownParams{ - Column1: iatas, - Column2: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - }) + rows, err := s.q.GetStatsPayloadBreakdown(ctx, sqlc.GetStatsPayloadBreakdownParams{Iatas: iatas, Since: rollupSince(since, 24*time.Hour)}) if err != nil { return nil, err } items := make([]api.PayloadBreakdownItem, 0, len(rows)) for _, v := range rows { - if v.PayloadType == nil { - continue - } items = append(items, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: v.PayloadType, + PayloadTypeName: api.PayloadTypeName(v.PayloadType), Count: v.Count, }) } return items, nil } -func (s *Store) GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { - rows, err := s.q.GetTopNodes(ctx, sqlc.GetTopNodesParams{ - Column1: iatas, - Limit: limit, - }) +func (s *Store) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { + rows, err := s.q.GetTopNodes(ctx, sqlc.GetTopNodesParams{Since: rollupSince(since, 7*24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } items := make([]api.TopNode, 0, len(rows)) for _, v := range rows { - var count int64 - if v.ObservationCount != nil { - count = *v.ObservationCount - } items = append(items, api.TopNode{ - NodeID: v.NodeID, - NodeName: v.Name, + NodeID: pgUUID(v.NodeID), + PublicKey: v.PublicKey, + NodeName: optional(v.Name), NodeType: v.NodeType, NodeTypeName: api.NodeTypeName(v.NodeType), IATA: v.Iata, - ObservationCount: count, + ObservationCount: v.ObservationCount, LastHeard: v.LastHeard.Time.UnixMilli(), }) } @@ -105,15 +113,7 @@ func (s *Store) GetStatsTopNodes(ctx context.Context, iatas []string, limit int3 } func (s *Store) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{ - Column1: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - Column2: iatas, - Limit: limit, - }) + rows, err := s.q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{Since: rollupSince(since, 24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } @@ -121,8 +121,8 @@ func (s *Store) GetStatsTopObservers(ctx context.Context, iatas []string, since for _, v := range rows { items = append(items, api.TopObserver{ ObserverID: v.ID, - DisplayName: v.DisplayName, - ObserverType: v.ObserverType, + DisplayName: optional(v.DisplayName), + ObserverType: optional(v.ObserverType), IATA: v.Iata, ObservationCount: v.ObservationCount, }) @@ -131,54 +131,70 @@ func (s *Store) GetStatsTopObservers(ctx context.Context, iatas []string, since } func (s *Store) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsTopAdvertisers(ctx, sqlc.GetStatsTopAdvertisersParams{ - Column1: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - Column2: iatas, - Limit: limit, - }) + rows, err := s.q.GetStatsTopAdvertisers(ctx, sqlc.GetStatsTopAdvertisersParams{Since: rollupSince(since, 24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } items := make([]api.TopAdvertiser, 0, len(rows)) for _, v := range rows { items = append(items, api.TopAdvertiser{ - NodeID: v.ID, - NodeName: v.Name, - NodeType: v.NodeType, - NodeTypeName: api.NodeTypeName(v.NodeType), - IATA: v.Iata, - AdvertCount: v.AdvertCount, - LastHeard: v.LastHeard.Time.UnixMilli(), + NodeID: pgUUID(v.NodeID), + PublicKey: v.PublicKey, + NodeName: optional(v.Name), + NodeType: v.NodeType, + NodeTypeName: api.NodeTypeName(v.NodeType), + IATA: v.Iata, + AdvertCount: v.AdvertCount, + FloodAdvertCount: v.FloodAdvertCount, + DirectAdvertCount: v.DirectAdvertCount, + LastHeard: v.LastHeard.Time.UnixMilli(), }) } return items, nil } -func (s *Store) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsTopTalkers(ctx, sqlc.GetStatsTopTalkersParams{ - Column1: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, +// GetStatsClockDrift returns repeaters/room servers whose current advert-derived clock +// drift exceeds the Store's configured threshold, worst first. +func (s *Store) GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { + thresholdSeconds := int32(s.clockDriftThreshold / time.Second) + rows, err := s.q.GetStatsClockDrift(ctx, sqlc.GetStatsClockDriftParams{ + Column1: thresholdSeconds, Column2: iatas, Limit: limit, }) if err != nil { return nil, err } - items := make([]api.TopTalker, 0, len(rows)) + items := make([]api.ClockDriftEntry, 0, len(rows)) for _, v := range rows { - var senderName string - if v.SenderName != nil { - senderName = *v.SenderName + entry := api.ClockDriftEntry{ + NodeID: v.ID, + NodeName: v.Name, + NodeType: v.NodeType, + NodeTypeName: api.NodeTypeName(v.NodeType), + ClockDriftSeconds: int(*v.DeviceClockDriftSeconds), + ClockCheckedAt: v.LastAdvertAt.Time.UnixMilli(), } + if len(v.Iatas) > 0 { + if err := json.Unmarshal(v.Iatas, &entry.IATAs); err != nil { + slog.Error("store: failed to unmarshal clock drift node iatas", "component", "db", "error", err) + entry.IATAs = []api.NodeIATA{} + } + } + items = append(items, entry) + } + return items, nil +} + +func (s *Store) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { + rows, err := s.q.GetStatsTopTalkers(ctx, sqlc.GetStatsTopTalkersParams{Since: rollupSince(since, 24*time.Hour), Iatas: iatas, RowLimit: limit}) + if err != nil { + return nil, err + } + items := make([]api.TopTalker, 0, len(rows)) + for _, v := range rows { items = append(items, api.TopTalker{ - SenderName: senderName, + SenderName: v.SenderName, MessageCount: v.MessageCount, LastSent: v.LastSent.Time.UnixMilli(), }) @@ -196,9 +212,12 @@ func (s *Store) GetRadioPresets(ctx context.Context, preset string, iatas []stri } items := make([]api.RadioPreset, 0, len(rows)) for _, v := range rows { + if v.Iata == nil { + continue + } items = append(items, api.RadioPreset{ Preset: v.Preset, - IATA: v.Iata, + IATA: *v.Iata, SourceType: v.SourceType, Count: v.Count, }) @@ -206,18 +225,32 @@ func (s *Store) GetRadioPresets(ctx context.Context, preset string, iatas []stri return items, nil } -func (s *Store) GetScopeStats(ctx context.Context) ([]api.ScopeStats, error) { - rows, err := s.q.GetScopeStats(ctx) +func (s *Store) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { + start := rollupSince(since, 7*24*time.Hour) + rows, err := s.q.GetScopeStats(ctx, sqlc.GetScopeStatsParams{Since: start, Iatas: iatas}) if err != nil { return nil, err } + hours, err := s.q.GetScopeStatsHourly(ctx, sqlc.GetScopeStatsHourlyParams{Since: start, Iatas: iatas}) + if err != nil { + return nil, err + } + byScope := make(map[string][]api.ScopeHour) + for _, h := range hours { + byScope[h.Name] = append(byScope[h.Name], api.ScopeHour{Hour: h.Hour.Time.UnixMilli(), Packets: h.Packets, Observers: h.Observers, Nodes: h.Nodes}) + } items := make([]api.ScopeStats, 0, len(rows)) for _, r := range rows { + hourly := byScope[r.Name] + if hourly == nil { + hourly = []api.ScopeHour{} + } items = append(items, api.ScopeStats{ Name: r.Name, PacketCount: r.PacketCount, ObserverCount: r.ObserverCount, NodeCount: r.NodeCount, + Hourly: hourly, }) } return items, nil @@ -239,30 +272,6 @@ func (s *Store) GetStatsNodeTypes(ctx context.Context, iatas []string) ([]api.No return result, nil } -func (s *Store) RefreshHourlyStats(ctx context.Context) error { - return s.q.RefreshHourlyStats(ctx) -} - -func (s *Store) RefreshTopNodes(ctx context.Context) error { - return s.q.RefreshTopNodes(ctx) -} - -func (s *Store) RefreshPayloadBreakdown(ctx context.Context) error { - return s.q.RefreshPayloadBreakdown(ctx) -} - -func (s *Store) RefreshTopTalkers(ctx context.Context) error { - return s.q.RefreshTopTalkers(ctx) -} - -func (s *Store) RefreshTopAdvertisers(ctx context.Context) error { - return s.q.RefreshTopAdvertisers(ctx) -} - -func (s *Store) RefreshTopObservers(ctx context.Context) error { - return s.q.RefreshTopObservers(ctx) -} - func (s *Store) RefreshRadioPresets(ctx context.Context) error { return s.q.RefreshRadioPresets(ctx) } diff --git a/db/stats_series.go b/db/stats_series.go new file mode 100644 index 00000000..c905f559 --- /dev/null +++ b/db/stats_series.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func (s *Store) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { + // Read the revision first: a roll landing mid-request then shows up as a newer revision next time. + rev, err := s.q.GetAnalyticsRevision(ctx) + if err != nil { + return nil, err + } + earliest, err := s.q.GetEarliestCompleteRollupHour(ctx) + if err != nil { + return nil, err + } + rows, err := s.q.GetStatsSeries(ctx, sqlc.GetStatsSeriesParams{Since: ts(since), Until: ts(until), Iatas: iatas}) + if err != nil { + return nil, err + } + out := &api.StatsSeries{ + Since: since.UnixMilli(), + Until: until.UnixMilli(), + Revision: rev, + Hours: make([]api.StatsSeriesHour, 0, len(rows)), + } + if earliest.Valid { + ms := earliest.Time.UnixMilli() + out.EarliestComplete = &ms + } + for _, r := range rows { + v := api.StatsSeriesValues{ + Observations: r.Observations, + UniquePackets: r.UniquePackets, + ActiveObservers: r.ActiveObservers, + ActiveIATAs: r.ActiveIatas, + ScopedPackets: r.ScopedPackets, + ActiveScopes: r.ActiveScopes, + MaxPathEntries: r.MaxPathEntries, + SNRSum: r.SnrSum, + SNRSamples: r.SnrSamples, + RSSISum: r.RssiSum, + RSSISamples: r.RssiSamples, + } + if !r.Hour.Valid { + out.Summary = v + continue + } + h := api.StatsSeriesHour{Hour: r.Hour.Time.UnixMilli(), Status: r.Status} + if r.Status == "complete" { + h.Values = &v + out.CompleteHours++ + } + out.Hours = append(out.Hours, h) + } + return out, nil +} diff --git a/db/stats_series_integration_test.go b/db/stats_series_integration_test.go new file mode 100644 index 00000000..627d5e1f --- /dev/null +++ b/db/stats_series_integration_test.go @@ -0,0 +1,126 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// seriesOracle computes a summary straight from raw rows over [since, until). +const seriesOracle = ` +WITH po AS ( + SELECT po.*, date_trunc('hour', po.heard_at, 'UTC') AS hour, p.scope_id + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.heard_at >= $1 AND po.heard_at < $2 + AND (cardinality($3::bpchar[]) = 0 OR po.iata = ANY($3::bpchar[])) + AND date_trunc('hour', po.heard_at, 'UTC') <> ALL($4::timestamptz[]) +), sig AS ( + SELECT * FROM po WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) +) +SELECT (SELECT count(*) FROM po), + (SELECT count(*) FROM (SELECT DISTINCT hour, packet_hash FROM po) d), + (SELECT count(DISTINCT observer_id) FROM po), + (SELECT count(DISTINCT iata) FROM po), + (SELECT count(*) FROM (SELECT DISTINCT hour, packet_hash FROM po WHERE scope_id IS NOT NULL) d), + (SELECT count(DISTINCT scope_id) FROM po), + (SELECT COALESCE(sum(snr), 0)::double precision FROM sig), (SELECT count(snr) FROM sig), + (SELECT COALESCE(sum(rssi), 0)::double precision FROM sig), (SELECT count(rssi) FROM sig)` + +func TestStatsSeriesPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + r.Close(ctx) + since, until := h0, h0.Add(3*time.Hour) + + for _, tc := range []struct { + name string + iatas []string + excluded []time.Time // hours not complete + maxPath int32 + }{ + {"all", nil, nil, 2}, + {"one IATA", []string{"YYZ"}, nil, 1}, + {"overlapping IATAs", []string{"YVR", "YYZ", "YVR"}, nil, 2}, + {"empty region", []string{""}, nil, 0}, + {"partial hour excluded", nil, []time.Time{h0.Add(time.Hour)}, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + for _, h := range tc.excluded { + if _, err := tx.Exec(ctx, "UPDATE analytics_rollup_hours SET status = 'partial' WHERE hour = $1", h); err != nil { + t.Fatal(err) + } + } + got, err := (&Store{q: sqlc.New(tx)}).GetStatsSeries(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + var want api.StatsSeriesValues + excluded := tc.excluded + if excluded == nil { + excluded = []time.Time{} + } + iatas := tc.iatas + if iatas == nil { + iatas = []string{} + } + if err := tx.QueryRow(ctx, seriesOracle, since, until, iatas, excluded).Scan(&want.Observations, &want.UniquePackets, + &want.ActiveObservers, &want.ActiveIATAs, &want.ScopedPackets, &want.ActiveScopes, + &want.SNRSum, &want.SNRSamples, &want.RSSISum, &want.RSSISamples); err != nil { + t.Fatal(err) + } + want.MaxPathEntries = tc.maxPath + if got.Summary != want { + t.Errorf("summary\n got %+v\nwant %+v", got.Summary, want) + } + if len(got.Hours) != 3 || got.Since != since.UnixMilli() || got.Until != until.UnixMilli() || got.EarliestComplete == nil { + t.Fatalf("series shape: %+v", got) + } + var obs, packets int64 + for _, h := range got.Hours { + partial := false + for _, e := range tc.excluded { + partial = partial || e.UnixMilli() == h.Hour + } + if partial != (h.Values == nil) || (partial && h.Status != "partial") { + t.Errorf("hour %d status %s values %v", h.Hour, h.Status, h.Values) + } + if h.Values != nil { + obs += h.Values.Observations + packets += h.Values.UniquePackets + } + } + // Cards and sparklines must agree for additive metrics. + if obs != got.Summary.Observations || packets != got.Summary.UniquePackets { + t.Errorf("hours sum to %d observations / %d packets, summary %d / %d", obs, packets, got.Summary.Observations, got.Summary.UniquePackets) + } + if got.CompleteHours != 3-len(tc.excluded) { + t.Errorf("complete hours %d", got.CompleteHours) + } + }) + } + + // Hours never registered (or not yet eligible) are missing. + got, err := store.GetStatsSeries(ctx, h0.Add(-2*time.Hour), h0, nil) + if err != nil || len(got.Hours) != 2 || got.Hours[0].Status != "missing" || got.Hours[0].Values != nil || got.CompleteHours != 0 { + t.Errorf("unregistered hours: %+v %v", got, err) + } +} diff --git a/db/stats_test.go b/db/stats_test.go index fec74d11..dad24c83 100644 --- a/db/stats_test.go +++ b/db/stats_test.go @@ -19,62 +19,72 @@ func TestGetStatsOverview(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetAnalyticsRevision(gomock.Any()).Return(int64(7), nil) + mock.EXPECT().GetEarliestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) + var params sqlc.GetStatsSeriesParams mock.EXPECT(). - GetStatsOverview(gomock.Any(), []string{"YVR"}). - Return(sqlc.GetStatsOverviewRow{ - TotalPackets: 100, - TotalObservations: 500, - ActiveObservers: 10, - ActiveIatas: 3, - }, nil) + GetStatsSeries(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, p sqlc.GetStatsSeriesParams) ([]sqlc.GetStatsSeriesRow, error) { + params = p + rows := []sqlc.GetStatsSeriesRow{{Status: "summary", Observations: 500, UniquePackets: 100, ActiveObservers: 10, ActiveIatas: 3}} + // The newest hour isn't rolled yet; the summary covers the other 23. + for h := p.Since.Time; h.Before(p.Until.Time); h = h.Add(time.Hour) { + status := "complete" + if h.Add(time.Hour).Equal(p.Until.Time) { + status = "missing" + } + rows = append(rows, sqlc.GetStatsSeriesRow{Hour: pgtype.Timestamptz{Time: h, Valid: true}, Status: status}) + } + return rows, nil + }) store := &Store{q: mock} result, err := store.GetStatsOverview(context.Background(), []string{"YVR"}) if err != nil { t.Fatalf("unexpected error: %v", err) } - if result.TotalPackets != 100 { - t.Errorf("expected TotalPackets 100, got %d", result.TotalPackets) + if result.TotalPackets != 100 || result.TotalObservations != 500 || result.ActiveObservers != 10 || result.ActiveIATAs != 3 { + t.Errorf("unexpected summary mapping: %+v", result) + } + if result.WindowHours != 23 || result.Until-result.Since != (24*time.Hour).Milliseconds() { + t.Errorf("window %d..%d (%d hours)", result.Since, result.Until, result.WindowHours) } - if result.WindowHours != 24 { - t.Errorf("expected WindowHours 24, got %d", result.WindowHours) + // The window ends at the last hour that can have been rolled. + wantUntil := time.Now().UTC().Add(-rollupDelay).Truncate(time.Hour).Add(time.Hour) + if !params.Until.Time.Equal(wantUntil) || len(params.Iatas) != 1 { + t.Errorf("series params %+v, want until %s", params, wantUntil) } } -func TestGetStatsTopNodes_NilObservationCount(t *testing.T) { +func TestGetStatsTopNodes_MissingNodeRow(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000001") lastHeard := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} - name := "test-node" + since := time.Date(2026, 1, 1, 12, 34, 0, 0, time.UTC) mock.EXPECT(). GetTopNodes(gomock.Any(), sqlc.GetTopNodesParams{ - Column1: []string{"YVR"}, - Limit: 5, + Since: pgtype.Timestamptz{Time: since.Truncate(time.Hour), Valid: true}, + Iatas: []string{"YVR"}, + RowLimit: 5, }). - Return([]sqlc.MvTopNodesByIatum{ - { - NodeID: nodeID, - Name: &name, - NodeType: 1, - Iata: "YVR", - ObservationCount: nil, - LastHeard: lastHeard, - }, + Return([]sqlc.GetTopNodesRow{ + {NodeID: pgtype.UUID{Bytes: nodeID, Valid: true}, PublicKey: "aa", Name: "test-node", NodeType: 1, Iata: "YVR", ObservationCount: 9, LastHeard: lastHeard}, + {PublicKey: "bb", Iata: "YVR", ObservationCount: 3, LastHeard: lastHeard}, }, nil) store := &Store{q: mock} - items, err := store.GetStatsTopNodes(context.Background(), []string{"YVR"}, 5) + items, err := store.GetStatsTopNodes(context.Background(), []string{"YVR"}, since, 5) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(items) != 1 { - t.Fatalf("expected 1 item, got %d", len(items)) + if len(items) != 2 || items[0].NodeID == nil || *items[0].NodeID != nodeID || *items[0].NodeName != "test-node" { + t.Fatalf("unexpected first node: %+v", items) } - if items[0].ObservationCount != 0 { - t.Errorf("expected ObservationCount 0 for nil, got %d", items[0].ObservationCount) + if items[1].NodeID != nil || items[1].NodeName != nil || items[1].PublicKey != "bb" { + t.Errorf("deleted node should keep its key and drop id/name: %+v", items[1]) } } @@ -91,8 +101,8 @@ func TestGetStatsTopObservers_IATATypeAssertion(t *testing.T) { Return([]sqlc.GetStatsTopObserversRow{ { ID: observerID, - DisplayName: &displayName, - ObserverType: &obsType, + DisplayName: displayName, + ObserverType: obsType, Iata: "YVR", ObservationCount: 42, }, @@ -114,6 +124,91 @@ func TestGetStatsTopObservers_IATATypeAssertion(t *testing.T) { } } +func TestGetStatsTopAdvertisers_FloodDirectSplit(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000003") + name := "test-node" + heardAt := pgtype.Timestamptz{Time: time.Now(), Valid: true} + + mock.EXPECT(). + GetStatsTopAdvertisers(gomock.Any(), gomock.Any()). + Return([]sqlc.GetStatsTopAdvertisersRow{ + { + NodeID: pgtype.UUID{Bytes: nodeID, Valid: true}, + PublicKey: "cc", + Name: name, + NodeType: 2, // repeater + AdvertCount: 10, + FloodAdvertCount: 7, + DirectAdvertCount: 3, + LastHeard: heardAt, + Iata: "YVR", + }, + }, nil) + + store := &Store{q: mock} + items, err := store.GetStatsTopAdvertisers(context.Background(), []string{"YVR"}, time.Now().Add(-time.Hour), 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + if items[0].AdvertCount != 10 { + t.Errorf("expected AdvertCount 10, got %d", items[0].AdvertCount) + } + if items[0].FloodAdvertCount != 7 { + t.Errorf("expected FloodAdvertCount 7, got %d", items[0].FloodAdvertCount) + } + if items[0].DirectAdvertCount != 3 { + t.Errorf("expected DirectAdvertCount 3, got %d", items[0].DirectAdvertCount) + } + if items[0].FloodAdvertCount+items[0].DirectAdvertCount != items[0].AdvertCount { + t.Error("expected FloodAdvertCount + DirectAdvertCount to equal AdvertCount") + } +} + +func TestGetStatsClockDrift_Mapping(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000004") + name := "drifty-repeater" + drift := int32(-600) + checkedAt := pgtype.Timestamptz{Time: time.Now(), Valid: true} + iatasJSON := []byte(`[{"iata":"YVR","lastHeard":1700000000000}]`) + + mock.EXPECT(). + GetStatsClockDrift(gomock.Any(), gomock.Any()). + Return([]sqlc.GetStatsClockDriftRow{ + { + ID: nodeID, + Name: &name, + NodeType: 2, // repeater + DeviceClockDriftSeconds: &drift, + LastAdvertAt: checkedAt, + Iatas: iatasJSON, + }, + }, nil) + + store := &Store{q: mock, clockDriftThreshold: 5 * time.Minute} + items, err := store.GetStatsClockDrift(context.Background(), []string{"YVR"}, 10) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + if items[0].ClockDriftSeconds != -600 { + t.Errorf("expected ClockDriftSeconds -600, got %d", items[0].ClockDriftSeconds) + } + if len(items[0].IATAs) != 1 || items[0].IATAs[0].IATA != "YVR" { + t.Errorf("expected 1 IATA entry for YVR, got %v", items[0].IATAs) + } +} + func TestGetStatsNodeTypes_Mapping(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) @@ -143,18 +238,32 @@ func TestGetScopeStats(t *testing.T) { mock := mockdb.NewMockQuerier(ctrl) mock.EXPECT(). - GetScopeStats(gomock.Any()). + GetScopeStats(gomock.Any(), gomock.Any()). Return([]sqlc.GetScopeStatsRow{ {Name: "default", PacketCount: 100, ObserverCount: 5, NodeCount: 20}, + {Name: "quiet"}, + }, nil) + hour := time.Date(2026, 1, 1, 3, 0, 0, 0, time.UTC) + mock.EXPECT(). + GetScopeStatsHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetScopeStatsHourlyRow{ + {Name: "default", Hour: pgtype.Timestamptz{Time: hour, Valid: true}, Packets: 60}, + {Name: "default", Hour: pgtype.Timestamptz{Time: hour.Add(time.Hour), Valid: true}, Packets: 40}, }, nil) store := &Store{q: mock} - items, err := store.GetScopeStats(context.Background()) + items, err := store.GetScopeStats(context.Background(), []string{"YVR"}, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(items) != 1 { - t.Fatalf("expected 1 item, got %d", len(items)) + if len(items) != 2 { + t.Fatalf("expected 2 items, got %d", len(items)) + } + if h := items[0].Hourly; len(h) != 2 || h[0].Hour != hour.UnixMilli() || h[0].Packets+h[1].Packets != items[0].PacketCount { + t.Errorf("hourly %+v doesn't split packetCount %d", h, items[0].PacketCount) + } + if items[1].Hourly == nil || len(items[1].Hourly) != 0 { + t.Errorf("a scope without packets must have hourly [], got %#v", items[1].Hourly) } if items[0].Name != "default" { t.Errorf("expected Name default, got %s", items[0].Name) diff --git a/db/store.go b/db/store.go index d2699a7b..c0f4c9f3 100644 --- a/db/store.go +++ b/db/store.go @@ -22,6 +22,7 @@ import ( // Store wraps the sqlc-generated Queries and implements both ingest.DB and api.Reader. type Store struct { q sqlc.Querier + pool *pgxpool.Pool // rollups need a dedicated connection; nil in mock-backed tests clockDriftThreshold time.Duration // see api.Node.ClockOutOfSync staleThreshold time.Duration // see api.NodeSummary.Stale } @@ -32,7 +33,7 @@ type Store struct { // staleThreshold is how long since last_seen before a node's Stale is reported true; see // internal/config.ResolvedConfig.NodeStaleThreshold. func New(pool *pgxpool.Pool, clockDriftThreshold, staleThreshold time.Duration) *Store { - return &Store{q: sqlc.New(pool), clockDriftThreshold: clockDriftThreshold, staleThreshold: staleThreshold} + return &Store{q: sqlc.New(pool), pool: pool, clockDriftThreshold: clockDriftThreshold, staleThreshold: staleThreshold} } func (s *Store) ResolvePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { @@ -83,6 +84,32 @@ func (s *Store) ResolvePathHashes(ctx context.Context, iata string, hashes [][]b return result, nil } +// ResolveEndpointHashes matches one-byte packet endpoints across advertised roles. +// Keep this separate from the infrastructure-only intermediate path resolver. +func (s *Store) ResolveEndpointHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + if len(hashes) == 0 { + return nil, nil + } + for _, hash := range hashes { + if len(hash) != 1 { + return nil, nil + } + } + rows, err := s.q.ResolveEndpointHashes(ctx, sqlc.ResolveEndpointHashesParams{Iata: iata, Column2: hashes}) + if err != nil { + return nil, err + } + result := make(map[string][]api.ResolvedPathEntry) + for _, row := range rows { + key := hex.EncodeToString(row.Hash) + result[key] = append(result[key], api.ResolvedPathEntry{ + NodeID: row.NodeID, Name: row.Name, Latitude: row.Latitude, + Longitude: row.Longitude, PublicKey: row.PublicKey, + }) + } + return result, nil +} + // nullableUUID returns nil for a zero UUID, or a pointer to the UUID otherwise. func nullableUUID(id uuid.UUID) *uuid.UUID { if id == (uuid.UUID{}) { @@ -104,7 +131,7 @@ func tristate(b *bool) string { } // toChannelMessage maps raw sqlc row fields to an api.ChannelMessage. -func toChannelMessage(id int64, packetHashHex string, channelHash []byte, senderName *string, content *string, sentAt pgtype.Timestamptz, observationCount int64) api.ChannelMessage { +func toChannelMessage(id int64, packetHashHex string, channelHash []byte, senderName *string, content *string, sentAt pgtype.Timestamptz, observationCount int64, scope *string, transport *bool) api.ChannelMessage { sn := "" if senderName != nil { sn = *senderName @@ -121,5 +148,23 @@ func toChannelMessage(id int64, packetHashHex string, channelHash []byte, sender Content: ct, SentAt: sentAt.Time.UnixMilli(), ObservationCount: observationCount, + Scope: scope, + ScopeStatus: api.RecordedChannelScopeStatus(scope, transport), + } +} + +// deleteInBatches repeats a delete until a batch comes back short. +func deleteInBatches(ctx context.Context, batchSize int32, del func(context.Context, int32) (int64, error)) error { + for { + n, err := del(ctx, batchSize) + if err != nil { + return err + } + if n < int64(batchSize) { + return nil + } + if err := ctx.Err(); err != nil { + return err + } } } diff --git a/db/store_test.go b/db/store_test.go index 8bfab391..41f43938 100644 --- a/db/store_test.go +++ b/db/store_test.go @@ -59,7 +59,7 @@ func TestToChannelMessage(t *testing.T) { channelHash := []byte{0xab} sentAt := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} - msg := toChannelMessage(42, "deadbeef", channelHash, &senderName, &content, sentAt, 7) + msg := toChannelMessage(42, "deadbeef", channelHash, &senderName, &content, sentAt, 7, nil, nil) if msg.ID != 42 { t.Errorf("expected ID 42, got %d", msg.ID) @@ -86,7 +86,7 @@ func TestToChannelMessage(t *testing.T) { func TestToChannelMessage_NilFields(t *testing.T) { sentAt := pgtype.Timestamptz{Time: time.UnixMilli(0), Valid: true} - msg := toChannelMessage(1, "abc", []byte{0x01}, nil, nil, sentAt, 0) + msg := toChannelMessage(1, "abc", []byte{0x01}, nil, nil, sentAt, 0, nil, nil) if msg.SenderName != "" { t.Errorf("expected empty SenderName, got %s", msg.SenderName) } @@ -179,3 +179,31 @@ func TestResolvePathHashes_Mapping(t *testing.T) { t.Errorf("expected Name %s, got %v", name, entries[0].Name) } } + +func TestDeleteInBatches_StopsOnError(t *testing.T) { + boom := errors.New("boom") + calls := 0 + err := deleteInBatches(context.Background(), 10, func(context.Context, int32) (int64, error) { + calls++ + if calls == 2 { + return 0, boom + } + return 10, nil + }) + if !errors.Is(err, boom) || calls != 2 { + t.Fatalf("err=%v calls=%d, want boom after 2 calls", err, calls) + } +} + +func TestDeleteInBatches_StopsWhenCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + err := deleteInBatches(ctx, 10, func(context.Context, int32) (int64, error) { + calls++ + cancel() + return 10, nil + }) + if !errors.Is(err, context.Canceled) || calls != 1 { + t.Fatalf("err=%v calls=%d, want context.Canceled after 1 call", err, calls) + } +} diff --git a/db/trace_pagination_integration_test.go b/db/trace_pagination_integration_test.go new file mode 100644 index 00000000..90ed19d5 --- /dev/null +++ b/db/trace_pagination_integration_test.go @@ -0,0 +1,258 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/hex" + "encoding/json" + "os" + "reflect" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" +) + +// Trace tags, not individual packets, are the entities being paginated. +func TestTraceTagPaginationPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) VALUES + (1,'scope-one',decode(repeat('01',16),'hex'),decode(repeat('01',8),'hex')), + (2,'scope-two',decode(repeat('02',16),'hex'),decode(repeat('02',8),'hex'))`); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + at := func(seconds int) time.Time { + if seconds == 0 { + return time.Time{} + } + return anchor.Add(time.Duration(seconds) * time.Second) + } + // Each packet is heard once when new and again (duplicate) at its last time. + for _, p := range []struct { + tag string + iata string + scope int32 + first, last int + kind string + hashes []string + }{ + {"aaaaaaaa", "YYZ", 1, 1, 40, "TRACE", []string{"aa"}}, + {"aaaaaaaa", "YVR", 1, 2, 10, "TRACE", []string{"aa", "bb"}}, + {"aaaaaaaa", "YVR", 2, 3, 25, "PING", []string{"aa", "bb", "cc"}}, + {"bbbbbbbb", "YYZ", 1, 4, 30, "TRACE", []string{"bb", "11", "22", "33"}}, + {"bbbbbbbb", "YYZ", 2, 5, 5, "PING", []string{"bb"}}, + {"cccccccc", "YVR", 1, 6, 20, "TRACE", []string{"cc"}}, + {"dddddddd", "YOW", 2, 7, 15, "PING", []string{"dd"}}, + } { + tag, _ := hex.DecodeString(p.tag) + snr := make([]float32, len(p.hashes)) + for i := range snr { + snr[i] = float32(i + int(p.tag[0]-'a') + 1) + } + payload, _ := json.Marshal(map[string]any{"type": p.kind, "pathHashes": p.hashes, "snrValues": snr}) + scope := p.scope + hash := append(append([]byte{}, tag...), byte(len(p.hashes)), byte(p.first)) + if _, err := store.UpsertPacket(ctx, ingest.UpsertPacketParams{PacketHash: hash, PayloadType: 9, RouteType: 1, + RawPayload: []byte{0}, RawHeader: []byte{0}, ParsedPayload: payload, ScopeID: &scope, TraceTag: tag}); err != nil { + t.Fatal(err) + } + for _, heard := range []int{p.first, p.last} { + if err := store.RecordTrace(ctx, ingest.TraceHearing{TraceTag: tag, IATA: p.iata, HeardAt: at(heard)}); err != nil { + t.Fatal(err) + } + } + } + paths := map[string][]string{"a": {"aa", "bb", "cc"}, "b": {"bb", "11", "22", "33"}, "c": {"cc"}, "d": {"dd"}} + snrs := map[string][]float32{"a": {1, 2, 3}, "b": {2, 3, 4, 5}, "c": {3}, "d": {4}} + summary := func(tag string, first, last int, count, iatas int64, kind string) api.TraceTagSummary { + return api.TraceTagSummary{TraceTag: strings.Repeat(tag, 8), FirstHeardAt: at(first).UnixMilli(), LastHeardAt: at(last).UnixMilli(), PacketCount: count, IATACount: iatas, TraceType: kind, PathHashes: paths[tag], SNRValues: snrs[tag]} + } + a, b, c, d := summary("a", 1, 40, 3, 2, "TRACE"), summary("b", 4, 30, 2, 1, "TRACE"), summary("c", 6, 20, 1, 1, "TRACE"), summary("d", 7, 15, 1, 1, "PING") + for _, tc := range []struct { + name string + iatas []string + scope, kind string + since, until, cursor int + want []api.TraceTagSummary + }{ + {"first page", nil, "", "", 0, 0, 0, []api.TraceTagSummary{a, b, c, d}}, + {"cursor at newest group", nil, "", "", 0, 0, 40, []api.TraceTagSummary{b, c, d}}, + {"cursor between packet times", nil, "", "", 0, 0, 30, []api.TraceTagSummary{c, d}}, + {"end of groups", nil, "", "", 0, 0, 15, []api.TraceTagSummary{}}, + {"region", []string{"YYZ"}, "", "", 0, 0, 35, []api.TraceTagSummary{b}}, + {"multiple regions", []string{"YYZ", "YVR", "YYZ"}, "", "", 0, 0, 35, []api.TraceTagSummary{b, c}}, + {"unknown region", []string{"ZZZ"}, "", "", 0, 0, 0, []api.TraceTagSummary{}}, + // Filters apply to the tag summary: its first scope, any-TRACE type, and first_heard_at. + {"scope", nil, "scope-one", "", 0, 0, 30, []api.TraceTagSummary{c}}, + {"first scope wins", nil, "scope-two", "", 0, 0, 0, []api.TraceTagSummary{d}}, + {"type", nil, "", "TRACE", 0, 0, 35, []api.TraceTagSummary{b, c}}, + {"mixed tag is TRACE", nil, "", "PING", 0, 0, 0, []api.TraceTagSummary{d}}, + {"time window on first heard", nil, "", "", 3, 6, 26, []api.TraceTagSummary{c}}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := store.ListTraceTags(ctx, tc.iatas, tc.scope, tc.kind, at(tc.since), at(tc.until), at(tc.cursor), "", 20) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("trace summaries differ: got %+v, want %+v", got, tc.want) + } + }) + } + // Reassemble pages using the same epoch-millisecond cursor exposed by the API. + reference := map[string]api.TraceTagSummary{a.TraceTag: a, b.TraceTag: b, c.TraceTag: c, d.TraceTag: d} + seen := map[string]bool{} + var cursor time.Time + total := 0 + for page := 0; page < 10; page++ { + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, cursor, "", 2) + if err != nil { + t.Fatal(err) + } + if len(rows) == 0 { + break + } + for _, row := range rows { + total++ + if seen[row.TraceTag] { + t.Errorf("pagination repeated trace tag %s", row.TraceTag) + } + seen[row.TraceTag] = true + if !reflect.DeepEqual(row, reference[row.TraceTag]) { + t.Errorf("pagination changed summary for %s", row.TraceTag) + } + } + cursor = time.UnixMilli(rows[len(rows)-1].LastHeardAt) + } + t.Logf("two-row pages returned %d summaries for %d distinct trace tags", total, len(seen)) + if total != 4 || len(seen) != 4 { + t.Errorf("expected exactly four complete trace summaries") + } + + if err := store.DeleteOldTraceTags(ctx, at(20)); err != nil { + t.Fatal(err) + } + if got, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, "", 20); err != nil || !reflect.DeepEqual(got, []api.TraceTagSummary{a, b, c}) { + t.Errorf("after cleanup got %+v, %v; want a, b, c", got, err) + } +} + +// Storing a TRACE packet records its contribution even if the hearing write that follows +// is lost; the next hearing replaces the packet's provisional times. +func TestTraceSummarySurvivesMissedHearingPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, "INSERT INTO iata_codes (iata) VALUES ('YVR')"); err != nil { + t.Fatal(err) + } + store := New(pool, 0, 0) + tag := []byte{0xee, 0xee, 0xee, 0xee} + payload := json.RawMessage(`{"type":"TRACE","pathHashes":["aa","bb"],"snrValues":[1,2]}`) + upsert := func(hash byte) { + t.Helper() + if _, err := store.UpsertPacket(ctx, ingest.UpsertPacketParams{PacketHash: []byte{hash}, PayloadType: 9, RouteType: 1, + RawPayload: []byte{0}, RawHeader: []byte{0}, ParsedPayload: payload, TraceTag: tag}); err != nil { + t.Fatal(err) + } + } + upsert(1) // the RecordTrace that would follow never runs + upsert(1) // a broker duplicate must not count twice + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, "", 10) + if err != nil || len(rows) != 1 || rows[0].PacketCount != 1 || len(rows[0].PathHashes) != 2 || rows[0].TraceType != "TRACE" { + t.Fatalf("summary after a missed hearing: %+v %v", rows, err) + } + heard := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if err := store.RecordTrace(ctx, ingest.TraceHearing{TraceTag: tag, IATA: "YVR", HeardAt: heard}); err != nil { + t.Fatal(err) + } + rows, err = store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, "", 10) + if err != nil || len(rows) != 1 || rows[0].FirstHeardAt != heard.UnixMilli() || rows[0].LastHeardAt != heard.UnixMilli() || rows[0].IATACount != 1 { + t.Errorf("first hearing should replace provisional times: %+v %v", rows, err) + } +} + +// Tags sharing a millisecond (second-resolution observer clocks, sub-ms times) must +// all survive paging with the (cursor, cursorTag) keyset. +func TestTraceTagPaginationTiesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, heard) VALUES + ('\x00000001', '2026-01-01 00:00:00+00', '2026-01-01 00:00:10+00', true), + ('\x00000002', '2026-01-01 00:00:00+00', '2026-01-01 00:00:10+00', true), + ('\x00000003', '2026-01-01 00:00:00+00', '2026-01-01 00:00:10+00', true), + ('\x00000004', '2026-01-01 00:00:00+00', '2026-01-01 00:00:10+00', true), + ('\x00000005', '2026-01-01 00:00:00+00', '2026-01-01 00:00:05.000300+00', true), + ('\x00000006', '2026-01-01 00:00:00+00', '2026-01-01 00:00:05.000700+00', true), + ('\x00000007', '2026-01-01 00:00:00+00', '2026-01-01 00:00:04+00', true)`); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + want := []string{"00000004", "00000003", "00000002", "00000001", "00000006", "00000005", "00000007"} + var got []string + var cursor time.Time + var cursorTag string + for page := 0; page < 10; page++ { + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, cursor, cursorTag, 2) + if err != nil { + t.Fatal(err) + } + if len(rows) == 0 { + break + } + for _, r := range rows { + got = append(got, r.TraceTag) + } + last := rows[len(rows)-1] + cursor, cursorTag = time.UnixMilli(last.LastHeardAt), last.TraceTag + } + if !reflect.DeepEqual(got, want) { + t.Errorf("keyset pages = %v, want %v", got, want) + } + + // Without cursorTag the cursor stays exclusive on the millisecond. + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Date(2026, 1, 1, 0, 0, 5, 0, time.UTC), "", 10) + if err != nil || len(rows) != 1 || rows[0].TraceTag != "00000007" { + t.Errorf("ms-only cursor: %+v %v", rows, err) + } +} diff --git a/db/traces.go b/db/traces.go index 2624564d..93ed18cd 100644 --- a/db/traces.go +++ b/db/traces.go @@ -7,10 +7,12 @@ import ( "context" "encoding/hex" "encoding/json" + "fmt" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/jackc/pgx/v5/pgtype" ) @@ -20,19 +22,23 @@ type tracePayload struct { SNRValues []float32 `json:"snrValues"` } -func (s *Store) UpsertTraceIATA(ctx context.Context, traceTag []byte, iata string, heardAt time.Time) error { - return s.q.UpsertTraceIATA(ctx, sqlc.UpsertTraceIATAParams{ - TraceTag: traceTag, - Iata: iata, - LastHeard: pgtype.Timestamptz{Time: heardAt, Valid: true}, +func (s *Store) RecordTrace(ctx context.Context, h ingest.TraceHearing) error { + return s.q.RecordTrace(ctx, sqlc.RecordTraceParams{ + TraceTag: h.TraceTag, + Iata: h.IATA, + HeardAt: pgtype.Timestamptz{Time: h.HeardAt, Valid: true}, }) } +func (s *Store) DeleteOldTraceTags(ctx context.Context, cutoff time.Time) error { + return s.q.DeleteOldTraceTags(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) +} + func (s *Store) DeleteOldTraceIATAs(ctx context.Context, cutoff time.Time) error { return s.q.DeleteOldTraceIATAs(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } -func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) { +func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { var sinceTS, untilTS, cursorTS pgtype.Timestamptz if !since.IsZero() { sinceTS = pgtype.Timestamptz{Time: since, Valid: true} @@ -43,6 +49,14 @@ func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceT if !cursor.IsZero() { cursorTS = pgtype.Timestamptz{Time: cursor, Valid: true} } + var tag []byte + if cursorTag != "" { + b, err := hex.DecodeString(cursorTag) + if err != nil { + return nil, fmt.Errorf("cursor tag: %w", err) + } + tag = b + } rows, err := s.q.ListTraceTags(ctx, sqlc.ListTraceTagsParams{ Column1: iatas, Column2: scope, @@ -51,6 +65,7 @@ func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceT Column5: cursorTS, Limit: limit, Column7: traceType, + Column8: tag, }) if err != nil { return nil, err @@ -110,21 +125,8 @@ func (s *Store) GetTraceByTag(ctx context.Context, tag string) (*api.TraceDetail } packet.RawPath = rawPath } - // fetch observations to get IATAs for route resolution - packetHashBytes, err := hex.DecodeString(r.PacketHashHex) - if err == nil { - obsRows, err := s.q.ListObservationsForPacket(ctx, packetHashBytes) - if err == nil && len(obsRows) > 0 { - iatas := make([]string, 0, len(obsRows)) - seen := make(map[string]struct{}) - for _, v := range obsRows { - if _, ok := seen[v.Iata]; !ok { - seen[v.Iata] = struct{}{} - iatas = append(iatas, v.Iata) - } - } - packet.ResolvedRoute = s.resolveTraceRoute(ctx, &parsed, iatas) - } + if len(r.Iatas) > 0 { + packet.ResolvedRoute = s.resolveTraceRoute(ctx, &parsed, r.Iatas) } detail.Packets = append(detail.Packets, packet) } diff --git a/db/traces_integration_test.go b/db/traces_integration_test.go new file mode 100644 index 00000000..51305443 --- /dev/null +++ b/db/traces_integration_test.go @@ -0,0 +1,133 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +type traceQueryCounter struct { + sqlc.DBTX + queries, observations int +} + +func (c *traceQueryCounter) Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) { + c.queries++ + if strings.Contains(sql, "-- name: ListObservationsForPacket ") { + c.observations++ + } + return c.DBTX.Query(ctx, sql, args...) +} + +// Real SQL and Store mapping are exercised against temporary tables only. +func TestTraceDetailPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"packets", "packet_observations", "observers", "nodes", "node_short_ids", "transport_scopes"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO nodes (id,public_key,node_type,name) VALUES + ('00000000-0000-0000-0000-000000000001',decode('aa'||repeat('00',31),'hex'),2,'YVR relay'), + ('00000000-0000-0000-0000-000000000002',decode('aa01'||repeat('00',30),'hex'),2,'YYJ relay'); +INSERT INTO node_short_ids (node_id,iata,prefix_4) VALUES + ('00000000-0000-0000-0000-000000000001','YVR','\xaa000000'), + ('00000000-0000-0000-0000-000000000002','YYJ','\xaa010000'); +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,trace_tag,parsed_payload,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,1,'\x00','\x00','\x01020304', + CASE WHEN i=100 THEN '{}'::jsonb ELSE '{"flags":0,"pathHashes":["aa"],"snrValues":[12.5]}'::jsonb END, + '2026-01-01 00:00:00+00'::timestamptz+i*interval '1 second', '2026-01-01 00:00:00+00'::timestamptz+i*interval '1 second'+interval '1 second' +FROM generate_series(1,100) i; +INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT i,decode(lpad(to_hex(i),64,'0'),'hex'),md5(i::text)::uuid,'YVR','2026-01-01 00:00:00+00'::timestamptz+interval '2 seconds',1,1,1 +FROM generate_series(1,100) i WHERE i<>99; +-- The first packet was heard in YYJ before YVR. A repeat YYJ observation must +-- not trigger another region lookup or change equal-confidence precedence. +INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES + (101,decode(lpad('1',64,'0'),'hex'),md5('101')::uuid,'YYJ','2026-01-01 00:00:00+00'::timestamptz,1,1,1), + (102,decode(lpad('1',64,'0'),'hex'),md5('102')::uuid,'YYJ','2026-01-01 00:00:00+00'::timestamptz+interval '3 seconds',1,1,1); +ANALYZE packets; ANALYZE packet_observations; ANALYZE nodes; ANALYZE node_short_ids;`) + if err != nil { + t.Fatal(err) + } + counter := &traceQueryCounter{DBTX: tx} + store := &Store{q: sqlc.New(counter)} + start := time.Now() + detail, err := store.GetTraceByTag(ctx, "01020304") + elapsed := time.Since(start) + if err != nil { + t.Fatal(err) + } + if detail == nil || detail.TraceTag != "01020304" || len(detail.Packets) != 100 { + t.Fatal("trace packets missing") + } + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i, p := range detail.Packets { + number := i + 1 + if p.PacketHash != fmt.Sprintf("%064x", number) || p.RouteType != 1 || p.FirstHeardAt != anchor.Add(time.Duration(number)*time.Second).UnixMilli() || p.LastHeardAt != p.FirstHeardAt+1000 { + t.Fatalf("wrong packet fields or order at %d", number) + } + if number == 100 { + if len(p.RawPath) != 0 || len(p.ResolvedRoute) != 0 { + t.Fatal("invented path for empty payload") + } + continue + } + if len(p.RawPath) != 1 || p.RawPath[0].Hash != "aa" || p.RawPath[0].SNR == nil || *p.RawPath[0].SNR != 12.5 { + t.Fatalf("wrong raw path/SNR at %d", number) + } + if number == 99 { + if len(p.ResolvedRoute) != 0 { + t.Fatal("unobserved packet must not borrow another packet's IATAs") + } + continue + } + want := "YVR relay" + if number == 1 { + want = "YYJ relay" + } + if len(p.ResolvedRoute) != 1 || p.ResolvedRoute[0].Confidence != "high" || len(p.ResolvedRoute[0].Nodes) != 1 || p.ResolvedRoute[0].Nodes[0].Name == nil || *p.ResolvedRoute[0].Nodes[0].Name != want { + t.Fatalf("wrong region precedence/resolution at %d", number) + } + if p.ResolvedRoute[0].SNR == nil || *p.ResolvedRoute[0].SNR != 12.5 { + t.Fatal("resolved SNR changed") + } + } + t.Logf("100-packet trace: %.3f ms, %d database queries, %d separate observation queries", float64(elapsed.Microseconds())/1000, counter.queries, counter.observations) + if counter.observations != 0 { + t.Errorf("extra observation queries: %d; want 0", counter.observations) + } + if counter.queries != 100 { + t.Errorf("got %d queries; want one packet query and 99 unchanged path-resolution queries", counter.queries) + } + counter.queries, counter.observations = 0, 0 + missing, err := store.GetTraceByTag(ctx, "00000000") + if err != nil || missing != nil || counter.queries != 1 { + t.Fatal("missing tag behavior changed") + } +} diff --git a/db/traces_test.go b/db/traces_test.go index 764860e3..0e3d4b48 100644 --- a/db/traces_test.go +++ b/db/traces_test.go @@ -24,7 +24,7 @@ func TestListTraceTags_Empty(t *testing.T) { Return([]sqlc.ListTraceTagsRow{}, nil) store := &Store{q: mock} - items, err := store.ListTraceTags(context.Background(), []string{"YVR"}, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + items, err := store.ListTraceTags(context.Background(), []string{"YVR"}, "", "", time.Time{}, time.Time{}, time.Time{}, "", 10) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -56,7 +56,7 @@ func TestListTraceTags_WithPayload(t *testing.T) { }, nil) store := &Store{q: mock} - items, err := store.ListTraceTags(context.Background(), []string{"YVR"}, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + items, err := store.ListTraceTags(context.Background(), []string{"YVR"}, "", "", time.Time{}, time.Time{}, time.Time{}, "", 10) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -112,15 +112,10 @@ func TestGetTraceByTag_WithPacket(t *testing.T) { FirstHeardAt: firstHeard, LastHeardAt: lastHeard, ParsedPayload: parsedPayload, + Iatas: []string{"YVR"}, }, }, nil) - mock.EXPECT(). - ListObservationsForPacket(gomock.Any(), []byte{0xaa, 0xbb, 0xcc, 0xdd}). - Return([]sqlc.ListObservationsForPacketRow{ - {Iata: "YVR"}, - }, nil) - mock.EXPECT(). ResolvePathHashesP2(gomock.Any(), sqlc.ResolvePathHashesP2Params{ Iata: "YVR", diff --git a/docs/docs.go b/docs/docs.go index 347a251b..1d20bed3 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4,11 +4,11 @@ package docs import "github.com/swaggo/swag" const docTemplate = `{ - "schemes": [[ marshal .Schemes ]], + "schemes": {{ marshal .Schemes }}, "swagger": "2.0", "info": { - "description": "[[escape .Description]]", - "title": "[[.Title]]", + "description": "{{escape .Description}}", + "title": "{{.Title}}", "termsOfService": "https://github.com/MeshCore-Beacon/beacon-server", "contact": { "name": "MeshCore Beacon", @@ -17,70 +17,949 @@ const docTemplate = `{ "license": { "name": "AGPL-3-or-later" }, - "version": "[[.Version]]" + "version": "{{.Version}}" }, - "host": "[[.Host]]", - "basePath": "[[.BasePath]]", + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", "paths": { + "/admin/accounts": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Includes active and inactive accounts, newest first. These records are not login users.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "List operator accounts", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "post": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Trims the name; names are case-sensitive and unique among active accounts. No credential or login is created.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Create an operator account", + "parameters": [ + { + "description": "Account name (1-128 Unicode characters, no control characters); body at most 4 KiB", + "name": "account", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/accounts/{id}": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns active or inactive account records.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Get an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "delete": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Soft deactivation preserves the record. Its name may be reused by a new account.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Deactivate an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "Deactivated" + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/backup": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint.", + "produces": [ + "application/gzip" + ], + "tags": [ + "Admin" + ], + "summary": "Download a private database and saved-config backup", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + }, + "headers": { + "Content-Disposition": { + "type": "string", + "description": "attachment; filename=beacon-backup.tar.gz" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "504": { + "description": "Gateway Timeout", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "507": { + "description": "Insufficient Storage", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/config": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns current CORS options, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Runtime origin updates are lost on restart.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Inspect selected running configuration", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "put": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Replaces only cors.allowed_origins immediately. Updates are serialized; concurrent valid updates are applied one at a time. Already-running requests may finish with the previous policy. Nothing is persisted; restart reloads saved configuration.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Update runtime CORS origins", + "parameters": [ + { + "description": "1-32 ASCII HTTP(S) origins, at most 512 bytes each; one hostname wildcard is supported, or a sole *. Empty/null lists and unsupported fields are rejected. Body at most 16 KiB.", + "name": "config", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/brokers": { "get": { "produces": [ "application/json" ], "tags": [ - "Brokers" + "Brokers" + ], + "summary": "List all MQTT brokers and their connection status", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_handlers.BrokerStatus" + } + } + } + } + } + }, + "/channels": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Channels" + ], + "summary": "List channels", + "parameters": [ + { + "type": "string", + "description": "Single-byte channel hash (hex)", + "name": "hash", + "in": "query" + }, + { + "type": "string", + "description": "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels", + "name": "iata", + "in": "query" + }, + { + "type": "string", + "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata", + "name": "iatas", + "in": "query" + }, + { + "type": "boolean", + "description": "Only channels Beacon can (true) or cannot (false) decrypt; omit for all", + "name": "keyKnown", + "in": "query" + }, + { + "type": "integer", + "description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning", + "name": "cursor", + "in": "query" + }, + { + "type": "string", + "description": "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed.", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/channels/{channelID}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Channels" + ], + "summary": "Get channel detail", + "parameters": [ + { + "type": "integer", + "description": "Channel integer ID", + "name": "channelID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Channel" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/channels/{channelID}/messages": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Channels" + ], + "summary": "List messages for a channel", + "parameters": [ + { + "type": "integer", + "description": "Channel integer ID", + "name": "channelID", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Return messages after this epoch ms", + "name": "since", + "in": "query" + }, + { + "type": "string", + "description": "Filter by IATA code(s), comma-separated e.g. YVR or YVR,YYJ", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "string", + "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", + "name": "scope", + "in": "query" + }, + { + "type": "integer", + "description": "Message ID of last item for pagination (results ordered newest first)", + "name": "cursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/iatas": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "IATAs" + ], + "summary": "List all IATA codes", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/iatas/{iata}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "IATAs" + ], + "summary": "Get a single IATA code", + "parameters": [ + { + "type": "string", + "description": "3-letter IATA code", + "name": "iata", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/iatas/{iata}/border": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "IATAs" + ], + "summary": "Get an IATA's GeoJSON border, if configured", + "parameters": [ + { + "type": "string", + "description": "3-letter IATA code", + "name": "iata", + "in": "path", + "required": true + } ], - "summary": "List all MQTT brokers and their connection status", "responses": { "200": { - "description": "OK", + "description": "GeoJSON Feature (Polygon or MultiPolygon geometry, with bbox)", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/internal_api_handlers.BrokerStatus" - } + "type": "object" + } + }, + "204": { + "description": "IATA exists but has no border configured" + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } } } } }, - "/channels": { + "/messages": { "get": { "produces": [ "application/json" ], "tags": [ - "Channels" + "Messages" ], - "summary": "List channels", + "summary": "List channel messages", "parameters": [ { - "type": "string", - "description": "Single-byte channel hash (hex)", - "name": "hash", + "type": "integer", + "description": "Filter by channel integer ID (mutually exclusive with channelHash)", + "name": "channelID", "in": "query" }, { "type": "string", - "description": "Filter by IATA code", - "name": "iata", + "description": "Filter by channel hash byte hex (mutually exclusive with channelID)", + "name": "channelHash", + "in": "query" + }, + { + "type": "integer", + "description": "Return messages after this epoch ms", + "name": "since", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ", + "description": "Filter by IATA code(s), comma-separated e.g. YVR or YVR,YYJ", "name": "iatas", "in": "query" }, { "type": "integer", - "description": "last_seen epoch ms of last item for pagination", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "string", + "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", + "name": "scope", + "in": "query" + }, + { + "type": "integer", + "description": "Message ID of last item for pagination (results ordered newest first)", "name": "cursor", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -89,7 +968,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary" + "type": "object" } }, "400": { @@ -107,29 +986,64 @@ const docTemplate = `{ } } }, - "/channels/{channelID}": { + "/messages/backfill": { "get": { "produces": [ "application/json" ], "tags": [ - "Channels" + "Messages" ], - "summary": "Get channel detail", + "summary": "Backfill messages after a given message ID", "parameters": [ { "type": "integer", - "description": "Channel integer ID", - "name": "channelID", - "in": "path", + "description": "Return messages after this ID (use last WS event message ID)", + "name": "afterId", + "in": "query", "required": true + }, + { + "type": "string", + "description": "Filter by IATA code(s), comma-separated", + "name": "iatas", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by transport scope name", + "name": "scope", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 100); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Channel" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelMessage" + } } }, "400": { @@ -138,8 +1052,8 @@ const docTemplate = `{ "$ref": "#/definitions/internal_api_handlers.APIError" } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -147,32 +1061,37 @@ const docTemplate = `{ } } }, - "/channels/{channelID}/messages": { + "/nodes": { "get": { "produces": [ "application/json" ], "tags": [ - "Channels" + "Nodes" ], - "summary": "List messages for a channel", + "summary": "List nodes", "parameters": [ { "type": "integer", - "description": "Channel integer ID", - "name": "channelID", - "in": "path", - "required": true + "description": "Node type integer (1=companion, 2=repeater, 3=room_server, 4=sensor)", + "name": "type", + "in": "query" }, { - "type": "integer", - "description": "Return messages after this epoch ms", - "name": "since", + "type": "string", + "description": "Node type name (companion, repeater, room_server, sensor)", + "name": "typeName", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YVR or YVR,YYJ", + "description": "Filter by single IATA code (case-insensitive)", + "name": "iata", + "in": "query" + }, + { + "type": "string", + "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", "name": "iatas", "in": "query" }, @@ -188,21 +1107,59 @@ const docTemplate = `{ "name": "region", "in": "query" }, + { + "type": "string", + "description": "Partial case-insensitive name match", + "name": "name", + "in": "query" + }, { "type": "string", "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", "name": "scope", "in": "query" }, + { + "type": "string", + "description": "Exact public key match (hex)", + "name": "pubkey", + "in": "query" + }, + { + "type": "string", + "description": "Partial public key match: hex prefix, case-insensitive", + "name": "pubkeyPrefix", + "in": "query" + }, + { + "type": "boolean", + "description": "Filter by multibyte path support (true/false); omit for no filter", + "name": "supportsMultibytePaths", + "in": "query" + }, + { + "type": "boolean", + "description": "Filter by multibyte trace support (true/false); omit for no filter", + "name": "supportsMultibyteTraces", + "in": "query" + }, + { + "type": "boolean", + "description": "Include each node's known neighbor IDs (neighborIds field). Bare ?neighbors or ?neighbors=true enables it; omit/false for none", + "name": "neighbors", + "in": "query" + }, { "type": "integer", - "description": "Message ID of last item for pagination (results ordered newest first)", + "description": "last_seen epoch ms of last item for pagination", "name": "cursor", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -211,7 +1168,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "type": "object" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary" } }, "400": { @@ -229,23 +1186,35 @@ const docTemplate = `{ } } }, - "/iatas": { + "/nodes/{nodeId}": { "get": { "produces": [ "application/json" ], "tags": [ - "IATAs" + "Nodes" + ], + "summary": "Get node detail", + "parameters": [ + { + "type": "string", + "description": "Node UUID", + "name": "nodeId", + "in": "path", + "required": true + } ], - "summary": "List all IATA codes", "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Node" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "404": { @@ -253,24 +1222,30 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, - "/iatas/{iata}": { + "/nodes/{nodeId}/neighbors": { "get": { "produces": [ "application/json" ], "tags": [ - "IATAs" + "Nodes" ], - "summary": "Get a single IATA code", + "summary": "List neighbors for a node", "parameters": [ { "type": "string", - "description": "3-letter IATA code", - "name": "iata", + "description": "Node UUID", + "name": "nodeId", "in": "path", "required": true } @@ -279,11 +1254,20 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeNeighbor" + } } }, - "404": { - "description": "Not Found", + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -291,36 +1275,53 @@ const docTemplate = `{ } } }, - "/iatas/{iata}/border": { + "/nodes/{nodeId}/observations": { "get": { "produces": [ "application/json" ], "tags": [ - "IATAs" + "Nodes" ], - "summary": "Get an IATA's GeoJSON border, if configured", + "summary": "List packet observations originating from a node", "parameters": [ { "type": "string", - "description": "3-letter IATA code", - "name": "iata", + "description": "Node UUID", + "name": "nodeId", "in": "path", "required": true + }, + { + "type": "integer", + "description": "Observation ID of last item for pagination", + "name": "cursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" } ], "responses": { "200": { - "description": "GeoJSON Feature (Polygon or MultiPolygon geometry, with bbox)", + "description": "OK", "schema": { - "type": "object" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_PacketObservationSummary" } }, - "204": { - "description": "IATA exists but has no border configured" + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -328,50 +1329,62 @@ const docTemplate = `{ } } }, - "/messages": { + "/observers": { "get": { "produces": [ "application/json" ], "tags": [ - "Messages" + "Observers" ], - "summary": "List channel messages", + "summary": "List observers", "parameters": [ { - "type": "integer", - "description": "Filter by channel integer ID (mutually exclusive with channelHash)", - "name": "channelID", + "type": "string", + "description": "Filter by single IATA code (case-insensitive)", + "name": "iata", "in": "query" }, { "type": "string", - "description": "Filter by channel hash byte hex (mutually exclusive with channelID)", - "name": "channelHash", + "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", + "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Return messages after this epoch ms", - "name": "since", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YVR or YVR,YYJ", - "name": "iatas", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", "in": "query" }, { - "type": "integer", - "description": "Filter by region ID, expands to member IATAs", - "name": "regionId", + "type": "string", + "description": "Filter by observer type (e.g. meshcoretomqtt, meshcore-ha)", + "name": "type", "in": "query" }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", - "name": "region", + "description": "Filter by broker name", + "name": "broker", + "in": "query" + }, + { + "type": "string", + "description": "Filter by status (online or offline)", + "name": "status", + "in": "query" + }, + { + "type": "string", + "description": "Partial case-insensitive display name match", + "name": "name", "in": "query" }, { @@ -382,13 +1395,15 @@ const docTemplate = `{ }, { "type": "integer", - "description": "Message ID of last item for pagination (results ordered newest first)", + "description": "last_seen epoch ms of last item for pagination", "name": "cursor", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -397,7 +1412,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "type": "object" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ObserverSummary" } }, "400": { @@ -415,62 +1430,29 @@ const docTemplate = `{ } } }, - "/messages/backfill": { + "/observers/{observerId}": { "get": { "produces": [ "application/json" ], "tags": [ - "Messages" + "Observers" ], - "summary": "Backfill messages after a given message ID", + "summary": "Get observer detail", "parameters": [ - { - "type": "integer", - "description": "Return messages after this ID (use last WS event message ID)", - "name": "afterId", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "Filter by IATA code(s), comma-separated", - "name": "iatas", - "in": "query" - }, - { - "type": "string", - "description": "Filter by region slug", - "name": "region", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by region ID", - "name": "regionId", - "in": "query" - }, { "type": "string", - "description": "Filter by transport scope name", - "name": "scope", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 100)", - "name": "limit", - "in": "query" + "description": "Observer UUID", + "name": "observerId", + "in": "path", + "required": true } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelMessage" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Observer" } }, "400": { @@ -479,6 +1461,12 @@ const docTemplate = `{ "$ref": "#/definitions/internal_api_handlers.APIError" } }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -488,104 +1476,39 @@ const docTemplate = `{ } } }, - "/nodes": { + "/observers/{observerId}/activity": { "get": { "produces": [ "application/json" ], "tags": [ - "Nodes" + "Observers" ], - "summary": "List nodes", + "summary": "Get observer heard-activity history", "parameters": [ - { - "type": "integer", - "description": "Node type integer (1=companion, 2=repeater, 3=room_server, 4=sensor)", - "name": "type", - "in": "query" - }, - { - "type": "string", - "description": "Node type name (companion, repeater, room_server, sensor)", - "name": "typeName", - "in": "query" - }, - { - "type": "string", - "description": "Filter by single IATA code (case-insensitive)", - "name": "iata", - "in": "query" - }, - { - "type": "string", - "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", - "name": "iatas", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by region ID, expands to member IATAs", - "name": "regionId", - "in": "query" - }, - { - "type": "string", - "description": "Filter by region slug, expands to member IATAs", - "name": "region", - "in": "query" - }, - { - "type": "string", - "description": "Partial case-insensitive name match", - "name": "name", - "in": "query" - }, { "type": "string", - "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", - "name": "scope", - "in": "query" + "description": "Observer UUID", + "name": "observerId", + "in": "path", + "required": true }, { "type": "string", - "description": "Exact public key match (hex)", - "name": "pubkey", + "description": "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h", + "name": "range", "in": "query" }, { "type": "string", - "description": "Partial public key match: hex prefix, case-insensitive", - "name": "pubkeyPrefix", - "in": "query" - }, - { - "type": "boolean", - "description": "Filter by multibyte path support (true/false); omit for no filter", - "name": "supportsMultibytePaths", - "in": "query" - }, - { - "type": "boolean", - "description": "Filter by multibyte trace support (true/false); omit for no filter", - "name": "supportsMultibyteTraces", - "in": "query" - }, - { - "type": "boolean", - "description": "Include each node's known neighbor IDs (neighborIds field). Bare ?neighbors or ?neighbors=true enables it; omit/false for none", - "name": "neighbors", - "in": "query" - }, - { - "type": "integer", - "description": "last_seen epoch ms of last item for pagination", - "name": "cursor", + "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", + "name": "interval", "in": "query" }, { "type": "integer", - "description": "Max results (default 50)", - "name": "limit", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until.", + "name": "until", "in": "query" } ], @@ -593,7 +1516,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity" } }, "400": { @@ -602,6 +1525,12 @@ const docTemplate = `{ "$ref": "#/definitions/internal_api_handlers.APIError" } }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -611,29 +1540,43 @@ const docTemplate = `{ } } }, - "/nodes/{nodeId}": { + "/observers/{observerId}/adverts": { "get": { "produces": [ "application/json" ], "tags": [ - "Nodes" + "Observers" ], - "summary": "Get node detail", + "summary": "List advert packets heard by an observer", "parameters": [ { "type": "string", - "description": "Node UUID", - "name": "nodeId", + "description": "Observer UUID", + "name": "observerId", "in": "path", "required": true + }, + { + "type": "integer", + "description": "Observation ID of last item for pagination", + "name": "cursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Node" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_AdvertObservation" } }, "400": { @@ -642,8 +1585,8 @@ const docTemplate = `{ "$ref": "#/definitions/internal_api_handlers.APIError" } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -651,32 +1594,47 @@ const docTemplate = `{ } } }, - "/nodes/{nodeId}/neighbors": { + "/observers/{observerId}/telemetry": { "get": { "produces": [ "application/json" ], "tags": [ - "Nodes" + "Observers" ], - "summary": "List neighbors for a node", + "summary": "Get observer telemetry history", "parameters": [ { "type": "string", - "description": "Node UUID", - "name": "nodeId", + "description": "Observer UUID", + "name": "observerId", "in": "path", "required": true + }, + { + "type": "string", + "description": "Duration window e.g. 24h, 48h, 168h (default 24h)", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Return points after this telemetry ID for WS reconnection backfill", + "name": "afterId", + "in": "query" + }, + { + "type": "string", + "description": "Bucketing interval: 1h (default), 6h, or 24h", + "name": "interval", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeNeighbor" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetry" } }, "400": { @@ -694,32 +1652,105 @@ const docTemplate = `{ } } }, - "/nodes/{nodeId}/observations": { + "/packets": { "get": { "produces": [ "application/json" ], "tags": [ - "Nodes" + "Packets" ], - "summary": "List packet observations originating from a node", + "summary": "List packets", "parameters": [ + { + "type": "integer", + "description": "Filter by payload type integer", + "name": "payloadType", + "in": "query" + }, { "type": "string", - "description": "Node UUID", - "name": "nodeId", - "in": "path", - "required": true + "description": "Filter by multiple payload types, comma-separated e.g. 2,4", + "name": "payloadTypes", + "in": "query" + }, + { + "type": "string", + "description": "Filter by payload type name (advert, grp_txt, txt_msg, trace, anon_req)", + "name": "payloadTypeName", + "in": "query" }, { "type": "integer", - "description": "Observation ID of last item for pagination", + "description": "Filter by route type (0=transport_flood, 1=flood, 2=direct, 3=transport_direct)", + "name": "routeType", + "in": "query" + }, + { + "type": "string", + "description": "Filter by multiple route types, comma-separated e.g. 0,1", + "name": "routeTypes", + "in": "query" + }, + { + "type": "string", + "description": "Filter by single IATA code (case-insensitive)", + "name": "iata", + "in": "query" + }, + { + "type": "string", + "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", + "name": "iatas", + "in": "query" + }, + { + "type": "string", + "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", + "name": "scope", + "in": "query" + }, + { + "type": "string", + "description": "Filter by multiple transport scope names, comma-separated e.g. %23bc,%23west", + "name": "scopes", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by first_heard_at \u003e= since (epoch ms)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by first_heard_at \u003c= until (epoch ms)", + "name": "until", + "in": "query" + }, + { + "type": "integer", + "description": "epoch ms of last item for pagination; last_heard_at, or site-local heard_at when iatas is set", "name": "cursor", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -728,7 +1759,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_PacketObservationSummary" + "type": "object" } }, "400": { @@ -746,79 +1777,70 @@ const docTemplate = `{ } } }, - "/observers": { + "/packets/backfill": { "get": { "produces": [ "application/json" ], "tags": [ - "Observers" + "Packets" ], - "summary": "List observers", + "summary": "Backfill packets after a given observation ID", "parameters": [ { - "type": "string", - "description": "Filter by single IATA code (case-insensitive)", - "name": "iata", - "in": "query" - }, - { - "type": "string", - "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", - "name": "iatas", - "in": "query" + "type": "integer", + "description": "Return packets with observations after this ID (use last WS event observation ID)", + "name": "afterObservationId", + "in": "query", + "required": true }, { "type": "integer", - "description": "Filter by region ID, expands to member IATAs", - "name": "regionId", + "description": "Filter by payload type integer", + "name": "payloadType", "in": "query" }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", - "name": "region", + "description": "Filter by payload type name", + "name": "payloadTypeName", "in": "query" }, { - "type": "string", - "description": "Filter by observer type (e.g. meshcoretomqtt, meshcore-ha)", - "name": "type", + "type": "integer", + "description": "Filter by route type", + "name": "routeType", "in": "query" }, { "type": "string", - "description": "Filter by broker name", - "name": "broker", + "description": "Filter by IATA code(s), comma-separated", + "name": "iatas", "in": "query" }, { "type": "string", - "description": "Filter by status (online or offline)", - "name": "status", + "description": "Filter by region slug", + "name": "region", "in": "query" }, { - "type": "string", - "description": "Partial case-insensitive display name match", - "name": "name", + "type": "integer", + "description": "Filter by region ID", + "name": "regionId", "in": "query" }, { "type": "string", - "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", + "description": "Filter by transport scope name", "name": "scope", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "last_seen epoch ms of last item for pagination", - "name": "cursor", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 100); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -827,7 +1849,10 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ObserverSummary" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketSummary" + } } }, "400": { @@ -845,20 +1870,20 @@ const docTemplate = `{ } } }, - "/observers/{observerId}": { + "/packets/{packetHash}": { "get": { "produces": [ "application/json" ], "tags": [ - "Observers" + "Packets" ], - "summary": "Get observer detail", + "summary": "Get full packet detail. For trace packets (payloadType=9), includes resolvedRoute", "parameters": [ { "type": "string", - "description": "Observer UUID", - "name": "observerId", + "description": "Packet hash (hex)", + "name": "packetHash", "in": "path", "required": true } @@ -867,17 +1892,51 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Observer" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Packet" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/internal_api_handlers.APIError" + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/regions": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Regions" + ], + "summary": "List all regions", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RegionSummary" + } } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -885,41 +1944,29 @@ const docTemplate = `{ } } }, - "/observers/{observerId}/adverts": { + "/regions/{regionId}": { "get": { "produces": [ "application/json" ], "tags": [ - "Observers" + "Regions" ], - "summary": "List advert packets heard by an observer", + "summary": "Get a single region", "parameters": [ { - "type": "string", - "description": "Observer UUID", - "name": "observerId", + "type": "integer", + "description": "Region ID", + "name": "regionId", "in": "path", "required": true - }, - { - "type": "integer", - "description": "Observation ID of last item for pagination", - "name": "cursor", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 50)", - "name": "limit", - "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_AdvertObservation" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Region" } }, "400": { @@ -928,6 +1975,12 @@ const docTemplate = `{ "$ref": "#/definitions/internal_api_handlers.APIError" } }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -937,39 +1990,46 @@ const docTemplate = `{ } } }, - "/observers/{observerId}/telemetry": { + "/routes": { "get": { "produces": [ "application/json" ], "tags": [ - "Observers" + "Routes" ], - "summary": "Get observer telemetry history", + "summary": "List known routes", "parameters": [ { "type": "string", - "description": "Observer UUID", - "name": "observerId", - "in": "path", - "required": true + "description": "Filter by IATA code", + "name": "iata", + "in": "query" }, { - "type": "string", - "description": "Duration window e.g. 24h, 48h, 168h (default 24h)", - "name": "range", + "type": "integer", + "description": "Filter by exact hop count", + "name": "hopCount", "in": "query" }, { "type": "integer", - "description": "Return points after this telemetry ID for WS reconnection backfill", - "name": "afterId", + "description": "Epoch ms timestamp of last item for pagination", + "name": "cursor", "in": "query" }, { - "type": "string", - "description": "Bucketing interval: 1h (default), 6h, or 24h", - "name": "interval", + "type": "integer", + "description": "id of the last item; with cursor, also returns later routes sharing that millisecond", + "name": "cursorId", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", "in": "query" } ], @@ -977,7 +2037,10 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetry" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + } } }, "400": { @@ -995,112 +2058,53 @@ const docTemplate = `{ } } }, - "/packets": { + "/routes/cross": { "get": { "produces": [ "application/json" ], "tags": [ - "Packets" + "Routes" ], - "summary": "List packets", + "summary": "Search for routes that cross IATA boundaries", "parameters": [ - { - "type": "integer", - "description": "Filter by payload type integer", - "name": "payloadType", - "in": "query" - }, - { - "type": "string", - "description": "Filter by multiple payload types, comma-separated e.g. 2,4", - "name": "payloadTypes", - "in": "query" - }, - { - "type": "string", - "description": "Filter by payload type name (advert, grp_txt, txt_msg, trace, anon_req)", - "name": "payloadTypeName", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by route type (0=transport_flood, 1=flood, 2=direct, 3=transport_direct)", - "name": "routeType", - "in": "query" - }, { "type": "string", - "description": "Filter by multiple route types, comma-separated e.g. 0,1", - "name": "routeTypes", - "in": "query" - }, - { - "type": "string", - "description": "Filter by single IATA code (case-insensitive)", - "name": "iata", - "in": "query" - }, - { - "type": "string", - "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", - "name": "iatas", - "in": "query" + "description": "Source node hash prefix (hex)", + "name": "fromHash", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", - "name": "scope", - "in": "query" + "description": "Source IATA code", + "name": "fromIata", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by multiple transport scope names, comma-separated e.g. %23bc,%23west", - "name": "scopes", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by region ID, expands to member IATAs", - "name": "regionId", - "in": "query" + "description": "Destination node hash prefix (hex)", + "name": "toHash", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", - "name": "region", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by first_heard_at \u003e= since (epoch ms)", - "name": "since", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by first_heard_at \u003c= until (epoch ms)", - "name": "until", - "in": "query" - }, - { - "type": "integer", - "description": "epoch ms of last item for pagination; last_heard_at, or site-local heard_at when iatas is set", - "name": "cursor", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 50)", - "name": "limit", - "in": "query" + "description": "Destination IATA code", + "name": "toIata", + "in": "query", + "required": true } ], "responses": { "200": { "description": "OK", "schema": { - "type": "object" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATARoute" + } } }, "400": { @@ -1118,70 +2122,36 @@ const docTemplate = `{ } } }, - "/packets/backfill": { + "/routes/search": { "get": { "produces": [ "application/json" ], "tags": [ - "Packets" + "Routes" ], - "summary": "Backfill packets after a given observation ID", + "summary": "Search known routes by source and destination hash", "parameters": [ { - "type": "integer", - "description": "Return packets with observations after this ID (use last WS event observation ID)", - "name": "afterObservationId", + "type": "string", + "description": "IATA code to search within", + "name": "iata", "in": "query", "required": true }, - { - "type": "integer", - "description": "Filter by payload type integer", - "name": "payloadType", - "in": "query" - }, - { - "type": "string", - "description": "Filter by payload type name", - "name": "payloadTypeName", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by route type", - "name": "routeType", - "in": "query" - }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated", - "name": "iatas", - "in": "query" - }, - { - "type": "string", - "description": "Filter by region slug", - "name": "region", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by region ID", - "name": "regionId", - "in": "query" + "description": "Source node hash prefix (hex)", + "name": "from", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by transport scope name", - "name": "scope", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 100)", - "name": "limit", - "in": "query" + "description": "Destination node hash prefix (hex)", + "name": "to", + "in": "query", + "required": true } ], "responses": { @@ -1190,7 +2160,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" } } }, @@ -1209,29 +2179,69 @@ const docTemplate = `{ } } }, - "/packets/{packetHash}": { + "/routes/{iata}/{pathKey}/observations": { "get": { + "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", "produces": [ "application/json" ], "tags": [ - "Packets" + "Routes" ], - "summary": "Get full packet detail. For trace packets (payloadType=9), includes resolvedRoute", + "summary": "Get retained reports matching a full saved route prefix sequence", "parameters": [ { "type": "string", - "description": "Packet hash (hex)", - "name": "packetHash", + "description": "Three-character IATA code", + "name": "iata", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Stable 32-hex pathKey from a known-route response", + "name": "pathKey", "in": "path", "required": true + }, + { + "type": "string", + "description": "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Window start epoch ms; provide with until (default last 24h)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Exclusive window end epoch ms; maximum span 30d, end may be up to 5 minutes ahead of server time (clock skew tolerance)", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Default 50; positive, capped at 200", + "name": "limit", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Packet" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence" } }, "400": { @@ -1255,27 +2265,54 @@ const docTemplate = `{ } } }, - "/regions": { + "/scopes": { "get": { + "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", "produces": [ "application/json" ], "tags": [ - "Regions" + "Scopes" + ], + "summary": "List transport scopes", + "parameters": [ + { + "type": "string", + "description": "Filter by IATA code(s), comma-separated", + "name": "iatas", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID", + "name": "regionId", + "in": "query" + } ], - "summary": "List all regions", "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RegionSummary" + "type": "string" } } }, - "404": { - "description": "Not Found", + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -1283,20 +2320,21 @@ const docTemplate = `{ } } }, - "/regions/{regionId}": { + "/scopes/{name}": { "get": { + "description": "packetCount sums packets per hour across every retained rollup hour; iatas are the IATAs that heard the scope's packets. Observer and node counts are current memberships.", "produces": [ "application/json" ], "tags": [ - "Regions" + "Scopes" ], - "summary": "Get a single region", + "summary": "Get scope detail by name", "parameters": [ { - "type": "integer", - "description": "Region ID", - "name": "regionId", + "type": "string", + "description": "Scope name e.g. %23bc (URL-encoded #bc)", + "name": "name", "in": "path", "required": true } @@ -1305,17 +2343,17 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Region" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail" } }, - "400": { - "description": "Bad Request", + "404": { + "description": "Not Found", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -1323,37 +2361,39 @@ const docTemplate = `{ } } }, - "/routes": { + "/stats/clock-drift": { "get": { "produces": [ "application/json" ], "tags": [ - "Routes" + "Stats" ], - "summary": "List known routes", + "summary": "Repeaters/room servers whose clock has drifted beyond the configured threshold, worst first", "parameters": [ { "type": "string", - "description": "Filter by IATA code", - "name": "iata", + "description": "Comma-separated IATA codes", + "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Filter by exact hop count", - "name": "hopCount", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", "in": "query" }, { - "type": "integer", - "description": "Epoch ms timestamp of last item for pagination", - "name": "cursor", + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1364,7 +2404,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry" } } }, @@ -1377,43 +2417,33 @@ const docTemplate = `{ } } }, - "/routes/cross": { + "/stats/node-types": { "get": { "produces": [ "application/json" ], "tags": [ - "Routes" + "Stats" ], - "summary": "Search for routes that cross IATA boundaries", + "summary": "Node type breakdown", "parameters": [ { "type": "string", - "description": "Source node hash prefix (hex)", - "name": "fromHash", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "Source IATA code", - "name": "fromIata", - "in": "query", - "required": true + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" }, { - "type": "string", - "description": "Destination node hash prefix (hex)", - "name": "toHash", - "in": "query", - "required": true + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" }, { "type": "string", - "description": "Destination IATA code", - "name": "toIata", - "in": "query", - "required": true + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" } ], "responses": { @@ -1422,16 +2452,10 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATARoute" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeTypeCount" } } }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/internal_api_handlers.APIError" - } - }, "500": { "description": "Internal Server Error", "schema": { @@ -1441,36 +2465,40 @@ const docTemplate = `{ } } }, - "/routes/search": { + "/stats/observations": { "get": { + "description": "Observation counts only; distinct packet and observer counts don't sum across IATAs, so read them from /stats/series.", "produces": [ "application/json" ], "tags": [ - "Routes" + "Stats" ], - "summary": "Search known routes by source and destination hash", + "summary": "Hourly observation counts per IATA", "parameters": [ { "type": "string", - "description": "IATA code to search within", - "name": "iata", - "in": "query", - "required": true + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" }, { - "type": "string", - "description": "Source node hash prefix (hex)", - "name": "from", - "in": "query", - "required": true + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" }, { "type": "string", - "description": "Destination node hash prefix (hex)", - "name": "to", - "in": "query", - "required": true + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Start of window epoch ms (default 7 days ago)", + "name": "since", + "in": "query" } ], "responses": { @@ -1479,16 +2507,10 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint" } } }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/internal_api_handlers.APIError" - } - }, "500": { "description": "Internal Server Error", "schema": { @@ -1498,40 +2520,81 @@ const docTemplate = `{ } } }, - "/scopes": { + "/stats/observer-comparison": { "get": { + "description": "Counts distinct retained packet hashes with route type 0 or 1, using reception time in [since, until). onlyA, onlyB and both partition totalPackets (the union). Repeated receptions count once. These are reported receptions, not a measure of radio packet loss or continuous observer coverage.", "produces": [ "application/json" ], "tags": [ - "Scopes" + "Stats" ], - "summary": "List transport scopes", + "summary": "Compare flood packets reported by two observers", "parameters": [ { "type": "string", - "description": "Filter by IATA code(s), comma-separated", - "name": "iatas", - "in": "query" + "description": "First observer UUID", + "name": "observerA", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by region slug", - "name": "region", + "description": "Second, distinct observer UUID", + "name": "observerB", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; later than since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Filter by region ID", + "description": "Region ID, expands to member IATAs", "name": "regionId", "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "type": "object" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -1539,39 +2602,51 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, - "/scopes/{name}": { + "/stats/overview": { "get": { + "description": "Summarises the 24 most recent hours that can have been rolled (an hour rolls about 95 minutes after it closes); since/until report that window. Packets count once per hour heard. Use /stats/series for sparklines.", "produces": [ "application/json" ], "tags": [ - "Scopes" + "Stats" ], - "summary": "Get scope detail by name", + "summary": "Network overview stats (last 24 rolled hours)", "parameters": [ { "type": "string", - "description": "Scope name e.g. %23bc (URL-encoded #bc)", - "name": "name", - "in": "path", - "required": true + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/internal_api_handlers.APIError" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview" } }, "500": { @@ -1583,31 +2658,46 @@ const docTemplate = `{ } } }, - "/stats/node-types": { + "/stats/paths": { "get": { + "description": "Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly path rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Node type breakdown", + "summary": "Received path-entry and hash-width distributions", "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, { "type": "string", - "description": "Comma-separated IATA codes", + "description": "Comma-separated reception IATA codes", "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Filter by region ID, expands to member IATAs", + "description": "Region ID, expands to member IATAs", "name": "regionId", "in": "query" }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", + "description": "Region slug, expands to member IATAs", "name": "region", "in": "query" } @@ -1616,10 +2706,13 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeTypeCount" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -1627,11 +2720,17 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, - "/stats/observations": { + "/stats/payload-breakdown": { "get": { "produces": [ "application/json" @@ -1639,7 +2738,7 @@ const docTemplate = `{ "tags": [ "Stats" ], - "summary": "Hourly observation time series", + "summary": "Observation counts by payload type (last 24h by default)", "parameters": [ { "type": "string", @@ -1661,7 +2760,7 @@ const docTemplate = `{ }, { "type": "integer", - "description": "Start of window epoch ms (default 7 days ago)", + "description": "Start of window epoch ms (default last 24h)", "name": "since", "in": "query" } @@ -1672,7 +2771,7 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem" } } }, @@ -1685,7 +2784,7 @@ const docTemplate = `{ } } }, - "/stats/overview": { + "/stats/radio-presets": { "get": { "produces": [ "application/json" @@ -1693,8 +2792,14 @@ const docTemplate = `{ "tags": [ "Stats" ], - "summary": "Network overview stats (last 24h)", + "summary": "Radio preset usage by IATA", "parameters": [ + { + "type": "string", + "description": "Filter by preset string e.g. 910.525,62.5,7", + "name": "preset", + "in": "query" + }, { "type": "string", "description": "Comma-separated IATA codes", @@ -1718,7 +2823,10 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RadioPreset" + } } }, "500": { @@ -1730,15 +2838,16 @@ const docTemplate = `{ } } }, - "/stats/payload-breakdown": { + "/stats/scopes": { "get": { + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour and adds the distinct observers and advertising nodes active in the scope each hour (empty hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Observation counts by payload type (last 24h by default)", + "summary": "Scope statistics", "parameters": [ { "type": "string", @@ -1746,6 +2855,12 @@ const docTemplate = `{ "name": "iatas", "in": "query" }, + { + "type": "string", + "description": "Single IATA code; used when iatas is absent", + "name": "iata", + "in": "query" + }, { "type": "integer", "description": "Filter by region ID, expands to member IATAs", @@ -1754,13 +2869,13 @@ const docTemplate = `{ }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", + "description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs", "name": "region", "in": "query" }, { "type": "integer", - "description": "Start of window epoch ms (default last 24h)", + "description": "Start of packet window epoch ms, rounded down to the hour (default 7 days ago)", "name": "since", "in": "query" } @@ -1771,10 +2886,16 @@ const docTemplate = `{ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats" } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -1784,37 +2905,46 @@ const docTemplate = `{ } } }, - "/stats/radio-presets": { + "/stats/series": { "get": { + "description": "One entry per UTC hour in [since, until) plus a summary over the complete hours. Both bounds round down to UTC hours and since is clamped to the rollup retention; the response reports the effective window. An hour is rolled about 95 minutes after it closes, so recent hours are \"missing\" until then; \"partial\" hours lost raw rows before they could be rolled and never get values. Counts (observations, packets) sum across hours, with a packet counted once in each hour it was heard; observers, IATAs and scopes are distinct across the whole window; averages are sum / samples. revision changes whenever any rolled hour changes.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Radio preset usage by IATA", + "summary": "Hourly network activity from the analytics rollups", "parameters": [ { - "type": "string", - "description": "Filter by preset string e.g. 910.525,62.5,7", - "name": "preset", - "in": "query" + "type": "integer", + "description": "Inclusive start, epoch milliseconds", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; the window is at most the rollup retention (default 90 days)", + "name": "until", + "in": "query", + "required": true }, { "type": "string", - "description": "Comma-separated IATA codes", + "description": "Comma-separated reception IATA codes", "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Filter by region ID, expands to member IATAs", + "description": "Region ID, expands to member IATAs", "name": "regionId", "in": "query" }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", + "description": "Region slug, expands to member IATAs", "name": "region", "in": "query" } @@ -1823,10 +2953,13 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RadioPreset" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -1834,27 +2967,71 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, - "/stats/scopes": { + "/stats/signal": { "get": { + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly signal rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Scope statistics", + "summary": "Reception signal distributions and hourly trends", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -1862,12 +3039,19 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, "/stats/top-advertisers": { "get": { + "description": "Each advert counts once per hour heard, however many requested IATAs heard it. nodeId is null when the node row has been deleted.", "produces": [ "application/json" ], @@ -1901,8 +3085,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1928,13 +3114,14 @@ const docTemplate = `{ }, "/stats/top-nodes": { "get": { + "description": "nodeId is null when the node row has been deleted; publicKey always identifies it.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Top N nodes by observation count (from materialized view)", + "summary": "Top N nodes by advert hearings (last 7 days by default)", "parameters": [ { "type": "string", @@ -1956,7 +3143,15 @@ const docTemplate = `{ }, { "type": "integer", - "description": "Max results (default 10)", + "description": "Start of window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2015,8 +3210,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2075,8 +3272,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2130,13 +3329,13 @@ const docTemplate = `{ }, { "type": "string", - "description": "Filter by transport scope name", + "description": "Filter by the tag's first transport scope name", "name": "scope", "in": "query" }, { "type": "string", - "description": "Filter by type: TRACE or PING (default: all)", + "description": "Filter by type: TRACE (any multi-hop packet) or PING (default: all)", "name": "type", "in": "query" }, @@ -2154,13 +3353,21 @@ const docTemplate = `{ }, { "type": "integer", - "description": "last_heard_at epoch ms of last item for pagination", + "description": "lastHeardAt (epoch ms) of the last item; returns older tags", "name": "cursor", "in": "query" }, { + "type": "string", + "description": "traceTag (hex) of the last item; with cursor, also returns later tags sharing that millisecond", + "name": "cursorTag", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2175,6 +3382,12 @@ const docTemplate = `{ } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -2226,6 +3439,97 @@ const docTemplate = `{ } }, "definitions": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Account": { + "type": "object", + "properties": { + "active": { + "type": "boolean" + }, + "created_at": { + "type": "string" + }, + "deactivated_at": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig": { + "type": "object", + "properties": { + "configured": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig": { + "type": "object", + "properties": { + "allow_credentials": { + "type": "boolean" + }, + "allowed_headers": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowed_methods": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowed_origins": { + "type": "array", + "items": { + "type": "string" + } + }, + "max_age": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig": { + "type": "object", + "properties": { + "auth": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig" + }, + "cors": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig" + }, + "ingest": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig": { + "type": "object", + "properties": { + "broker_count": { + "description": "BrokerCount counts configured broker workers, not active MQTT connections\nor a configurable processing-worker pool.", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.AdvertObservation": { "type": "object", "properties": { @@ -2299,6 +3603,7 @@ const docTemplate = `{ "type": "integer" }, "messageCount": { + "description": "lifetime count; not reduced by retention", "type": "integer" }, "name": { @@ -2329,6 +3634,19 @@ const docTemplate = `{ "description": "hex-encoded packet hash for correlation with packet events", "type": "string" }, + "scope": { + "description": "matched scope on the first stored packet; null when none was recorded", + "type": "string" + }, + "scopeStatus": { + "type": "string", + "enum": [ + "matched", + "unscoped", + "unknown", + "unavailable" + ] + }, "senderName": { "description": "display name from the decrypted payload", "type": "string" @@ -2339,6 +3657,26 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage": { + "type": "object", + "properties": { + "hasMore": { + "type": "boolean" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary" + } + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary": { "type": "object", "properties": { @@ -2367,6 +3705,49 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry": { + "type": "object", + "properties": { + "clockCheckedAt": { + "description": "epoch ms", + "type": "integer" + }, + "clockDriftSeconds": { + "description": "signed; +ve = device ahead of server", + "type": "integer" + }, + "iatas": { + "description": "IATAs this node has been heard in", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeIATA" + } + }, + "nodeId": { + "type": "string" + }, + "nodeName": { + "type": "string" + }, + "nodeType": { + "type": "integer" + }, + "nodeTypeName": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop": { "type": "object", "properties": { @@ -2473,6 +3854,10 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" + }, + "pathKey": { + "description": "stable identity within this IATA; use for route evidence links", + "type": "string" } } }, @@ -2568,6 +3953,10 @@ const docTemplate = `{ "description": "UUID of the associated observer row, if any", "type": "string" }, + "possiblyForeign": { + "description": "PossiblyForeign compares a repeater's stored position with this server's\nconfigured IATA border union. Omitted when disabled, unknown, or not repeater.", + "type": "boolean" + }, "publicKey": { "description": "hex-encoded Ed25519 public key", "type": "string" @@ -2698,6 +4087,10 @@ const docTemplate = `{ "description": "UUID of the associated observer row, if any", "type": "string" }, + "possiblyForeign": { + "description": "PossiblyForeign compares a repeater's stored position with this server's\nconfigured IATA border union. Omitted when disabled, unknown, or not repeater.", + "type": "boolean" + }, "publicKey": { "description": "hex-encoded Ed25519 public key", "type": "string" @@ -2729,9 +4122,6 @@ const docTemplate = `{ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint": { "type": "object", "properties": { - "activeObservers": { - "type": "integer" - }, "hour": { "description": "epoch ms, start of the 1-hour bucket", "type": "integer" @@ -2741,9 +4131,6 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" - }, - "uniquePackets": { - "type": "integer" } } }, @@ -2794,6 +4181,7 @@ const docTemplate = `{ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -2839,13 +4227,132 @@ const docTemplate = `{ "type": "string" }, "statusMetadata": { - "description": "raw /status JSON payload" + "description": "raw /status JSON payload", + "type": "object" }, "uptimeSeconds": { "type": "integer" } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { + "type": "object", + "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, + "interval": { + "type": "string" + }, + "payloadTypes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem" + } + }, + "points": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint" + } + }, + "radio": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio" + }, + "range": { + "type": "string" + }, + "rawFrom": { + "description": "start of the raw tail, epoch ms", + "type": "integer" + }, + "rolledUntil": { + "description": "Hourly only. Buckets before rolledUntil come from rollups and from rawFrom on from raw rows;\nwhen rawFrom is later, the hours between are uncovered, not quiet.", + "type": "integer" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive end, epoch ms; live requests include the current partial bucket", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint": { + "type": "object", + "properties": { + "airtimeMs": { + "type": "number" + }, + "observations": { + "type": "integer" + }, + "rssiAvg": { + "type": "number" + }, + "snrAvg": { + "type": "number" + }, + "snrMin": { + "type": "number" + }, + "t": { + "description": "bucket start, epoch ms", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio": { + "type": "object", + "properties": { + "bwKhz": { + "type": "number" + }, + "cr": { + "type": "integer" + }, + "freqMhz": { + "type": "number" + }, + "preambleSymbols": { + "type": "integer" + }, + "sf": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "description": "stored observations within windowStart/windowEnd", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { @@ -2863,6 +4370,37 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison": { + "type": "object", + "properties": { + "both": { + "type": "integer" + }, + "observerA": { + "type": "string" + }, + "observerB": { + "type": "string" + }, + "onlyA": { + "type": "integer" + }, + "onlyB": { + "type": "integer" + }, + "since": { + "description": "inclusive, epoch milliseconds", + "type": "integer" + }, + "totalPackets": { + "type": "integer" + }, + "until": { + "description": "exclusive, epoch milliseconds", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary": { "type": "object", "properties": { @@ -2918,10 +4456,10 @@ const docTemplate = `{ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetryPoint": { "type": "object", "properties": { - "airtimeRxPct": { + "airtimeRxSecs": { "type": "number" }, - "airtimeTxPct": { + "airtimeTxSecs": { "type": "number" }, "batteryMv": { @@ -3053,6 +4591,30 @@ const docTemplate = `{ }, "id": { "type": "string" + }, + "pathBytes": { + "description": "hex-encoded accumulated path hashes", + "type": "string" + }, + "pathLength": { + "description": "PathLength/PathBytes and captured endpoint resolutions are stored on the observation,\nso list/backfill reads need no per-hash resolution queries. Legacy observations have\nno endpoint snapshot. ResolvedPath remains a detail/opted-in WS feature.", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketPathLength" + } + ] + }, + "resolvedDestination": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" + }, + "resolvedPath": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" + } + }, + "resolvedSource": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" } } }, @@ -3100,7 +4662,7 @@ const docTemplate = `{ } }, "resolvedSource": { - "description": "ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type\ncarries a resolvable one: an exact match for ADVERT's full pubkey, an ambiguous\nhash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte\nsource/destination hashes. Nil when the payload type doesn't carry one at all (e.g.\nGRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and\nResolveExactNode for how each is built.", + "description": "ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type\ncarries one. Prefer the snapshot captured at ingest; legacy observations without a\nsnapshot use the current node registry. Endpoint matching itself is unchanged:\nan exact match for ADVERT's full pubkey, an ambiguous\nhash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte\nsource/destination hashes. Nil when the payload type doesn't carry one at all (e.g.\nGRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and\nResolveExactNode for how each is built.", "allOf": [ { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" @@ -3225,7 +4787,7 @@ const docTemplate = `{ "type": "string" }, "summary": { - "description": "human-readable payload summary", + "description": "advert name, or the ACK/TRACE/PING summary; omitted when unavailable", "type": "string" } } @@ -3258,7 +4820,7 @@ const docTemplate = `{ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary": { "type": "object", "properties": { "hasMore": { @@ -3267,7 +4829,7 @@ const docTemplate = `{ "items": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeSummary" } }, "nextCursor": { @@ -3275,7 +4837,7 @@ const docTemplate = `{ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ObserverSummary": { "type": "object", "properties": { "hasMore": { @@ -3284,7 +4846,7 @@ const docTemplate = `{ "items": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary" } }, "nextCursor": { @@ -3292,7 +4854,7 @@ const docTemplate = `{ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ObserverSummary": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_PacketObservationSummary": { "type": "object", "properties": { "hasMore": { @@ -3301,7 +4863,7 @@ const docTemplate = `{ "items": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketObservationSummary" } }, "nextCursor": { @@ -3309,19 +4871,101 @@ const docTemplate = `{ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_PacketObservationSummary": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth": { "type": "object", "properties": { - "hasMore": { - "type": "boolean" + "bytes": { + "type": "integer" + }, + "receptions": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour": { + "type": "object", + "properties": { + "empty": { + "type": "integer" + }, + "hour": { + "type": "integer" + }, + "maxEntries": { + "description": "longest path received that hour across the requested IATAs; 0 if only empty paths", + "type": "integer" + }, + "oneByte": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "threeByte": { + "type": "integer" + }, + "trace": { + "type": "integer" + }, + "twoByte": { + "type": "integer" + }, + "unclassified": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin": { + "type": "object", + "properties": { + "entries": { + "type": "integer" + }, + "receptions": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats": { + "type": "object", + "properties": { + "empty": { + "type": "integer" + }, + "hashWidths": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth" + } + }, + "hashed": { + "type": "integer" + }, + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour" + } }, - "items": { + "pathLengths": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketObservationSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin" } }, - "nextCursor": { + "receptions": { + "type": "integer" + }, + "since": { + "type": "integer" + }, + "trace": { + "type": "integer" + }, + "unclassified": { + "type": "integer" + }, + "until": { "type": "integer" } } @@ -3465,6 +5109,47 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence": { + "type": "object", + "properties": { + "generatedAt": { + "type": "integer" + }, + "hasMore": { + "type": "boolean" + }, + "hashSize": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation" + } + }, + "matchAvailable": { + "type": "boolean" + }, + "matchType": { + "type": "string" + }, + "nextPageCursor": { + "type": "string" + }, + "pathBytes": { + "type": "string" + }, + "route": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + }, + "windowEnd": { + "type": "integer" + }, + "windowStart": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop": { "type": "object", "properties": { @@ -3485,6 +5170,38 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation": { + "type": "object", + "properties": { + "heardAt": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "observerId": { + "type": "string" + }, + "observerName": { + "type": "string" + }, + "packetHash": { + "type": "string" + }, + "payloadType": { + "type": "integer" + }, + "payloadTypeName": { + "type": "string" + }, + "rssi": { + "type": "integer" + }, + "snr": { + "type": "number" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail": { "type": "object", "properties": { @@ -3511,9 +5228,36 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the UTC hour", + "type": "integer" + }, + "nodes": { + "description": "distinct nodes whose adverts were heard in the scope that hour", + "type": "integer" + }, + "observers": { + "description": "distinct observers that heard the scope's packets that hour", + "type": "integer" + }, + "packets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats": { "type": "object", "properties": { + "hourly": { + "description": "packetCount by hour, oldest first; hours with none omitted", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour" + } + }, "name": { "description": "normalized scope name e.g. \"#bc\"", "type": "string" @@ -3532,6 +5276,86 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "lower": { + "type": "number" + }, + "upper": { + "type": "number" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "rssiAverage": { + "type": "number" + }, + "rssiSamples": { + "type": "integer" + }, + "snrAverage": { + "type": "number" + }, + "snrSamples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric": { + "type": "object", + "properties": { + "average": { + "type": "number" + }, + "histogram": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin" + } + }, + "samples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats": { + "type": "object", + "properties": { + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour" + } + }, + "receptions": { + "type": "integer" + }, + "rssi": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "since": { + "type": "integer" + }, + "snr": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview": { "type": "object", "properties": { @@ -3541,14 +5365,126 @@ const docTemplate = `{ "activeObservers": { "type": "integer" }, + "since": { + "description": "epoch ms, start of the window", + "type": "integer" + }, "totalObservations": { "type": "integer" }, "totalPackets": { + "description": "distinct per hour (see StatsSeriesValues)", + "type": "integer" + }, + "until": { + "description": "epoch ms, exclusive end of the window", "type": "integer" }, "windowHours": { - "description": "always 24 for now", + "description": "complete rolled hours the totals cover, at most 24", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries": { + "type": "object", + "properties": { + "completeHours": { + "description": "hours contributing to summary", + "type": "integer" + }, + "earliestComplete": { + "description": "first complete hour held, epoch ms; null if none", + "type": "integer" + }, + "hours": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour" + } + }, + "revision": { + "description": "changes whenever rolled hours, their status or coverage change", + "type": "integer" + }, + "since": { + "description": "effective start, epoch ms on a UTC hour", + "type": "integer" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + }, + "until": { + "description": "effective exclusive end, epoch ms on a UTC hour", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the hour", + "type": "integer" + }, + "status": { + "description": "partial: raw rows deleted before it was rolled", + "type": "string", + "enum": [ + "complete", + "partial", + "missing" + ] + }, + "values": { + "description": "null unless complete", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + } + ] + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues": { + "type": "object", + "properties": { + "activeIatas": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeObservers": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeScopes": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "maxPathEntries": { + "type": "integer" + }, + "observations": { + "type": "integer" + }, + "rssiSamples": { + "type": "integer" + }, + "rssiSum": { + "description": "average = rssiSum / rssiSamples", + "type": "number" + }, + "scopedPackets": { + "type": "integer" + }, + "snrSamples": { + "type": "integer" + }, + "snrSum": { + "description": "average = snrSum / snrSamples", + "type": "number" + }, + "uniquePackets": { "type": "integer" } } @@ -3559,6 +5495,13 @@ const docTemplate = `{ "advertCount": { "type": "integer" }, + "directAdvertCount": { + "type": "integer" + }, + "floodAdvertCount": { + "description": "FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed:\nflood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path;\ndirect = route type 2 (direct) or 3 (transport_direct), routed along a known path.\nFloodAdvertCount + DirectAdvertCount == AdvertCount.", + "type": "integer" + }, "iata": { "type": "string" }, @@ -3567,6 +5510,7 @@ const docTemplate = `{ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -3577,6 +5521,10 @@ const docTemplate = `{ }, "nodeTypeName": { "type": "string" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, @@ -3591,6 +5539,7 @@ const docTemplate = `{ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -3604,6 +5553,10 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, @@ -3744,6 +5697,45 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest": { + "type": "object", + "required": [ + "allowed_origins" + ], + "properties": { + "allowed_origins": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest": { + "type": "object", + "required": [ + "cors" + ], + "properties": { + "cors": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse": { + "type": "object", + "properties": { + "config": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig" + }, + "persisted": { + "type": "boolean" + }, + "requires_restart": { + "type": "boolean" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { @@ -3769,6 +5761,14 @@ const docTemplate = `{ } } }, + "securityDefinitions": { + "AdminKey": { + "description": "Enter Bearer followed by the configured operator key. Use HTTPS.", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + }, "tags": [ { "description": "Airport/location codes that group observers and packets", @@ -3811,16 +5811,16 @@ const docTemplate = `{ // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ - Version: "1.6.0", + Version: "2.0.0", Host: "localhost:8080", BasePath: "/api/v1", Schemes: []string{"http", "https"}, Title: "MeshCore Beacon API", - Description: "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.", + Description: "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.\nREST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget.\nWebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open.", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, - LeftDelim: "[[", - RightDelim: "]]", + LeftDelim: "{{", + RightDelim: "}}", } func init() { diff --git a/docs/historical-stats.md b/docs/historical-stats.md new file mode 100644 index 00000000..c039ad5a --- /dev/null +++ b/docs/historical-stats.md @@ -0,0 +1,90 @@ +# Historical stats + +## The rule + +Historical stats never scan `packet_observations` per request or per refresh. They read +hourly rollup tables filled once per closed hour by one background task. Raw rows are read +only for per-entity drill-downs (a packet, a trace, a route), observer comparison and +sub-hour observer activity. + +## Rollup contract + +- **Hours.** `hour` is the UTC `date_trunc('hour', heard_at)` and leads every rollup table's + primary key. Rollup tables have no foreign keys: history outlives deleted observers, nodes + and IATAs. +- **Eligibility.** Hour `[H, H+1h)` can be rolled once `now() >= H + 95 min`. Ingest clamps + `heard_at` to ±30 minutes of server time, so nothing can still land in it. +- **Registration.** Every eligible hour since the earliest retained observation gets a row + in `analytics_rollup_hours`, including hours with no traffic. Status is `missing`, + `complete` or `partial`. +- **Rolling.** One process holds the advisory lock (`analytics_rollup` task, every 5 + minutes, plus a catch-up at boot). Each hour is rolled in one REPEATABLE READ transaction: + copy its observations (joined to packet dimensions) into a temp table, delete and reinsert + every family, hash the result, mark the hour complete. A serialization failure skips the + hour until the next pass. +- **Revision.** `analytics_state.revision` increases when an hour's content hash changes or + an hour turns partial; `analytics_raw_state.coverage` increases when retention deletes + rollup hours. Cache keys for rollup-backed endpoints include their sum (each counter only + grows), read through a 10 s memo. +- **Raw deletion.** Cleanup deletes packets older than `packets.retention`, but holds the + cutoff 35 minutes behind the oldest `missing` hour, for at most 24 hours (it logs a + warning past 6). Each batch advances `analytics_raw_state.raw_deleted_before` to the + newest deleted `last_heard_at`. Cleanup and the rollup never write the same row. +- **Partial hours.** A not-yet-complete hour starting at or before `raw_deleted_before + 30 min` + lost raw rows before it was rolled. It is marked `partial` and never rolled; readers + report it without values. +- **Re-rolls.** Hours are not re-rolled periodically. A message stored by channel backfill (at + startup, and when keys are imported at runtime) queues every hour its packet was heard in + `analytics_dirty_hours`, in the same statement as the message insert. Each pass re-rolls a + few complete ones while their raw rows remain. A queued hour still waiting for its first roll + keeps its entry unless that roll's snapshot already included it. +- **Retention.** Rollups are kept for `analytics.rollup_retention` (default 90 days), which + is also the longest window `/stats/series` accepts. + +## Combining hours + +| Metric | Across hours and IATAs | +| --- | --- | +| Observations, receptions, payload counts | Sum | +| Unique packets, adverts, messages, scoped packets | Sum of hourly IATA-set counts: each is counted once per hour it was heard, so one heard across an hour boundary counts twice (measured +1.1%) | +| Active observers, IATAs, scopes | Distinct across the window | +| Active scope observers and nodes (per scope, per hour) | Distinct within the hour, across the requested IATAs | +| Max path entries | Max | +| SNR / RSSI averages | Σ sum / Σ samples, never an average of averages | + +Windows snap to UTC hours on the server, and responses report the effective window. +`/stats/overview` spans the 24 newest eligible hours; its `windowHours` counts the complete +ones its totals cover. + +## IATA sets + +A packet heard in YVR and YYZ appears in both IATAs' per-IATA rows, so summing per-IATA +distinct counts overcounts (2.26× on production data). The `*_sets` tables instead group +each distinct packet, advert or message under the exact sorted set of IATAs that heard it +that hour (`iatas char(3)[]`, kept canonical by a CHECK). For any requested set S, the rows +where `iatas && S` count every item heard in S exactly once. This is exact for one IATA, a +region or all IATAs. + +## Raw or current, by design + +- Packet, trace and route detail, the packet list, observer comparison, and observer + activity below one hour. Hourly observer activity reads raw rows for the hours after the + newest complete one (at most the last 24 hours, the raw holdback), so the unrolled tail isn't + shown as zero. `rolledUntil` and `rawFrom` in the response mark any hours still uncovered. +- Current memberships: scope observer and node counts, node types, clock drift, radio presets. +- `/traces` reads `trace_tags`, a per-tag summary kept at ingest, not a rollup. A new packet + adds its count, type and payload in the statement that stores it; hearings set the times. + +## Adding a metric + +1. Add a family table in a new numbered migration (`001_baseline.sql` is frozen from 2.0.0): `hour` + first in the primary key, no foreign keys, and a set table with + `CHECK (iata_set_is_canonical(iatas))` plus a GIN index if it needs distinct counts. +2. Add a delete in `DeleteRollupHour`, a `Roll*` insert over `rollup_obs` in + `db/queries/rollup.sql`, a line in `RollupContentHash` and `DeleteOldRollups`, and the + call in `rollHour` (`db/rollup.go`). +3. Read it with an hour-bounded query, using the UNION-ALL IATA guard (see `series.sql`) for + per-IATA tables or `iatas && @iatas` for set tables. +4. Cache it with `cachedRollup` so its key carries the revision. +5. Add a raw-SQL oracle case to `db/rollup_integration_test.go`. +6. Run `sqlc generate`, regenerate mocks and swagger. diff --git a/docs/swagger.json b/docs/swagger.json index aa876733..809064f6 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -5,7 +5,7 @@ ], "swagger": "2.0", "info": { - "description": "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.", + "description": "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.\nREST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget.\nWebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open.", "title": "MeshCore Beacon API", "termsOfService": "https://github.com/MeshCore-Beacon/beacon-server", "contact": { @@ -15,70 +15,949 @@ "license": { "name": "AGPL-3-or-later" }, - "version": "1.6.0" + "version": "2.0.0" }, "host": "localhost:8080", "basePath": "/api/v1", "paths": { + "/admin/accounts": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Includes active and inactive accounts, newest first. These records are not login users.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "List operator accounts", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "post": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Trims the name; names are case-sensitive and unique among active accounts. No credential or login is created.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Create an operator account", + "parameters": [ + { + "description": "Account name (1-128 Unicode characters, no control characters); body at most 4 KiB", + "name": "account", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/accounts/{id}": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns active or inactive account records.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Get an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "delete": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Soft deactivation preserves the record. Its name may be reused by a new account.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Deactivate an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "Deactivated" + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/backup": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint.", + "produces": [ + "application/gzip" + ], + "tags": [ + "Admin" + ], + "summary": "Download a private database and saved-config backup", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + }, + "headers": { + "Content-Disposition": { + "type": "string", + "description": "attachment; filename=beacon-backup.tar.gz" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "504": { + "description": "Gateway Timeout", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "507": { + "description": "Insufficient Storage", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/config": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns current CORS options, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Runtime origin updates are lost on restart.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Inspect selected running configuration", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "put": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Replaces only cors.allowed_origins immediately. Updates are serialized; concurrent valid updates are applied one at a time. Already-running requests may finish with the previous policy. Nothing is persisted; restart reloads saved configuration.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Update runtime CORS origins", + "parameters": [ + { + "description": "1-32 ASCII HTTP(S) origins, at most 512 bytes each; one hostname wildcard is supported, or a sole *. Empty/null lists and unsupported fields are rejected. Body at most 16 KiB.", + "name": "config", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/brokers": { "get": { "produces": [ "application/json" ], "tags": [ - "Brokers" + "Brokers" + ], + "summary": "List all MQTT brokers and their connection status", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/internal_api_handlers.BrokerStatus" + } + } + } + } + } + }, + "/channels": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Channels" + ], + "summary": "List channels", + "parameters": [ + { + "type": "string", + "description": "Single-byte channel hash (hex)", + "name": "hash", + "in": "query" + }, + { + "type": "string", + "description": "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels", + "name": "iata", + "in": "query" + }, + { + "type": "string", + "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata", + "name": "iatas", + "in": "query" + }, + { + "type": "boolean", + "description": "Only channels Beacon can (true) or cannot (false) decrypt; omit for all", + "name": "keyKnown", + "in": "query" + }, + { + "type": "integer", + "description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning", + "name": "cursor", + "in": "query" + }, + { + "type": "string", + "description": "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed.", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/channels/{channelID}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Channels" + ], + "summary": "Get channel detail", + "parameters": [ + { + "type": "integer", + "description": "Channel integer ID", + "name": "channelID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Channel" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/channels/{channelID}/messages": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Channels" + ], + "summary": "List messages for a channel", + "parameters": [ + { + "type": "integer", + "description": "Channel integer ID", + "name": "channelID", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Return messages after this epoch ms", + "name": "since", + "in": "query" + }, + { + "type": "string", + "description": "Filter by IATA code(s), comma-separated e.g. YVR or YVR,YYJ", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "string", + "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", + "name": "scope", + "in": "query" + }, + { + "type": "integer", + "description": "Message ID of last item for pagination (results ordered newest first)", + "name": "cursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/iatas": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "IATAs" + ], + "summary": "List all IATA codes", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/iatas/{iata}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "IATAs" + ], + "summary": "Get a single IATA code", + "parameters": [ + { + "type": "string", + "description": "3-letter IATA code", + "name": "iata", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/iatas/{iata}/border": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "IATAs" + ], + "summary": "Get an IATA's GeoJSON border, if configured", + "parameters": [ + { + "type": "string", + "description": "3-letter IATA code", + "name": "iata", + "in": "path", + "required": true + } ], - "summary": "List all MQTT brokers and their connection status", "responses": { "200": { - "description": "OK", + "description": "GeoJSON Feature (Polygon or MultiPolygon geometry, with bbox)", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/internal_api_handlers.BrokerStatus" - } + "type": "object" + } + }, + "204": { + "description": "IATA exists but has no border configured" + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } } } } }, - "/channels": { + "/messages": { "get": { "produces": [ "application/json" ], "tags": [ - "Channels" + "Messages" ], - "summary": "List channels", + "summary": "List channel messages", "parameters": [ { - "type": "string", - "description": "Single-byte channel hash (hex)", - "name": "hash", + "type": "integer", + "description": "Filter by channel integer ID (mutually exclusive with channelHash)", + "name": "channelID", "in": "query" }, { "type": "string", - "description": "Filter by IATA code", - "name": "iata", + "description": "Filter by channel hash byte hex (mutually exclusive with channelID)", + "name": "channelHash", + "in": "query" + }, + { + "type": "integer", + "description": "Return messages after this epoch ms", + "name": "since", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ", + "description": "Filter by IATA code(s), comma-separated e.g. YVR or YVR,YYJ", "name": "iatas", "in": "query" }, { "type": "integer", - "description": "last_seen epoch ms of last item for pagination", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "string", + "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", + "name": "scope", + "in": "query" + }, + { + "type": "integer", + "description": "Message ID of last item for pagination (results ordered newest first)", "name": "cursor", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -87,7 +966,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary" + "type": "object" } }, "400": { @@ -105,29 +984,64 @@ } } }, - "/channels/{channelID}": { + "/messages/backfill": { "get": { "produces": [ "application/json" ], "tags": [ - "Channels" + "Messages" ], - "summary": "Get channel detail", + "summary": "Backfill messages after a given message ID", "parameters": [ { "type": "integer", - "description": "Channel integer ID", - "name": "channelID", - "in": "path", + "description": "Return messages after this ID (use last WS event message ID)", + "name": "afterId", + "in": "query", "required": true + }, + { + "type": "string", + "description": "Filter by IATA code(s), comma-separated", + "name": "iatas", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by transport scope name", + "name": "scope", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 100); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Channel" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelMessage" + } } }, "400": { @@ -136,8 +1050,8 @@ "$ref": "#/definitions/internal_api_handlers.APIError" } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -145,32 +1059,37 @@ } } }, - "/channels/{channelID}/messages": { + "/nodes": { "get": { "produces": [ "application/json" ], "tags": [ - "Channels" + "Nodes" ], - "summary": "List messages for a channel", + "summary": "List nodes", "parameters": [ { "type": "integer", - "description": "Channel integer ID", - "name": "channelID", - "in": "path", - "required": true + "description": "Node type integer (1=companion, 2=repeater, 3=room_server, 4=sensor)", + "name": "type", + "in": "query" }, { - "type": "integer", - "description": "Return messages after this epoch ms", - "name": "since", + "type": "string", + "description": "Node type name (companion, repeater, room_server, sensor)", + "name": "typeName", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YVR or YVR,YYJ", + "description": "Filter by single IATA code (case-insensitive)", + "name": "iata", + "in": "query" + }, + { + "type": "string", + "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", "name": "iatas", "in": "query" }, @@ -186,21 +1105,59 @@ "name": "region", "in": "query" }, + { + "type": "string", + "description": "Partial case-insensitive name match", + "name": "name", + "in": "query" + }, { "type": "string", "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", "name": "scope", "in": "query" }, + { + "type": "string", + "description": "Exact public key match (hex)", + "name": "pubkey", + "in": "query" + }, + { + "type": "string", + "description": "Partial public key match: hex prefix, case-insensitive", + "name": "pubkeyPrefix", + "in": "query" + }, + { + "type": "boolean", + "description": "Filter by multibyte path support (true/false); omit for no filter", + "name": "supportsMultibytePaths", + "in": "query" + }, + { + "type": "boolean", + "description": "Filter by multibyte trace support (true/false); omit for no filter", + "name": "supportsMultibyteTraces", + "in": "query" + }, + { + "type": "boolean", + "description": "Include each node's known neighbor IDs (neighborIds field). Bare ?neighbors or ?neighbors=true enables it; omit/false for none", + "name": "neighbors", + "in": "query" + }, { "type": "integer", - "description": "Message ID of last item for pagination (results ordered newest first)", + "description": "last_seen epoch ms of last item for pagination", "name": "cursor", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -209,7 +1166,7 @@ "200": { "description": "OK", "schema": { - "type": "object" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary" } }, "400": { @@ -227,23 +1184,35 @@ } } }, - "/iatas": { + "/nodes/{nodeId}": { "get": { "produces": [ "application/json" ], "tags": [ - "IATAs" + "Nodes" + ], + "summary": "Get node detail", + "parameters": [ + { + "type": "string", + "description": "Node UUID", + "name": "nodeId", + "in": "path", + "required": true + } ], - "summary": "List all IATA codes", "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Node" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "404": { @@ -251,24 +1220,30 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, - "/iatas/{iata}": { + "/nodes/{nodeId}/neighbors": { "get": { "produces": [ "application/json" ], "tags": [ - "IATAs" + "Nodes" ], - "summary": "Get a single IATA code", + "summary": "List neighbors for a node", "parameters": [ { "type": "string", - "description": "3-letter IATA code", - "name": "iata", + "description": "Node UUID", + "name": "nodeId", "in": "path", "required": true } @@ -277,11 +1252,20 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeNeighbor" + } } }, - "404": { - "description": "Not Found", + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -289,36 +1273,53 @@ } } }, - "/iatas/{iata}/border": { + "/nodes/{nodeId}/observations": { "get": { "produces": [ "application/json" ], "tags": [ - "IATAs" + "Nodes" ], - "summary": "Get an IATA's GeoJSON border, if configured", + "summary": "List packet observations originating from a node", "parameters": [ { "type": "string", - "description": "3-letter IATA code", - "name": "iata", + "description": "Node UUID", + "name": "nodeId", "in": "path", "required": true + }, + { + "type": "integer", + "description": "Observation ID of last item for pagination", + "name": "cursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" } ], "responses": { "200": { - "description": "GeoJSON Feature (Polygon or MultiPolygon geometry, with bbox)", + "description": "OK", "schema": { - "type": "object" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_PacketObservationSummary" } }, - "204": { - "description": "IATA exists but has no border configured" + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -326,50 +1327,62 @@ } } }, - "/messages": { + "/observers": { "get": { "produces": [ "application/json" ], "tags": [ - "Messages" + "Observers" ], - "summary": "List channel messages", + "summary": "List observers", "parameters": [ { - "type": "integer", - "description": "Filter by channel integer ID (mutually exclusive with channelHash)", - "name": "channelID", + "type": "string", + "description": "Filter by single IATA code (case-insensitive)", + "name": "iata", "in": "query" }, { "type": "string", - "description": "Filter by channel hash byte hex (mutually exclusive with channelID)", - "name": "channelHash", + "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", + "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Return messages after this epoch ms", - "name": "since", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YVR or YVR,YYJ", - "name": "iatas", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", "in": "query" }, { - "type": "integer", - "description": "Filter by region ID, expands to member IATAs", - "name": "regionId", + "type": "string", + "description": "Filter by observer type (e.g. meshcoretomqtt, meshcore-ha)", + "name": "type", "in": "query" }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", - "name": "region", + "description": "Filter by broker name", + "name": "broker", + "in": "query" + }, + { + "type": "string", + "description": "Filter by status (online or offline)", + "name": "status", + "in": "query" + }, + { + "type": "string", + "description": "Partial case-insensitive display name match", + "name": "name", "in": "query" }, { @@ -380,13 +1393,15 @@ }, { "type": "integer", - "description": "Message ID of last item for pagination (results ordered newest first)", + "description": "last_seen epoch ms of last item for pagination", "name": "cursor", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -395,7 +1410,7 @@ "200": { "description": "OK", "schema": { - "type": "object" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ObserverSummary" } }, "400": { @@ -413,62 +1428,29 @@ } } }, - "/messages/backfill": { + "/observers/{observerId}": { "get": { "produces": [ "application/json" ], "tags": [ - "Messages" + "Observers" ], - "summary": "Backfill messages after a given message ID", + "summary": "Get observer detail", "parameters": [ - { - "type": "integer", - "description": "Return messages after this ID (use last WS event message ID)", - "name": "afterId", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "Filter by IATA code(s), comma-separated", - "name": "iatas", - "in": "query" - }, - { - "type": "string", - "description": "Filter by region slug", - "name": "region", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by region ID", - "name": "regionId", - "in": "query" - }, { "type": "string", - "description": "Filter by transport scope name", - "name": "scope", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 100)", - "name": "limit", - "in": "query" + "description": "Observer UUID", + "name": "observerId", + "in": "path", + "required": true } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelMessage" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Observer" } }, "400": { @@ -477,6 +1459,12 @@ "$ref": "#/definitions/internal_api_handlers.APIError" } }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -486,104 +1474,39 @@ } } }, - "/nodes": { + "/observers/{observerId}/activity": { "get": { "produces": [ "application/json" ], "tags": [ - "Nodes" + "Observers" ], - "summary": "List nodes", + "summary": "Get observer heard-activity history", "parameters": [ - { - "type": "integer", - "description": "Node type integer (1=companion, 2=repeater, 3=room_server, 4=sensor)", - "name": "type", - "in": "query" - }, - { - "type": "string", - "description": "Node type name (companion, repeater, room_server, sensor)", - "name": "typeName", - "in": "query" - }, - { - "type": "string", - "description": "Filter by single IATA code (case-insensitive)", - "name": "iata", - "in": "query" - }, - { - "type": "string", - "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", - "name": "iatas", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by region ID, expands to member IATAs", - "name": "regionId", - "in": "query" - }, - { - "type": "string", - "description": "Filter by region slug, expands to member IATAs", - "name": "region", - "in": "query" - }, - { - "type": "string", - "description": "Partial case-insensitive name match", - "name": "name", - "in": "query" - }, { "type": "string", - "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", - "name": "scope", - "in": "query" + "description": "Observer UUID", + "name": "observerId", + "in": "path", + "required": true }, { "type": "string", - "description": "Exact public key match (hex)", - "name": "pubkey", + "description": "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h", + "name": "range", "in": "query" }, { "type": "string", - "description": "Partial public key match: hex prefix, case-insensitive", - "name": "pubkeyPrefix", - "in": "query" - }, - { - "type": "boolean", - "description": "Filter by multibyte path support (true/false); omit for no filter", - "name": "supportsMultibytePaths", - "in": "query" - }, - { - "type": "boolean", - "description": "Filter by multibyte trace support (true/false); omit for no filter", - "name": "supportsMultibyteTraces", - "in": "query" - }, - { - "type": "boolean", - "description": "Include each node's known neighbor IDs (neighborIds field). Bare ?neighbors or ?neighbors=true enables it; omit/false for none", - "name": "neighbors", - "in": "query" - }, - { - "type": "integer", - "description": "last_seen epoch ms of last item for pagination", - "name": "cursor", + "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", + "name": "interval", "in": "query" }, { "type": "integer", - "description": "Max results (default 50)", - "name": "limit", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until.", + "name": "until", "in": "query" } ], @@ -591,7 +1514,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity" } }, "400": { @@ -600,6 +1523,12 @@ "$ref": "#/definitions/internal_api_handlers.APIError" } }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -609,29 +1538,43 @@ } } }, - "/nodes/{nodeId}": { + "/observers/{observerId}/adverts": { "get": { "produces": [ "application/json" ], "tags": [ - "Nodes" + "Observers" ], - "summary": "Get node detail", + "summary": "List advert packets heard by an observer", "parameters": [ { "type": "string", - "description": "Node UUID", - "name": "nodeId", + "description": "Observer UUID", + "name": "observerId", "in": "path", "required": true + }, + { + "type": "integer", + "description": "Observation ID of last item for pagination", + "name": "cursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Node" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_AdvertObservation" } }, "400": { @@ -640,8 +1583,8 @@ "$ref": "#/definitions/internal_api_handlers.APIError" } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -649,32 +1592,47 @@ } } }, - "/nodes/{nodeId}/neighbors": { + "/observers/{observerId}/telemetry": { "get": { "produces": [ "application/json" ], "tags": [ - "Nodes" + "Observers" ], - "summary": "List neighbors for a node", + "summary": "Get observer telemetry history", "parameters": [ { "type": "string", - "description": "Node UUID", - "name": "nodeId", + "description": "Observer UUID", + "name": "observerId", "in": "path", "required": true + }, + { + "type": "string", + "description": "Duration window e.g. 24h, 48h, 168h (default 24h)", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Return points after this telemetry ID for WS reconnection backfill", + "name": "afterId", + "in": "query" + }, + { + "type": "string", + "description": "Bucketing interval: 1h (default), 6h, or 24h", + "name": "interval", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeNeighbor" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetry" } }, "400": { @@ -692,32 +1650,105 @@ } } }, - "/nodes/{nodeId}/observations": { + "/packets": { "get": { "produces": [ "application/json" ], "tags": [ - "Nodes" + "Packets" ], - "summary": "List packet observations originating from a node", + "summary": "List packets", "parameters": [ + { + "type": "integer", + "description": "Filter by payload type integer", + "name": "payloadType", + "in": "query" + }, { "type": "string", - "description": "Node UUID", - "name": "nodeId", - "in": "path", - "required": true + "description": "Filter by multiple payload types, comma-separated e.g. 2,4", + "name": "payloadTypes", + "in": "query" + }, + { + "type": "string", + "description": "Filter by payload type name (advert, grp_txt, txt_msg, trace, anon_req)", + "name": "payloadTypeName", + "in": "query" }, { "type": "integer", - "description": "Observation ID of last item for pagination", + "description": "Filter by route type (0=transport_flood, 1=flood, 2=direct, 3=transport_direct)", + "name": "routeType", + "in": "query" + }, + { + "type": "string", + "description": "Filter by multiple route types, comma-separated e.g. 0,1", + "name": "routeTypes", + "in": "query" + }, + { + "type": "string", + "description": "Filter by single IATA code (case-insensitive)", + "name": "iata", + "in": "query" + }, + { + "type": "string", + "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", + "name": "iatas", + "in": "query" + }, + { + "type": "string", + "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", + "name": "scope", + "in": "query" + }, + { + "type": "string", + "description": "Filter by multiple transport scope names, comma-separated e.g. %23bc,%23west", + "name": "scopes", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by first_heard_at \u003e= since (epoch ms)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by first_heard_at \u003c= until (epoch ms)", + "name": "until", + "in": "query" + }, + { + "type": "integer", + "description": "epoch ms of last item for pagination; last_heard_at, or site-local heard_at when iatas is set", "name": "cursor", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -726,7 +1757,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_PacketObservationSummary" + "type": "object" } }, "400": { @@ -744,79 +1775,70 @@ } } }, - "/observers": { + "/packets/backfill": { "get": { "produces": [ "application/json" ], "tags": [ - "Observers" + "Packets" ], - "summary": "List observers", + "summary": "Backfill packets after a given observation ID", "parameters": [ { - "type": "string", - "description": "Filter by single IATA code (case-insensitive)", - "name": "iata", - "in": "query" - }, - { - "type": "string", - "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", - "name": "iatas", - "in": "query" + "type": "integer", + "description": "Return packets with observations after this ID (use last WS event observation ID)", + "name": "afterObservationId", + "in": "query", + "required": true }, { "type": "integer", - "description": "Filter by region ID, expands to member IATAs", - "name": "regionId", + "description": "Filter by payload type integer", + "name": "payloadType", "in": "query" }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", - "name": "region", + "description": "Filter by payload type name", + "name": "payloadTypeName", "in": "query" }, { - "type": "string", - "description": "Filter by observer type (e.g. meshcoretomqtt, meshcore-ha)", - "name": "type", + "type": "integer", + "description": "Filter by route type", + "name": "routeType", "in": "query" }, { "type": "string", - "description": "Filter by broker name", - "name": "broker", + "description": "Filter by IATA code(s), comma-separated", + "name": "iatas", "in": "query" }, { "type": "string", - "description": "Filter by status (online or offline)", - "name": "status", + "description": "Filter by region slug", + "name": "region", "in": "query" }, { - "type": "string", - "description": "Partial case-insensitive display name match", - "name": "name", + "type": "integer", + "description": "Filter by region ID", + "name": "regionId", "in": "query" }, { "type": "string", - "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", + "description": "Filter by transport scope name", "name": "scope", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "last_seen epoch ms of last item for pagination", - "name": "cursor", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 100); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -825,7 +1847,10 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ObserverSummary" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketSummary" + } } }, "400": { @@ -843,20 +1868,20 @@ } } }, - "/observers/{observerId}": { + "/packets/{packetHash}": { "get": { "produces": [ "application/json" ], "tags": [ - "Observers" + "Packets" ], - "summary": "Get observer detail", + "summary": "Get full packet detail. For trace packets (payloadType=9), includes resolvedRoute", "parameters": [ { "type": "string", - "description": "Observer UUID", - "name": "observerId", + "description": "Packet hash (hex)", + "name": "packetHash", "in": "path", "required": true } @@ -865,17 +1890,51 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Observer" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Packet" } }, "400": { "description": "Bad Request", "schema": { - "$ref": "#/definitions/internal_api_handlers.APIError" + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "/regions": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Regions" + ], + "summary": "List all regions", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RegionSummary" + } } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -883,41 +1942,29 @@ } } }, - "/observers/{observerId}/adverts": { + "/regions/{regionId}": { "get": { "produces": [ "application/json" ], "tags": [ - "Observers" + "Regions" ], - "summary": "List advert packets heard by an observer", + "summary": "Get a single region", "parameters": [ { - "type": "string", - "description": "Observer UUID", - "name": "observerId", + "type": "integer", + "description": "Region ID", + "name": "regionId", "in": "path", "required": true - }, - { - "type": "integer", - "description": "Observation ID of last item for pagination", - "name": "cursor", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 50)", - "name": "limit", - "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_AdvertObservation" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Region" } }, "400": { @@ -926,6 +1973,12 @@ "$ref": "#/definitions/internal_api_handlers.APIError" } }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -935,39 +1988,46 @@ } } }, - "/observers/{observerId}/telemetry": { + "/routes": { "get": { "produces": [ "application/json" ], "tags": [ - "Observers" + "Routes" ], - "summary": "Get observer telemetry history", + "summary": "List known routes", "parameters": [ { "type": "string", - "description": "Observer UUID", - "name": "observerId", - "in": "path", - "required": true + "description": "Filter by IATA code", + "name": "iata", + "in": "query" }, { - "type": "string", - "description": "Duration window e.g. 24h, 48h, 168h (default 24h)", - "name": "range", + "type": "integer", + "description": "Filter by exact hop count", + "name": "hopCount", "in": "query" }, { "type": "integer", - "description": "Return points after this telemetry ID for WS reconnection backfill", - "name": "afterId", + "description": "Epoch ms timestamp of last item for pagination", + "name": "cursor", "in": "query" }, { - "type": "string", - "description": "Bucketing interval: 1h (default), 6h, or 24h", - "name": "interval", + "type": "integer", + "description": "id of the last item; with cursor, also returns later routes sharing that millisecond", + "name": "cursorId", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 50); must be positive, values above 200 are clamped", + "name": "limit", "in": "query" } ], @@ -975,7 +2035,10 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetry" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + } } }, "400": { @@ -993,112 +2056,53 @@ } } }, - "/packets": { + "/routes/cross": { "get": { "produces": [ "application/json" ], "tags": [ - "Packets" + "Routes" ], - "summary": "List packets", + "summary": "Search for routes that cross IATA boundaries", "parameters": [ - { - "type": "integer", - "description": "Filter by payload type integer", - "name": "payloadType", - "in": "query" - }, - { - "type": "string", - "description": "Filter by multiple payload types, comma-separated e.g. 2,4", - "name": "payloadTypes", - "in": "query" - }, - { - "type": "string", - "description": "Filter by payload type name (advert, grp_txt, txt_msg, trace, anon_req)", - "name": "payloadTypeName", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by route type (0=transport_flood, 1=flood, 2=direct, 3=transport_direct)", - "name": "routeType", - "in": "query" - }, { "type": "string", - "description": "Filter by multiple route types, comma-separated e.g. 0,1", - "name": "routeTypes", - "in": "query" - }, - { - "type": "string", - "description": "Filter by single IATA code (case-insensitive)", - "name": "iata", - "in": "query" - }, - { - "type": "string", - "description": "Filter by multiple IATA codes, comma-separated e.g. YVR,YYJ", - "name": "iatas", - "in": "query" + "description": "Source node hash prefix (hex)", + "name": "fromHash", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by transport scope name e.g. %23bc (URL-encoded #bc)", - "name": "scope", - "in": "query" + "description": "Source IATA code", + "name": "fromIata", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by multiple transport scope names, comma-separated e.g. %23bc,%23west", - "name": "scopes", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by region ID, expands to member IATAs", - "name": "regionId", - "in": "query" + "description": "Destination node hash prefix (hex)", + "name": "toHash", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", - "name": "region", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by first_heard_at \u003e= since (epoch ms)", - "name": "since", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by first_heard_at \u003c= until (epoch ms)", - "name": "until", - "in": "query" - }, - { - "type": "integer", - "description": "epoch ms of last item for pagination; last_heard_at, or site-local heard_at when iatas is set", - "name": "cursor", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 50)", - "name": "limit", - "in": "query" + "description": "Destination IATA code", + "name": "toIata", + "in": "query", + "required": true } ], "responses": { "200": { "description": "OK", "schema": { - "type": "object" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATARoute" + } } }, "400": { @@ -1116,70 +2120,36 @@ } } }, - "/packets/backfill": { + "/routes/search": { "get": { "produces": [ "application/json" ], "tags": [ - "Packets" + "Routes" ], - "summary": "Backfill packets after a given observation ID", + "summary": "Search known routes by source and destination hash", "parameters": [ { - "type": "integer", - "description": "Return packets with observations after this ID (use last WS event observation ID)", - "name": "afterObservationId", + "type": "string", + "description": "IATA code to search within", + "name": "iata", "in": "query", "required": true }, - { - "type": "integer", - "description": "Filter by payload type integer", - "name": "payloadType", - "in": "query" - }, - { - "type": "string", - "description": "Filter by payload type name", - "name": "payloadTypeName", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by route type", - "name": "routeType", - "in": "query" - }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated", - "name": "iatas", - "in": "query" - }, - { - "type": "string", - "description": "Filter by region slug", - "name": "region", - "in": "query" - }, - { - "type": "integer", - "description": "Filter by region ID", - "name": "regionId", - "in": "query" + "description": "Source node hash prefix (hex)", + "name": "from", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by transport scope name", - "name": "scope", - "in": "query" - }, - { - "type": "integer", - "description": "Max results (default 100)", - "name": "limit", - "in": "query" + "description": "Destination node hash prefix (hex)", + "name": "to", + "in": "query", + "required": true } ], "responses": { @@ -1188,7 +2158,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" } } }, @@ -1207,29 +2177,69 @@ } } }, - "/packets/{packetHash}": { + "/routes/{iata}/{pathKey}/observations": { "get": { + "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", "produces": [ "application/json" ], "tags": [ - "Packets" + "Routes" ], - "summary": "Get full packet detail. For trace packets (payloadType=9), includes resolvedRoute", + "summary": "Get retained reports matching a full saved route prefix sequence", "parameters": [ { "type": "string", - "description": "Packet hash (hex)", - "name": "packetHash", + "description": "Three-character IATA code", + "name": "iata", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Stable 32-hex pathKey from a known-route response", + "name": "pathKey", "in": "path", "required": true + }, + { + "type": "string", + "description": "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Window start epoch ms; provide with until (default last 24h)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Exclusive window end epoch ms; maximum span 30d, end may be up to 5 minutes ahead of server time (clock skew tolerance)", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Default 50; positive, capped at 200", + "name": "limit", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Packet" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence" } }, "400": { @@ -1253,27 +2263,54 @@ } } }, - "/regions": { + "/scopes": { "get": { + "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", "produces": [ "application/json" ], "tags": [ - "Regions" + "Scopes" + ], + "summary": "List transport scopes", + "parameters": [ + { + "type": "string", + "description": "Filter by IATA code(s), comma-separated", + "name": "iatas", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID", + "name": "regionId", + "in": "query" + } ], - "summary": "List all regions", "responses": { "200": { "description": "OK", "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RegionSummary" + "type": "string" } } }, - "404": { - "description": "Not Found", + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -1281,20 +2318,21 @@ } } }, - "/regions/{regionId}": { + "/scopes/{name}": { "get": { + "description": "packetCount sums packets per hour across every retained rollup hour; iatas are the IATAs that heard the scope's packets. Observer and node counts are current memberships.", "produces": [ "application/json" ], "tags": [ - "Regions" + "Scopes" ], - "summary": "Get a single region", + "summary": "Get scope detail by name", "parameters": [ { - "type": "integer", - "description": "Region ID", - "name": "regionId", + "type": "string", + "description": "Scope name e.g. %23bc (URL-encoded #bc)", + "name": "name", "in": "path", "required": true } @@ -1303,17 +2341,17 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Region" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail" } }, - "400": { - "description": "Bad Request", + "404": { + "description": "Not Found", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -1321,37 +2359,39 @@ } } }, - "/routes": { + "/stats/clock-drift": { "get": { "produces": [ "application/json" ], "tags": [ - "Routes" + "Stats" ], - "summary": "List known routes", + "summary": "Repeaters/room servers whose clock has drifted beyond the configured threshold, worst first", "parameters": [ { "type": "string", - "description": "Filter by IATA code", - "name": "iata", + "description": "Comma-separated IATA codes", + "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Filter by exact hop count", - "name": "hopCount", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", "in": "query" }, { - "type": "integer", - "description": "Epoch ms timestamp of last item for pagination", - "name": "cursor", + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1362,7 +2402,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry" } } }, @@ -1375,43 +2415,33 @@ } } }, - "/routes/cross": { + "/stats/node-types": { "get": { "produces": [ "application/json" ], "tags": [ - "Routes" + "Stats" ], - "summary": "Search for routes that cross IATA boundaries", + "summary": "Node type breakdown", "parameters": [ { "type": "string", - "description": "Source node hash prefix (hex)", - "name": "fromHash", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "Source IATA code", - "name": "fromIata", - "in": "query", - "required": true + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" }, { - "type": "string", - "description": "Destination node hash prefix (hex)", - "name": "toHash", - "in": "query", - "required": true + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" }, { "type": "string", - "description": "Destination IATA code", - "name": "toIata", - "in": "query", - "required": true + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" } ], "responses": { @@ -1420,16 +2450,10 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATARoute" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeTypeCount" } } }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/internal_api_handlers.APIError" - } - }, "500": { "description": "Internal Server Error", "schema": { @@ -1439,36 +2463,40 @@ } } }, - "/routes/search": { + "/stats/observations": { "get": { + "description": "Observation counts only; distinct packet and observer counts don't sum across IATAs, so read them from /stats/series.", "produces": [ "application/json" ], "tags": [ - "Routes" + "Stats" ], - "summary": "Search known routes by source and destination hash", + "summary": "Hourly observation counts per IATA", "parameters": [ { "type": "string", - "description": "IATA code to search within", - "name": "iata", - "in": "query", - "required": true + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" }, { - "type": "string", - "description": "Source node hash prefix (hex)", - "name": "from", - "in": "query", - "required": true + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" }, { "type": "string", - "description": "Destination node hash prefix (hex)", - "name": "to", - "in": "query", - "required": true + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Start of window epoch ms (default 7 days ago)", + "name": "since", + "in": "query" } ], "responses": { @@ -1477,16 +2505,10 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint" } } }, - "400": { - "description": "Bad Request", - "schema": { - "$ref": "#/definitions/internal_api_handlers.APIError" - } - }, "500": { "description": "Internal Server Error", "schema": { @@ -1496,40 +2518,81 @@ } } }, - "/scopes": { + "/stats/observer-comparison": { "get": { + "description": "Counts distinct retained packet hashes with route type 0 or 1, using reception time in [since, until). onlyA, onlyB and both partition totalPackets (the union). Repeated receptions count once. These are reported receptions, not a measure of radio packet loss or continuous observer coverage.", "produces": [ "application/json" ], "tags": [ - "Scopes" + "Stats" ], - "summary": "List transport scopes", + "summary": "Compare flood packets reported by two observers", "parameters": [ { "type": "string", - "description": "Filter by IATA code(s), comma-separated", - "name": "iatas", - "in": "query" + "description": "First observer UUID", + "name": "observerA", + "in": "query", + "required": true }, { "type": "string", - "description": "Filter by region slug", - "name": "region", + "description": "Second, distinct observer UUID", + "name": "observerB", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; later than since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Filter by region ID", + "description": "Region ID, expands to member IATAs", "name": "regionId", "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "type": "object" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -1537,39 +2600,51 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, - "/scopes/{name}": { + "/stats/overview": { "get": { + "description": "Summarises the 24 most recent hours that can have been rolled (an hour rolls about 95 minutes after it closes); since/until report that window. Packets count once per hour heard. Use /stats/series for sparklines.", "produces": [ "application/json" ], "tags": [ - "Scopes" + "Stats" ], - "summary": "Get scope detail by name", + "summary": "Network overview stats (last 24 rolled hours)", "parameters": [ { "type": "string", - "description": "Scope name e.g. %23bc (URL-encoded #bc)", - "name": "name", - "in": "path", - "required": true + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail" - } - }, - "404": { - "description": "Not Found", - "schema": { - "$ref": "#/definitions/internal_api_handlers.APIError" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview" } }, "500": { @@ -1581,31 +2656,46 @@ } } }, - "/stats/node-types": { + "/stats/paths": { "get": { + "description": "Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly path rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Node type breakdown", + "summary": "Received path-entry and hash-width distributions", "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, { "type": "string", - "description": "Comma-separated IATA codes", + "description": "Comma-separated reception IATA codes", "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Filter by region ID, expands to member IATAs", + "description": "Region ID, expands to member IATAs", "name": "regionId", "in": "query" }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", + "description": "Region slug, expands to member IATAs", "name": "region", "in": "query" } @@ -1614,10 +2704,13 @@ "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeTypeCount" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -1625,11 +2718,17 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, - "/stats/observations": { + "/stats/payload-breakdown": { "get": { "produces": [ "application/json" @@ -1637,7 +2736,7 @@ "tags": [ "Stats" ], - "summary": "Hourly observation time series", + "summary": "Observation counts by payload type (last 24h by default)", "parameters": [ { "type": "string", @@ -1659,7 +2758,7 @@ }, { "type": "integer", - "description": "Start of window epoch ms (default 7 days ago)", + "description": "Start of window epoch ms (default last 24h)", "name": "since", "in": "query" } @@ -1670,7 +2769,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem" } } }, @@ -1683,7 +2782,7 @@ } } }, - "/stats/overview": { + "/stats/radio-presets": { "get": { "produces": [ "application/json" @@ -1691,8 +2790,14 @@ "tags": [ "Stats" ], - "summary": "Network overview stats (last 24h)", + "summary": "Radio preset usage by IATA", "parameters": [ + { + "type": "string", + "description": "Filter by preset string e.g. 910.525,62.5,7", + "name": "preset", + "in": "query" + }, { "type": "string", "description": "Comma-separated IATA codes", @@ -1716,7 +2821,10 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RadioPreset" + } } }, "500": { @@ -1728,15 +2836,16 @@ } } }, - "/stats/payload-breakdown": { + "/stats/scopes": { "get": { + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour and adds the distinct observers and advertising nodes active in the scope each hour (empty hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Observation counts by payload type (last 24h by default)", + "summary": "Scope statistics", "parameters": [ { "type": "string", @@ -1744,6 +2853,12 @@ "name": "iatas", "in": "query" }, + { + "type": "string", + "description": "Single IATA code; used when iatas is absent", + "name": "iata", + "in": "query" + }, { "type": "integer", "description": "Filter by region ID, expands to member IATAs", @@ -1752,13 +2867,13 @@ }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", + "description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs", "name": "region", "in": "query" }, { "type": "integer", - "description": "Start of window epoch ms (default last 24h)", + "description": "Start of packet window epoch ms, rounded down to the hour (default 7 days ago)", "name": "since", "in": "query" } @@ -1769,10 +2884,16 @@ "schema": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats" } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -1782,37 +2903,46 @@ } } }, - "/stats/radio-presets": { + "/stats/series": { "get": { + "description": "One entry per UTC hour in [since, until) plus a summary over the complete hours. Both bounds round down to UTC hours and since is clamped to the rollup retention; the response reports the effective window. An hour is rolled about 95 minutes after it closes, so recent hours are \"missing\" until then; \"partial\" hours lost raw rows before they could be rolled and never get values. Counts (observations, packets) sum across hours, with a packet counted once in each hour it was heard; observers, IATAs and scopes are distinct across the whole window; averages are sum / samples. revision changes whenever any rolled hour changes.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Radio preset usage by IATA", + "summary": "Hourly network activity from the analytics rollups", "parameters": [ { - "type": "string", - "description": "Filter by preset string e.g. 910.525,62.5,7", - "name": "preset", - "in": "query" + "type": "integer", + "description": "Inclusive start, epoch milliseconds", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; the window is at most the rollup retention (default 90 days)", + "name": "until", + "in": "query", + "required": true }, { "type": "string", - "description": "Comma-separated IATA codes", + "description": "Comma-separated reception IATA codes", "name": "iatas", "in": "query" }, { "type": "integer", - "description": "Filter by region ID, expands to member IATAs", + "description": "Region ID, expands to member IATAs", "name": "regionId", "in": "query" }, { "type": "string", - "description": "Filter by region slug, expands to member IATAs", + "description": "Region slug, expands to member IATAs", "name": "region", "in": "query" } @@ -1821,10 +2951,13 @@ "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RadioPreset" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -1832,27 +2965,71 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, - "/stats/scopes": { + "/stats/signal": { "get": { + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly signal rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Scope statistics", + "summary": "Reception signal distributions and hourly trends", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats" - } + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -1860,12 +3037,19 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } }, "/stats/top-advertisers": { "get": { + "description": "Each advert counts once per hour heard, however many requested IATAs heard it. nodeId is null when the node row has been deleted.", "produces": [ "application/json" ], @@ -1899,8 +3083,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1926,13 +3112,14 @@ }, "/stats/top-nodes": { "get": { + "description": "nodeId is null when the node row has been deleted; publicKey always identifies it.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Top N nodes by observation count (from materialized view)", + "summary": "Top N nodes by advert hearings (last 7 days by default)", "parameters": [ { "type": "string", @@ -1954,7 +3141,15 @@ }, { "type": "integer", - "description": "Max results (default 10)", + "description": "Start of window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2013,8 +3208,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2073,8 +3270,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2128,13 +3327,13 @@ }, { "type": "string", - "description": "Filter by transport scope name", + "description": "Filter by the tag's first transport scope name", "name": "scope", "in": "query" }, { "type": "string", - "description": "Filter by type: TRACE or PING (default: all)", + "description": "Filter by type: TRACE (any multi-hop packet) or PING (default: all)", "name": "type", "in": "query" }, @@ -2152,13 +3351,21 @@ }, { "type": "integer", - "description": "last_heard_at epoch ms of last item for pagination", + "description": "lastHeardAt (epoch ms) of the last item; returns older tags", "name": "cursor", "in": "query" }, { + "type": "string", + "description": "traceTag (hex) of the last item; with cursor, also returns later tags sharing that millisecond", + "name": "cursorTag", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2173,6 +3380,12 @@ } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -2224,6 +3437,97 @@ } }, "definitions": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Account": { + "type": "object", + "properties": { + "active": { + "type": "boolean" + }, + "created_at": { + "type": "string" + }, + "deactivated_at": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig": { + "type": "object", + "properties": { + "configured": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig": { + "type": "object", + "properties": { + "allow_credentials": { + "type": "boolean" + }, + "allowed_headers": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowed_methods": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowed_origins": { + "type": "array", + "items": { + "type": "string" + } + }, + "max_age": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig": { + "type": "object", + "properties": { + "auth": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig" + }, + "cors": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig" + }, + "ingest": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig": { + "type": "object", + "properties": { + "broker_count": { + "description": "BrokerCount counts configured broker workers, not active MQTT connections\nor a configurable processing-worker pool.", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.AdvertObservation": { "type": "object", "properties": { @@ -2297,6 +3601,7 @@ "type": "integer" }, "messageCount": { + "description": "lifetime count; not reduced by retention", "type": "integer" }, "name": { @@ -2327,6 +3632,19 @@ "description": "hex-encoded packet hash for correlation with packet events", "type": "string" }, + "scope": { + "description": "matched scope on the first stored packet; null when none was recorded", + "type": "string" + }, + "scopeStatus": { + "type": "string", + "enum": [ + "matched", + "unscoped", + "unknown", + "unavailable" + ] + }, "senderName": { "description": "display name from the decrypted payload", "type": "string" @@ -2337,6 +3655,26 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage": { + "type": "object", + "properties": { + "hasMore": { + "type": "boolean" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary" + } + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary": { "type": "object", "properties": { @@ -2365,6 +3703,49 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry": { + "type": "object", + "properties": { + "clockCheckedAt": { + "description": "epoch ms", + "type": "integer" + }, + "clockDriftSeconds": { + "description": "signed; +ve = device ahead of server", + "type": "integer" + }, + "iatas": { + "description": "IATAs this node has been heard in", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeIATA" + } + }, + "nodeId": { + "type": "string" + }, + "nodeName": { + "type": "string" + }, + "nodeType": { + "type": "integer" + }, + "nodeTypeName": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop": { "type": "object", "properties": { @@ -2471,6 +3852,10 @@ }, "observationCount": { "type": "integer" + }, + "pathKey": { + "description": "stable identity within this IATA; use for route evidence links", + "type": "string" } } }, @@ -2566,6 +3951,10 @@ "description": "UUID of the associated observer row, if any", "type": "string" }, + "possiblyForeign": { + "description": "PossiblyForeign compares a repeater's stored position with this server's\nconfigured IATA border union. Omitted when disabled, unknown, or not repeater.", + "type": "boolean" + }, "publicKey": { "description": "hex-encoded Ed25519 public key", "type": "string" @@ -2696,6 +4085,10 @@ "description": "UUID of the associated observer row, if any", "type": "string" }, + "possiblyForeign": { + "description": "PossiblyForeign compares a repeater's stored position with this server's\nconfigured IATA border union. Omitted when disabled, unknown, or not repeater.", + "type": "boolean" + }, "publicKey": { "description": "hex-encoded Ed25519 public key", "type": "string" @@ -2727,9 +4120,6 @@ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint": { "type": "object", "properties": { - "activeObservers": { - "type": "integer" - }, "hour": { "description": "epoch ms, start of the 1-hour bucket", "type": "integer" @@ -2739,9 +4129,6 @@ }, "observationCount": { "type": "integer" - }, - "uniquePackets": { - "type": "integer" } } }, @@ -2792,6 +4179,7 @@ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -2837,13 +4225,132 @@ "type": "string" }, "statusMetadata": { - "description": "raw /status JSON payload" + "description": "raw /status JSON payload", + "type": "object" }, "uptimeSeconds": { "type": "integer" } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { + "type": "object", + "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, + "interval": { + "type": "string" + }, + "payloadTypes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem" + } + }, + "points": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint" + } + }, + "radio": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio" + }, + "range": { + "type": "string" + }, + "rawFrom": { + "description": "start of the raw tail, epoch ms", + "type": "integer" + }, + "rolledUntil": { + "description": "Hourly only. Buckets before rolledUntil come from rollups and from rawFrom on from raw rows;\nwhen rawFrom is later, the hours between are uncovered, not quiet.", + "type": "integer" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive end, epoch ms; live requests include the current partial bucket", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint": { + "type": "object", + "properties": { + "airtimeMs": { + "type": "number" + }, + "observations": { + "type": "integer" + }, + "rssiAvg": { + "type": "number" + }, + "snrAvg": { + "type": "number" + }, + "snrMin": { + "type": "number" + }, + "t": { + "description": "bucket start, epoch ms", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio": { + "type": "object", + "properties": { + "bwKhz": { + "type": "number" + }, + "cr": { + "type": "integer" + }, + "freqMhz": { + "type": "number" + }, + "preambleSymbols": { + "type": "integer" + }, + "sf": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "description": "stored observations within windowStart/windowEnd", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { @@ -2861,6 +4368,37 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison": { + "type": "object", + "properties": { + "both": { + "type": "integer" + }, + "observerA": { + "type": "string" + }, + "observerB": { + "type": "string" + }, + "onlyA": { + "type": "integer" + }, + "onlyB": { + "type": "integer" + }, + "since": { + "description": "inclusive, epoch milliseconds", + "type": "integer" + }, + "totalPackets": { + "type": "integer" + }, + "until": { + "description": "exclusive, epoch milliseconds", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary": { "type": "object", "properties": { @@ -2916,10 +4454,10 @@ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetryPoint": { "type": "object", "properties": { - "airtimeRxPct": { + "airtimeRxSecs": { "type": "number" }, - "airtimeTxPct": { + "airtimeTxSecs": { "type": "number" }, "batteryMv": { @@ -3051,6 +4589,30 @@ }, "id": { "type": "string" + }, + "pathBytes": { + "description": "hex-encoded accumulated path hashes", + "type": "string" + }, + "pathLength": { + "description": "PathLength/PathBytes and captured endpoint resolutions are stored on the observation,\nso list/backfill reads need no per-hash resolution queries. Legacy observations have\nno endpoint snapshot. ResolvedPath remains a detail/opted-in WS feature.", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketPathLength" + } + ] + }, + "resolvedDestination": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" + }, + "resolvedPath": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" + } + }, + "resolvedSource": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" } } }, @@ -3098,7 +4660,7 @@ } }, "resolvedSource": { - "description": "ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type\ncarries a resolvable one: an exact match for ADVERT's full pubkey, an ambiguous\nhash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte\nsource/destination hashes. Nil when the payload type doesn't carry one at all (e.g.\nGRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and\nResolveExactNode for how each is built.", + "description": "ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type\ncarries one. Prefer the snapshot captured at ingest; legacy observations without a\nsnapshot use the current node registry. Endpoint matching itself is unchanged:\nan exact match for ADVERT's full pubkey, an ambiguous\nhash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte\nsource/destination hashes. Nil when the payload type doesn't carry one at all (e.g.\nGRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and\nResolveExactNode for how each is built.", "allOf": [ { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" @@ -3223,7 +4785,7 @@ "type": "string" }, "summary": { - "description": "human-readable payload summary", + "description": "advert name, or the ACK/TRACE/PING summary; omitted when unavailable", "type": "string" } } @@ -3256,7 +4818,7 @@ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary": { "type": "object", "properties": { "hasMore": { @@ -3265,7 +4827,7 @@ "items": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeSummary" } }, "nextCursor": { @@ -3273,7 +4835,7 @@ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ObserverSummary": { "type": "object", "properties": { "hasMore": { @@ -3282,7 +4844,7 @@ "items": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary" } }, "nextCursor": { @@ -3290,7 +4852,7 @@ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ObserverSummary": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_PacketObservationSummary": { "type": "object", "properties": { "hasMore": { @@ -3299,7 +4861,7 @@ "items": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketObservationSummary" } }, "nextCursor": { @@ -3307,19 +4869,101 @@ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_PacketObservationSummary": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth": { "type": "object", "properties": { - "hasMore": { - "type": "boolean" + "bytes": { + "type": "integer" + }, + "receptions": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour": { + "type": "object", + "properties": { + "empty": { + "type": "integer" + }, + "hour": { + "type": "integer" + }, + "maxEntries": { + "description": "longest path received that hour across the requested IATAs; 0 if only empty paths", + "type": "integer" + }, + "oneByte": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "threeByte": { + "type": "integer" + }, + "trace": { + "type": "integer" + }, + "twoByte": { + "type": "integer" + }, + "unclassified": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin": { + "type": "object", + "properties": { + "entries": { + "type": "integer" + }, + "receptions": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats": { + "type": "object", + "properties": { + "empty": { + "type": "integer" + }, + "hashWidths": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth" + } + }, + "hashed": { + "type": "integer" + }, + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour" + } }, - "items": { + "pathLengths": { "type": "array", "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketObservationSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin" } }, - "nextCursor": { + "receptions": { + "type": "integer" + }, + "since": { + "type": "integer" + }, + "trace": { + "type": "integer" + }, + "unclassified": { + "type": "integer" + }, + "until": { "type": "integer" } } @@ -3463,6 +5107,47 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence": { + "type": "object", + "properties": { + "generatedAt": { + "type": "integer" + }, + "hasMore": { + "type": "boolean" + }, + "hashSize": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation" + } + }, + "matchAvailable": { + "type": "boolean" + }, + "matchType": { + "type": "string" + }, + "nextPageCursor": { + "type": "string" + }, + "pathBytes": { + "type": "string" + }, + "route": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + }, + "windowEnd": { + "type": "integer" + }, + "windowStart": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop": { "type": "object", "properties": { @@ -3483,6 +5168,38 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation": { + "type": "object", + "properties": { + "heardAt": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "observerId": { + "type": "string" + }, + "observerName": { + "type": "string" + }, + "packetHash": { + "type": "string" + }, + "payloadType": { + "type": "integer" + }, + "payloadTypeName": { + "type": "string" + }, + "rssi": { + "type": "integer" + }, + "snr": { + "type": "number" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail": { "type": "object", "properties": { @@ -3509,9 +5226,36 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the UTC hour", + "type": "integer" + }, + "nodes": { + "description": "distinct nodes whose adverts were heard in the scope that hour", + "type": "integer" + }, + "observers": { + "description": "distinct observers that heard the scope's packets that hour", + "type": "integer" + }, + "packets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats": { "type": "object", "properties": { + "hourly": { + "description": "packetCount by hour, oldest first; hours with none omitted", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour" + } + }, "name": { "description": "normalized scope name e.g. \"#bc\"", "type": "string" @@ -3530,6 +5274,86 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "lower": { + "type": "number" + }, + "upper": { + "type": "number" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "rssiAverage": { + "type": "number" + }, + "rssiSamples": { + "type": "integer" + }, + "snrAverage": { + "type": "number" + }, + "snrSamples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric": { + "type": "object", + "properties": { + "average": { + "type": "number" + }, + "histogram": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin" + } + }, + "samples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats": { + "type": "object", + "properties": { + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour" + } + }, + "receptions": { + "type": "integer" + }, + "rssi": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "since": { + "type": "integer" + }, + "snr": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview": { "type": "object", "properties": { @@ -3539,14 +5363,126 @@ "activeObservers": { "type": "integer" }, + "since": { + "description": "epoch ms, start of the window", + "type": "integer" + }, "totalObservations": { "type": "integer" }, "totalPackets": { + "description": "distinct per hour (see StatsSeriesValues)", + "type": "integer" + }, + "until": { + "description": "epoch ms, exclusive end of the window", "type": "integer" }, "windowHours": { - "description": "always 24 for now", + "description": "complete rolled hours the totals cover, at most 24", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries": { + "type": "object", + "properties": { + "completeHours": { + "description": "hours contributing to summary", + "type": "integer" + }, + "earliestComplete": { + "description": "first complete hour held, epoch ms; null if none", + "type": "integer" + }, + "hours": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour" + } + }, + "revision": { + "description": "changes whenever rolled hours, their status or coverage change", + "type": "integer" + }, + "since": { + "description": "effective start, epoch ms on a UTC hour", + "type": "integer" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + }, + "until": { + "description": "effective exclusive end, epoch ms on a UTC hour", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the hour", + "type": "integer" + }, + "status": { + "description": "partial: raw rows deleted before it was rolled", + "type": "string", + "enum": [ + "complete", + "partial", + "missing" + ] + }, + "values": { + "description": "null unless complete", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + } + ] + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues": { + "type": "object", + "properties": { + "activeIatas": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeObservers": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeScopes": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "maxPathEntries": { + "type": "integer" + }, + "observations": { + "type": "integer" + }, + "rssiSamples": { + "type": "integer" + }, + "rssiSum": { + "description": "average = rssiSum / rssiSamples", + "type": "number" + }, + "scopedPackets": { + "type": "integer" + }, + "snrSamples": { + "type": "integer" + }, + "snrSum": { + "description": "average = snrSum / snrSamples", + "type": "number" + }, + "uniquePackets": { "type": "integer" } } @@ -3557,6 +5493,13 @@ "advertCount": { "type": "integer" }, + "directAdvertCount": { + "type": "integer" + }, + "floodAdvertCount": { + "description": "FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed:\nflood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path;\ndirect = route type 2 (direct) or 3 (transport_direct), routed along a known path.\nFloodAdvertCount + DirectAdvertCount == AdvertCount.", + "type": "integer" + }, "iata": { "type": "string" }, @@ -3565,6 +5508,7 @@ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -3575,6 +5519,10 @@ }, "nodeTypeName": { "type": "string" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, @@ -3589,6 +5537,7 @@ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -3602,6 +5551,10 @@ }, "observationCount": { "type": "integer" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, @@ -3742,6 +5695,45 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest": { + "type": "object", + "required": [ + "allowed_origins" + ], + "properties": { + "allowed_origins": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest": { + "type": "object", + "required": [ + "cors" + ], + "properties": { + "cors": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse": { + "type": "object", + "properties": { + "config": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig" + }, + "persisted": { + "type": "boolean" + }, + "requires_restart": { + "type": "boolean" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { @@ -3767,6 +5759,14 @@ } } }, + "securityDefinitions": { + "AdminKey": { + "description": "Enter Bearer followed by the configured operator key. Use HTTPS.", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + }, "tags": [ { "description": "Airport/location codes that group observers and packets", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index d0914d78..3e872fd8 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,5 +1,66 @@ basePath: /api/v1 definitions: + github_com_MeshCore-Beacon_beacon-server_internal_api.Account: + properties: + active: + type: boolean + created_at: + type: string + deactivated_at: + type: string + id: + type: string + name: + type: string + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList: + properties: + items: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account' + type: array + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig: + properties: + configured: + type: boolean + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig: + properties: + allow_credentials: + type: boolean + allowed_headers: + items: + type: string + type: array + allowed_methods: + items: + type: string + type: array + allowed_origins: + items: + type: string + type: array + max_age: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig: + properties: + auth: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig' + cors: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig' + ingest: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig' + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig: + properties: + broker_count: + description: |- + BrokerCount counts configured broker workers, not active MQTT connections + or a configurable processing-worker pool. + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.AdvertObservation: properties: heardAt: @@ -52,6 +113,7 @@ definitions: description: epoch ms, time of most recent message type: integer messageCount: + description: lifetime count; not reduced by retention type: integer name: description: display name from config or nil @@ -74,6 +136,17 @@ definitions: packetHash: description: hex-encoded packet hash for correlation with packet events type: string + scope: + description: matched scope on the first stored packet; null when none was + recorded + type: string + scopeStatus: + enum: + - matched + - unscoped + - unknown + - unavailable + type: string senderName: description: display name from the decrypted payload type: string @@ -81,6 +154,19 @@ definitions: description: epoch ms, from the sender's embedded timestamp type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage: + properties: + hasMore: + type: boolean + items: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary' + type: array + nextCursor: + type: integer + nextPageCursor: + type: string + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary: properties: channelHash: @@ -101,6 +187,35 @@ definitions: description: display name from config or nil type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry: + properties: + clockCheckedAt: + description: epoch ms + type: integer + clockDriftSeconds: + description: signed; +ve = device ahead of server + type: integer + iatas: + description: IATAs this node has been heard in + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeIATA' + type: array + nodeId: + type: string + nodeName: + type: string + nodeType: + type: integer + nodeTypeName: + type: string + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest: + properties: + name: + type: string + required: + - name + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop: properties: fromIata: @@ -169,6 +284,9 @@ definitions: type: integer observationCount: type: integer + pathKey: + description: stable identity within this IATA; use for route evidence links + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.Node: properties: @@ -242,6 +360,11 @@ definitions: observerId: description: UUID of the associated observer row, if any type: string + possiblyForeign: + description: |- + PossiblyForeign compares a repeater's stored position with this server's + configured IATA border union. Omitted when disabled, unknown, or not repeater. + type: boolean publicKey: description: hex-encoded Ed25519 public key type: string @@ -337,6 +460,11 @@ definitions: observerId: description: UUID of the associated observer row, if any type: string + possiblyForeign: + description: |- + PossiblyForeign compares a repeater's stored position with this server's + configured IATA border union. Omitted when disabled, unknown, or not repeater. + type: boolean publicKey: description: hex-encoded Ed25519 public key type: string @@ -362,8 +490,6 @@ definitions: type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint: properties: - activeObservers: - type: integer hour: description: epoch ms, start of the 1-hour bucket type: integer @@ -371,8 +497,6 @@ definitions: type: string observationCount: type: integer - uniquePackets: - type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.Observer: properties: @@ -408,6 +532,7 @@ definitions: description: epoch ms type: integer observationCount: + description: legacy cumulative presence counter; includes non-packet events type: integer observerType: description: e.g. "meshcoretomqtt", "meshcoreha" @@ -442,9 +567,93 @@ definitions: type: string statusMetadata: description: raw /status JSON payload + type: object uptimeSeconds: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity: + properties: + generatedAt: + description: response computation time, not proof of continuous coverage + type: integer + interval: + type: string + payloadTypes: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem' + type: array + points: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint' + type: array + radio: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio' + range: + type: string + rawFrom: + description: start of the raw tail, epoch ms + type: integer + rolledUntil: + description: |- + Hourly only. Buckets before rolledUntil come from rollups and from rawFrom on from raw rows; + when rawFrom is later, the hours between are uncovered, not quiet. + type: integer + source: + description: raw or hourly; missing records do not prove an outage + type: string + summary: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary' + windowEnd: + description: exclusive end, epoch ms; live requests include the current partial + bucket + type: integer + windowStart: + description: inclusive complete-bucket start, epoch ms + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint: + properties: + airtimeMs: + type: number + observations: + type: integer + rssiAvg: + type: number + snrAvg: + type: number + snrMin: + type: number + t: + description: bucket start, epoch ms + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio: + properties: + bwKhz: + type: number + cr: + type: integer + freqMhz: + type: number + preambleSymbols: + type: integer + sf: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary: + properties: + lastCompleteHour: + type: integer + lastCompleteHourEnd: + type: integer + lastCompleteHourStart: + type: integer + latestRecordedAt: + type: integer + recordedPackets: + description: stored observations within windowStart/windowEnd + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker: properties: lastPacketAt: @@ -457,6 +666,27 @@ definitions: description: broker name e.g. "mqtt1" type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison: + properties: + both: + type: integer + observerA: + type: string + observerB: + type: string + onlyA: + type: integer + onlyB: + type: integer + since: + description: inclusive, epoch milliseconds + type: integer + totalPackets: + type: integer + until: + description: exclusive, epoch milliseconds + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary: properties: displayName: @@ -495,9 +725,9 @@ definitions: type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetryPoint: properties: - airtimeRxPct: + airtimeRxSecs: type: number - airtimeTxPct: + airtimeTxSecs: type: number batteryMv: type: integer @@ -590,6 +820,24 @@ definitions: type: string id: type: string + pathBytes: + description: hex-encoded accumulated path hashes + type: string + pathLength: + allOf: + - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketPathLength' + description: |- + PathLength/PathBytes and captured endpoint resolutions are stored on the observation, + so list/backfill reads need no per-hash resolution queries. Legacy observations have + no endpoint snapshot. ResolvedPath remains a detail/opted-in WS feature. + resolvedDestination: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop' + resolvedPath: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop' + type: array + resolvedSource: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop' type: object github_com_MeshCore-Beacon_beacon-server_internal_api.PacketObservationDetail: properties: @@ -626,7 +874,9 @@ definitions: - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop' description: |- ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type - carries a resolvable one: an exact match for ADVERT's full pubkey, an ambiguous + carries one. Prefer the snapshot captured at ingest; legacy observations without a + snapshot use the current node registry. Endpoint matching itself is unchanged: + an exact match for ADVERT's full pubkey, an ambiguous hash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte source/destination hashes. Nil when the payload type doesn't carry one at all (e.g. GRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and @@ -712,7 +962,7 @@ definitions: description: matched transport scope name e.g. "#bc" type: string summary: - description: human-readable payload summary + description: advert name, or the ACK/TRACE/PING summary; omitted when unavailable type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.PacketTransportCodes: @@ -733,17 +983,6 @@ definitions: nextCursor: type: integer type: object - github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary: - properties: - hasMore: - type: boolean - items: - items: - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary' - type: array - nextCursor: - type: integer - type: object github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary: properties: hasMore: @@ -777,6 +1016,72 @@ definitions: nextCursor: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth: + properties: + bytes: + type: integer + receptions: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour: + properties: + empty: + type: integer + hour: + type: integer + maxEntries: + description: longest path received that hour across the requested IATAs; 0 + if only empty paths + type: integer + oneByte: + type: integer + receptions: + type: integer + threeByte: + type: integer + trace: + type: integer + twoByte: + type: integer + unclassified: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin: + properties: + entries: + type: integer + receptions: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats: + properties: + empty: + type: integer + hashWidths: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth' + type: array + hashed: + type: integer + hourly: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour' + type: array + pathLengths: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin' + type: array + receptions: + type: integer + since: + type: integer + trace: + type: integer + unclassified: + type: integer + until: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem: properties: count: @@ -872,6 +1177,33 @@ definitions: description: hex-encoded prefix used for resolution type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence: + properties: + generatedAt: + type: integer + hasMore: + type: boolean + hashSize: + type: integer + items: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation' + type: array + matchAvailable: + type: boolean + matchType: + type: string + nextPageCursor: + type: string + pathBytes: + type: string + route: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute' + windowEnd: + type: integer + windowStart: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop: properties: hashBytes: @@ -884,6 +1216,27 @@ definitions: nodeId: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation: + properties: + heardAt: + type: integer + id: + type: integer + observerId: + type: string + observerName: + type: string + packetHash: + type: string + payloadType: + type: integer + payloadTypeName: + type: string + rssi: + type: integer + snr: + type: number + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail: properties: iataCount: @@ -901,8 +1254,27 @@ definitions: packetCount: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour: + properties: + hour: + description: epoch ms, start of the UTC hour + type: integer + nodes: + description: distinct nodes whose adverts were heard in the scope that hour + type: integer + observers: + description: distinct observers that heard the scope's packets that hour + type: integer + packets: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats: properties: + hourly: + description: packetCount by hour, oldest first; hours with none omitted + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour' + type: array name: description: normalized scope name e.g. "#bc" type: string @@ -916,30 +1288,170 @@ definitions: description: distinct packets matched to this scope type: integer type: object - github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview: + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin: properties: - activeIatas: + count: type: integer - activeObservers: + lower: + type: number + upper: + type: number + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour: + properties: + hour: type: integer - totalObservations: + receptions: type: integer - totalPackets: + rssiAverage: + type: number + rssiSamples: + type: integer + snrAverage: + type: number + snrSamples: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric: + properties: + average: + type: number + histogram: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin' + type: array + samples: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats: + properties: + hourly: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour' + type: array + receptions: + type: integer + rssi: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric' + since: + type: integer + snr: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric' + until: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview: + properties: + activeIatas: + type: integer + activeObservers: + type: integer + since: + description: epoch ms, start of the window + type: integer + totalObservations: + type: integer + totalPackets: + description: distinct per hour (see StatsSeriesValues) + type: integer + until: + description: epoch ms, exclusive end of the window type: integer windowHours: - description: always 24 for now + description: complete rolled hours the totals cover, at most 24 + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries: + properties: + completeHours: + description: hours contributing to summary + type: integer + earliestComplete: + description: first complete hour held, epoch ms; null if none + type: integer + hours: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour' + type: array + revision: + description: changes whenever rolled hours, their status or coverage change + type: integer + since: + description: effective start, epoch ms on a UTC hour + type: integer + summary: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues' + until: + description: effective exclusive end, epoch ms on a UTC hour + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour: + properties: + hour: + description: epoch ms, start of the hour + type: integer + status: + description: 'partial: raw rows deleted before it was rolled' + enum: + - complete + - partial + - missing + type: string + values: + allOf: + - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues' + description: null unless complete + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues: + properties: + activeIatas: + description: distinct across the hour or window + type: integer + activeObservers: + description: distinct across the hour or window + type: integer + activeScopes: + description: distinct across the hour or window + type: integer + maxPathEntries: + type: integer + observations: + type: integer + rssiSamples: + type: integer + rssiSum: + description: average = rssiSum / rssiSamples + type: number + scopedPackets: + type: integer + snrSamples: + type: integer + snrSum: + description: average = snrSum / snrSamples + type: number + uniquePackets: type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TopAdvertiser: properties: advertCount: type: integer + directAdvertCount: + type: integer + floodAdvertCount: + description: |- + FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed: + flood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path; + direct = route type 2 (direct) or 3 (transport_direct), routed along a known path. + FloodAdvertCount + DirectAdvertCount == AdvertCount. + type: integer iata: type: string lastHeard: description: epoch ms type: integer nodeId: + description: null once the node row has been deleted type: string nodeName: type: string @@ -947,6 +1459,9 @@ definitions: type: integer nodeTypeName: type: string + publicKey: + description: hex + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TopNode: properties: @@ -956,6 +1471,7 @@ definitions: description: epoch ms type: integer nodeId: + description: null once the node row has been deleted type: string nodeName: type: string @@ -965,6 +1481,9 @@ definitions: type: string observationCount: type: integer + publicKey: + description: hex + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TopObserver: properties: @@ -1062,6 +1581,31 @@ definitions: description: TRACE or PING type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest: + properties: + allowed_origins: + items: + type: string + type: array + required: + - allowed_origins + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest: + properties: + cors: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest' + required: + - cors + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse: + properties: + config: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig' + persisted: + type: boolean + requires_restart: + type: boolean + type: object internal_api_handlers.APIError: properties: code: @@ -1083,14 +1627,375 @@ info: contact: name: MeshCore Beacon url: https://github.com/MeshCore-Beacon/beacon-server - description: MeshCore network observation backend. Ingests LoRa packets from MQTT - brokers, stores in PostgreSQL, and streams live events via WebSocket. + description: |- + MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket. + REST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget. + WebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open. license: name: AGPL-3-or-later termsOfService: https://github.com/MeshCore-Beacon/beacon-server title: MeshCore Beacon API - version: 1.6.0 + version: 2.0.0 paths: + /admin/accounts: + get: + description: Includes active and inactive accounts, newest first. These records + are not login users. + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList' + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: List operator accounts + tags: + - Admin + post: + consumes: + - application/json + description: Trims the name; names are case-sensitive and unique among active + accounts. No credential or login is created. + parameters: + - description: Account name (1-128 Unicode characters, no control characters); + body at most 4 KiB + in: body + name: account + required: true + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest' + produces: + - application/json + responses: + "201": + description: Created + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account' + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "409": + description: Conflict + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "413": + description: Request Entity Too Large + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "415": + description: Unsupported Media Type + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Create an operator account + tags: + - Admin + /admin/accounts/{id}: + delete: + description: Soft deactivation preserves the record. Its name may be reused + by a new account. + parameters: + - description: Account UUID + format: uuid + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "204": + description: Deactivated + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "404": + description: Not Found + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "409": + description: Conflict + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Deactivate an operator account + tags: + - Admin + get: + description: Returns active or inactive account records. + parameters: + - description: Account UUID + format: uuid + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account' + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "404": + description: Not Found + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Get an operator account + tags: + - Admin + /admin/backup: + get: + description: Opt-in operator export. Returns only a completed bundle; one export/transfer + at a time. Contains secrets. Login sessions, external deployment files and + import are outside this endpoint. + produces: + - application/gzip + responses: + "200": + description: OK + headers: + Content-Disposition: + description: attachment; filename=beacon-backup.tar.gz + type: string + schema: + type: file + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "409": + description: Conflict + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "504": + description: Gateway Timeout + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "507": + description: Insufficient Storage + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Download a private database and saved-config backup + tags: + - Admin + /admin/config: + get: + description: Returns current CORS options, auth configuration status and configured + broker count. Credential fields and other configuration are excluded. Runtime + origin updates are lost on restart. + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig' + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Inspect selected running configuration + tags: + - Admin + put: + consumes: + - application/json + description: Replaces only cors.allowed_origins immediately. Updates are serialized; + concurrent valid updates are applied one at a time. Already-running requests + may finish with the previous policy. Nothing is persisted; restart reloads + saved configuration. + parameters: + - description: 1-32 ASCII HTTP(S) origins, at most 512 bytes each; one hostname + wildcard is supported, or a sole *. Empty/null lists and unsupported fields + are rejected. Body at most 16 KiB. + in: body + name: config + required: true + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse' + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "413": + description: Request Entity Too Large + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "415": + description: Unsupported Media Type + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Update runtime CORS origins + tags: + - Admin /brokers: get: produces: @@ -1112,20 +2017,37 @@ paths: in: query name: hash type: string - - description: Filter by IATA code + - description: 'Filter by IATA code: channels MeshMapper lists there, config + channels scoped to a region containing it, and Beacon-wide config channels' in: query name: iata type: string - - description: Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ + - description: Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; + same membership rule as iata in: query name: iatas type: string - - description: last_seen epoch ms of last item for pagination + - description: Only channels Beacon can (true) or cannot (false) decrypt; omit + for all + in: query + name: keyKnown + type: boolean + - description: last_seen epoch ms of last item for pagination; 0 starts from + the beginning in: query name: cursor type: integer - - description: Max results (default 50) + - description: Opaque nextPageCursor from a previous response; preserves timestamp + ties and precision. Cannot be combined with a positive cursor; cursor=0 + is allowed. + in: query + name: pageCursor + type: string + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1134,7 +2056,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary' + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage' "400": description: Bad Request schema: @@ -1169,6 +2091,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get channel detail tags: - Channels @@ -1205,8 +2131,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1238,8 +2167,8 @@ paths: items: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA' type: array - "404": - description: Not Found + "500": + description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' summary: List all IATA codes @@ -1264,6 +2193,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get a single IATA code tags: - IATAs @@ -1288,6 +2221,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get an IATA's GeoJSON border, if configured tags: - IATAs @@ -1327,8 +2264,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1373,8 +2313,11 @@ paths: in: query name: scope type: string - - description: Max results (default 100) + - description: Max results (default 100); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1457,8 +2400,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1502,6 +2448,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get node detail tags: - Nodes @@ -1545,8 +2495,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1610,8 +2563,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1655,9 +2611,59 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get observer detail tags: - Observers + /observers/{observerId}/activity: + get: + parameters: + - description: Observer UUID + in: path + name: observerId + required: true + type: string + - description: Trailing window as a Go duration, max 720h (default 24h); max + 48h when interval is under 1h + in: query + name: range + type: string + - description: 'Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)' + in: query + name: interval + type: string + - description: Optional exclusive end in epoch milliseconds; aligned down to + a complete bucket, at most 30 days old. Activity uses this window; summary + freshness/latestRecordedAt and the last complete hour are measured at response + generation time, independently of until. + in: query + name: until + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Get observer heard-activity history + tags: + - Observers /observers/{observerId}/adverts: get: parameters: @@ -1670,8 +2676,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1792,8 +2801,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1882,8 +2894,11 @@ paths: in: query name: scope type: string - - description: Max results (default 100) + - description: Max results (default 100); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1917,8 +2932,8 @@ paths: items: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RegionSummary' type: array - "404": - description: Not Found + "500": + description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' summary: List all regions @@ -1947,6 +2962,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get a single region tags: - Regions @@ -1957,16 +2976,86 @@ paths: in: query name: iata type: string - - description: Filter by exact hop count + - description: Filter by exact hop count + in: query + name: hopCount + type: integer + - description: Epoch ms timestamp of last item for pagination + in: query + name: cursor + type: integer + - description: id of the last item; with cursor, also returns later routes sharing + that millisecond + in: query + name: cursorId + type: integer + - description: Max results (default 50); must be positive, values above 200 + are clamped + in: query + maximum: 200 + minimum: 1 + name: limit + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute' + type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: List known routes + tags: + - Routes + /routes/{iata}/{pathKey}/observations: + get: + description: Matches the saved IATA, complete path bytes and hash width. Excludes + TRACE and unclassified reports; other widths and search-result subsegments + are not included. Byte matches do not prove hop identities or delivery. The + route and its historical counter can outlive raw evidence. Cursors retain + full timestamp precision and pin route/window scope. + parameters: + - description: Three-character IATA code + in: path + name: iata + required: true + type: string + - description: Stable 32-hex pathKey from a known-route response + in: path + name: pathKey + required: true + type: string + - description: Server-anchored duration (default 24h, max 720h); exclusive with + since/until/pageCursor in: query - name: hopCount + name: range + type: string + - description: Window start epoch ms; provide with until (default last 24h) + in: query + name: since type: integer - - description: Epoch ms timestamp of last item for pagination + - description: Exclusive window end epoch ms; maximum span 30d, end may be up + to 5 minutes ahead of server time (clock skew tolerance) in: query - name: cursor + name: until type: integer - - description: Max results (default 50) + - description: Opaque precise cursor from nextPageCursor; window is pinned + in: query + name: pageCursor + type: string + - description: Default 50; positive, capped at 200 in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1975,14 +3064,20 @@ paths: "200": description: OK schema: - items: - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute' - type: array + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/internal_api_handlers.APIError' "500": description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: List known routes + summary: Get retained reports matching a full saved route prefix sequence tags: - Routes /routes/cross: @@ -2068,6 +3163,11 @@ paths: - Routes /scopes: get: + description: Without filters, lists every stored scope name, including imported + names retained after an importer is disabled or a source is removed. With + IATA or region filters, lists only manual scopes configured for a matching + region and imported scopes whose current MeshMapper catalogue includes a matching + IATA; observed traffic does not add scopes. parameters: - description: Filter by IATA code(s), comma-separated in: query @@ -2087,7 +3187,13 @@ paths: "200": description: OK schema: - type: object + items: + type: string + type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' "500": description: Internal Server Error schema: @@ -2097,6 +3203,9 @@ paths: - Scopes /scopes/{name}: get: + description: packetCount sums packets per hour across every retained rollup + hour; iatas are the IATAs that heard the scope's packets. Observer and node + counts are current memberships. parameters: - description: 'Scope name e.g. %23bc (URL-encoded #bc)' in: path @@ -2121,6 +3230,45 @@ paths: summary: Get scope detail by name tags: - Scopes + /stats/clock-drift: + get: + parameters: + - description: Comma-separated IATA codes + in: query + name: iatas + type: string + - description: Filter by region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Filter by region slug, expands to member IATAs + in: query + name: region + type: string + - description: Max results (default 10); must be positive, values above 200 + are clamped + in: query + maximum: 200 + minimum: 1 + name: limit + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Repeaters/room servers whose clock has drifted beyond the configured + threshold, worst first + tags: + - Stats /stats/node-types: get: parameters: @@ -2154,6 +3302,8 @@ paths: - Stats /stats/observations: get: + description: Observation counts only; distinct packet and observer counts don't + sum across IATAs, so read them from /stats/series. parameters: - description: Comma-separated IATA codes in: query @@ -2184,11 +3334,79 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: Hourly observation time series + summary: Hourly observation counts per IATA + tags: + - Stats + /stats/observer-comparison: + get: + description: Counts distinct retained packet hashes with route type 0 or 1, + using reception time in [since, until). onlyA, onlyB and both partition totalPackets + (the union). Repeated receptions count once. These are reported receptions, + not a measure of radio packet loss or continuous observer coverage. + parameters: + - description: First observer UUID + in: query + name: observerA + required: true + type: string + - description: Second, distinct observer UUID + in: query + name: observerB + required: true + type: string + - description: Inclusive start, epoch milliseconds (0 through 253402300799999) + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; later than since + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Compare flood packets reported by two observers tags: - Stats /stats/overview: get: + description: Summarises the 24 most recent hours that can have been rolled (an + hour rolls about 95 minutes after it closes); since/until report that window. + Packets count once per hour heard. Use /stats/series for sparklines. parameters: - description: Comma-separated IATA codes in: query @@ -2213,7 +3431,64 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: Network overview stats (last 24h) + summary: Network overview stats (last 24 rolled hours) + tags: + - Stats + /stats/paths: + get: + description: Counts stored receptions in [since, until), at most 30 days. Hash + widths count only validated nonempty ordinary header paths (1/2/3 bytes). + Empty paths do not vote for a width. TRACE header paths contain signal readings + and are separate. Missing payload type or inconsistent/unsupported path metadata + is unclassified. The four categories partition total receptions. Flood paths + accumulate entries; direct paths contain remaining entries, so counts do not + measure distance or a complete traversed route. Both boundaries round down + to UTC hours and the response reports that effective window. Reads the hourly + path rollup; an hour appears only after it is rolled, about 95 minutes after + it ends, and hours without data are omitted. + parameters: + - description: Inclusive start, epoch milliseconds (0 through 253402300799999) + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; at most 30 days after since + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Received path-entry and hash-width distributions tags: - Stats /stats/payload-breakdown: @@ -2288,6 +3563,38 @@ paths: - Stats /stats/scopes: get: + description: 'Packets are those heard since the window start, each counted once + per hour however many of the requested IATAs heard it. Observers and nodes + are current memberships: observers filter by the IATA they last reported from, + nodes by their IATA memberships. hourly splits packetCount by UTC hour and + adds the distinct observers and advertising nodes active in the scope each + hour (empty hours omitted; outage gaps come from /stats/series hour status). + With filters, lists only manual scopes configured for a matching region and + imported scopes whose current MeshMapper catalogue includes a matching IATA; + those remain listed with zero counts. An empty region returns an empty array.' + parameters: + - description: Comma-separated IATA codes + in: query + name: iatas + type: string + - description: Single IATA code; used when iatas is absent + in: query + name: iata + type: string + - description: Filter by region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Filter by region slug, expands to member IATAs; combined with + explicit IATAs + in: query + name: region + type: string + - description: Start of packet window epoch ms, rounded down to the hour (default + 7 days ago) + in: query + name: since + type: integer produces: - application/json responses: @@ -2297,6 +3604,10 @@ paths: items: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats' type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' "500": description: Internal Server Error schema: @@ -2304,8 +3615,123 @@ paths: summary: Scope statistics tags: - Stats + /stats/series: + get: + description: One entry per UTC hour in [since, until) plus a summary over the + complete hours. Both bounds round down to UTC hours and since is clamped to + the rollup retention; the response reports the effective window. An hour is + rolled about 95 minutes after it closes, so recent hours are "missing" until + then; "partial" hours lost raw rows before they could be rolled and never + get values. Counts (observations, packets) sum across hours, with a packet + counted once in each hour it was heard; observers, IATAs and scopes are distinct + across the whole window; averages are sum / samples. revision changes whenever + any rolled hour changes. + parameters: + - description: Inclusive start, epoch milliseconds + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; the window is at most the + rollup retention (default 90 days) + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Hourly network activity from the analytics rollups + tags: + - Stats + /stats/signal: + get: + description: Aggregates stored observer receptions in [since, until), at most + 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero + unavailable sentinel are excluded per metric; actual zero SNR with nonzero + RSSI remains valid. Averages are null without samples. Histogram bounds are + lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries + round down to UTC hours and the response reports that effective window. Reads + the hourly signal rollup; an hour appears only after it is rolled, about 95 + minutes after it ends, and hours without data are omitted. These last-hop + readings do not measure end-to-end quality or packet loss. + parameters: + - description: Inclusive start, epoch milliseconds (0 through 253402300799999) + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; at most 30 days after since + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Reception signal distributions and hourly trends + tags: + - Stats /stats/top-advertisers: get: + description: Each advert counts once per hour heard, however many requested + IATAs heard it. nodeId is null when the node row has been deleted. parameters: - description: Comma-separated IATA codes in: query @@ -2323,8 +3749,11 @@ paths: in: query name: since type: integer - - description: Max results (default 10) + - description: Max results (default 10); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2345,6 +3774,8 @@ paths: - Stats /stats/top-nodes: get: + description: nodeId is null when the node row has been deleted; publicKey always + identifies it. parameters: - description: Comma-separated IATA codes in: query @@ -2358,8 +3789,16 @@ paths: in: query name: region type: string - - description: Max results (default 10) + - description: Start of window epoch ms, rounded down to the hour (default 7 + days ago) + in: query + name: since + type: integer + - description: Max results (default 10); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2375,7 +3814,7 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: Top N nodes by observation count (from materialized view) + summary: Top N nodes by advert hearings (last 7 days by default) tags: - Stats /stats/top-observers: @@ -2397,8 +3836,11 @@ paths: in: query name: since type: integer - - description: Max results (default 10) + - description: Max results (default 10); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2436,8 +3878,11 @@ paths: in: query name: since type: integer - - description: Max results (default 10) + - description: Max results (default 10); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2472,11 +3917,12 @@ paths: in: query name: regionId type: integer - - description: Filter by transport scope name + - description: Filter by the tag's first transport scope name in: query name: scope type: string - - description: 'Filter by type: TRACE or PING (default: all)' + - description: 'Filter by type: TRACE (any multi-hop packet) or PING (default: + all)' in: query name: type type: string @@ -2488,12 +3934,20 @@ paths: in: query name: until type: integer - - description: last_heard_at epoch ms of last item for pagination + - description: lastHeardAt (epoch ms) of the last item; returns older tags in: query name: cursor type: integer - - description: Max results (default 50) + - description: traceTag (hex) of the last item; with cursor, also returns later + tags sharing that millisecond + in: query + name: cursorTag + type: string + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2505,6 +3959,10 @@ paths: items: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TraceTagSummary' type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' "500": description: Internal Server Error schema: @@ -2541,6 +3999,12 @@ paths: schemes: - http - https +securityDefinitions: + AdminKey: + description: Enter Bearer followed by the configured operator key. Use HTTPS. + in: header + name: Authorization + type: apiKey swagger: "2.0" tags: - description: Airport/location codes that group observers and packets diff --git a/env.example b/env.example index 96bf825d..c7a152ef 100644 --- a/env.example +++ b/env.example @@ -1,8 +1,22 @@ +# Local development starter. Every variable is documented at +# https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/configuration.md + LISTEN_ADDR=:8080 +# Path to the YAML config (default: config.yaml). +# CONFIG_PATH=config.yaml + +# Optional admin bearer key, at least 16 characters with no whitespace. +# If set, overrides auth.api_key (even when empty). +# BEACON_API_KEY= + +# Optional application logging overrides (defaults: info / text). +# LOG_LEVEL=info +# LOG_FORMAT=json + POSTGRES_DSN=postgres://beacon:beacon@localhost:5432/beacon?sslmode=disable -# Redis (optional — leave REDIS_ADDR unset to disable caching) +# Redis (optional; leave REDIS_ADDR unset to disable caching) REDIS_ADDR=localhost:6379 REDIS_PASSWORD= REDIS_DB=0 @@ -14,3 +28,7 @@ MQTT_BROKER_1_PASSWORD= MQTT_BROKER_2_URL=wss://mqtt2.meshcore.ca:443 MQTT_BROKER_2_USERNAME= MQTT_BROKER_2_PASSWORD= + +# Optional private CPU captures. Set both; the deadline must be within 72 hours. +# BEACON_CPU_PROFILE_DIR=/profiles +# BEACON_CPU_PROFILE_UNTIL=2026-10-02T12:00:00Z diff --git a/go.mod b/go.mod index 6f981fab..0fc10fac 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/MeshCore-Beacon/beacon-server -go 1.26.5 +go 1.26.6 require ( github.com/alicebob/miniredis/v2 v2.38.0 @@ -8,6 +8,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.5.1 github.com/go-chi/chi/v5 v5.3.0 github.com/go-chi/cors v1.2.2 + github.com/go-chi/httprate v0.16.0 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.9.2 github.com/joho/godotenv v1.5.1 @@ -17,6 +18,7 @@ require ( github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.6 go.uber.org/mock v0.6.0 + golang.org/x/sync v0.21.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -33,16 +35,18 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect github.com/yuin/gopher-lua v1.1.1 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect - golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.39.0 // indirect golang.org/x/tools v0.47.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index ef86ff73..b4d9df21 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8= +github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -88,6 +90,8 @@ github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= diff --git a/internal/api/account_requests.go b/internal/api/account_requests.go new file mode 100644 index 00000000..f040f980 --- /dev/null +++ b/internal/api/account_requests.go @@ -0,0 +1,8 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +type CreateAccountRequest struct { + Name string `json:"name" validate:"required"` +} diff --git a/internal/api/accounts.go b/internal/api/accounts.go new file mode 100644 index 00000000..df64fbf3 --- /dev/null +++ b/internal/api/accounts.go @@ -0,0 +1,60 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "context" + "errors" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/google/uuid" +) + +var ( + ErrAccountNotFound = errors.New("account not found") + ErrAccountInactive = errors.New("account is already inactive") + ErrAccountNameConflict = errors.New("an active account already uses this name") + ErrAccountNameInvalid = errors.New("account name must be 1-128 characters without control characters") +) + +// Account is an operator-defined record, not a login or authentication principal. +type Account struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + DeactivatedAt *time.Time `json:"deactivated_at,omitempty"` + Active bool `json:"active"` +} + +type AccountList struct { + Items []Account `json:"items"` +} + +// AccountStore is separate from public Reader/cache and ingest write interfaces. +type AccountStore interface { + CreateAccount(context.Context, string) (Account, error) + ListAccounts(context.Context) ([]Account, error) + GetAccount(context.Context, uuid.UUID) (Account, error) + DeactivateAccount(context.Context, uuid.UUID) error +} + +// NormalizeAccountName applies the same validation to every account writer. +func NormalizeAccountName(name string) (string, error) { + if !utf8.ValidString(name) { + return "", ErrAccountNameInvalid + } + name = strings.TrimSpace(name) + if length := utf8.RuneCountInString(name); length < 1 || length > 128 { + return "", ErrAccountNameInvalid + } + for _, r := range name { + if unicode.IsControl(r) { + return "", ErrAccountNameInvalid + } + } + return name, nil +} diff --git a/internal/api/accounts_test.go b/internal/api/accounts_test.go new file mode 100644 index 00000000..be4e247c --- /dev/null +++ b/internal/api/accounts_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "errors" + "strings" + "testing" +) + +func TestNormalizeAccountName(t *testing.T) { + for _, tc := range []struct { + input, want string + invalid bool + }{ + {" Montréal 🦀 \t", "Montréal 🦀", false}, {"A", "A", false}, + {strings.Repeat("é", 128), strings.Repeat("é", 128), false}, + {"", "", true}, {"\u00a0\t", "", true}, {strings.Repeat("é", 129), "", true}, + {"x\x00y", "", true}, {"x\ny", "", true}, {"\xff", "", true}, + } { + got, err := NormalizeAccountName(tc.input) + if got != tc.want || errors.Is(err, ErrAccountNameInvalid) != tc.invalid { + t.Fatalf("name validation: got %q, error=%v", got, err) + } + } +} diff --git a/internal/api/admin.go b/internal/api/admin.go new file mode 100644 index 00000000..e71b38c6 --- /dev/null +++ b/internal/api/admin.go @@ -0,0 +1,46 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +// AdminConfig is a whitelist of selected nonsecret running settings. +// It is not a serialization of the full file or environment configuration. +type AdminConfig struct { + Auth AdminAuthConfig `json:"auth"` + CORS AdminCORSConfig `json:"cors"` + Ingest AdminIngestConfig `json:"ingest"` +} + +type AdminAuthConfig struct { + Configured bool `json:"configured"` +} + +// AdminCORSConfig reports the current options supplied to the CORS middleware, +// with Beacon defaults applied, before that library's matching normalization. +type AdminCORSConfig struct { + AllowedOrigins []string `json:"allowed_origins"` + AllowedMethods []string `json:"allowed_methods"` + AllowedHeaders []string `json:"allowed_headers"` + AllowCredentials bool `json:"allow_credentials"` + MaxAge int `json:"max_age"` +} + +type AdminIngestConfig struct { + // BrokerCount counts configured broker workers, not active MQTT connections + // or a configurable processing-worker pool. + BrokerCount int `json:"broker_count"` +} + +type UpdateAdminConfigRequest struct { + CORS *UpdateAdminCORSRequest `json:"cors" validate:"required"` +} + +type UpdateAdminCORSRequest struct { + AllowedOrigins *[]string `json:"allowed_origins" validate:"required"` +} + +type UpdateAdminConfigResponse struct { + Config AdminConfig `json:"config"` + Persisted bool `json:"persisted"` + RequiresRestart bool `json:"requires_restart"` +} diff --git a/internal/api/channel_scope_test.go b/internal/api/channel_scope_test.go new file mode 100644 index 00000000..f553a628 --- /dev/null +++ b/internal/api/channel_scope_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import "testing" + +func TestRecordedChannelScopeStatus(t *testing.T) { + name := "#yow" + for _, tc := range []struct { + name *string + transport, valid bool + want string + }{ + {&name, true, true, "matched"}, {nil, true, true, "unknown"}, + {nil, false, true, "unscoped"}, {nil, false, false, "unavailable"}, + {&name, false, false, "matched"}, + } { + var transport *bool + if tc.valid { + transport = &tc.transport + } + if got := RecordedChannelScopeStatus(tc.name, transport); string(got) != tc.want { + t.Fatalf("got %s, want %s", got, tc.want) + } + } +} diff --git a/internal/api/channels.go b/internal/api/channels.go index ec91009e..50ed66f4 100644 --- a/internal/api/channels.go +++ b/internal/api/channels.go @@ -3,16 +3,85 @@ package api +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +// ChannelPage adds a precise cursor while retaining the numeric cursor for older clients. +type ChannelPage struct { + Page[ChannelSummary] + NextPageCursor *string `json:"nextPageCursor,omitempty"` +} + +// ChannelCursor identifies a boundary in (last_seen DESC, id DESC) order. +type ChannelCursor struct { + LastSeen time.Time + ID int32 +} + +func (c ChannelCursor) String() string { + return fmt.Sprintf("v1:%d:%d", c.LastSeen.UnixMicro(), c.ID) +} + +var errChannelCursor = errors.New("invalid channel page cursor") + +// ParseChannelCursor parses the opaque, versioned cursor returned by ChannelPage. +func ParseChannelCursor(raw string) (*ChannelCursor, error) { + if len(raw) > 64 { + return nil, errChannelCursor + } + parts := strings.Split(raw, ":") + if len(parts) != 3 || parts[0] != "v1" { + return nil, errChannelCursor + } + micros, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, errChannelCursor + } + id, err := strconv.ParseInt(parts[2], 10, 32) + if err != nil || id <= 0 { + return nil, errChannelCursor + } + at := time.UnixMicro(micros).UTC() + if at.Year() < 1 || at.Year() > 9999 { + return nil, errChannelCursor + } + return &ChannelCursor{LastSeen: at, ID: int32(id)}, nil +} + // ChannelMessage represents a single decrypted channel message. // Only messages for channels with a known key are stored and returned. type ChannelMessage struct { - ID int64 `json:"id"` - PacketHash string `json:"packetHash"` // hex-encoded packet hash for correlation with packet events - ChannelHash string `json:"channelHash"` // hex-encoded single-byte channel hash - SenderName string `json:"senderName"` // display name from the decrypted payload - Content string `json:"content"` // decrypted message text - SentAt int64 `json:"sentAt"` // epoch ms, from the sender's embedded timestamp - ObservationCount int64 `json:"observationCount"` // number of packet_observations rows for this message's packet hash + ID int64 `json:"id"` + PacketHash string `json:"packetHash"` // hex-encoded packet hash for correlation with packet events + ChannelHash string `json:"channelHash"` // hex-encoded single-byte channel hash + SenderName string `json:"senderName"` // display name from the decrypted payload + Content string `json:"content"` // decrypted message text + SentAt int64 `json:"sentAt"` // epoch ms, from the sender's embedded timestamp + ObservationCount int64 `json:"observationCount"` // number of packet_observations rows for this message's packet hash + Scope *string `json:"scope"` // matched scope on the first stored packet; null when none was recorded + ScopeStatus ChannelScopeStatus `json:"scopeStatus" enums:"matched,unscoped,unknown,unavailable"` +} + +// ChannelScopeStatus describes stored packet evidence, not a channel key or all reception paths. +type ChannelScopeStatus string + +// RecordedChannelScopeStatus preserves missing capture metadata as unavailable. +func RecordedChannelScopeStatus(name *string, transport *bool) ChannelScopeStatus { + if name != nil { + return "matched" + } + if transport == nil { + return "unavailable" + } + if *transport { + return "unknown" + } + return "unscoped" } // ChannelSummary is the minimal channel representation used in list responses. @@ -32,5 +101,5 @@ type Channel struct { ChannelSummary Hashtag *string `json:"hashtag,omitempty"` // tag name without # prefix; non-nil only for hashtag channels KeyFingerprint *string `json:"keyFingerprint,omitempty"` // first 8 bytes of SHA256(key), hex-encoded - MessageCount int64 `json:"messageCount"` + MessageCount int64 `json:"messageCount"` // lifetime count; not reduced by retention } diff --git a/internal/api/channels_test.go b/internal/api/channels_test.go new file mode 100644 index 00000000..bf53c499 --- /dev/null +++ b/internal/api/channels_test.go @@ -0,0 +1,34 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "strings" + "testing" + "time" +) + +func TestChannelCursorRoundTrip(t *testing.T) { + for _, at := range []time.Time{ + time.Date(2026, 9, 8, 12, 0, 0, 123456000, time.FixedZone("offset", 3600)), + time.UnixMicro(0), time.UnixMicro(-1), + time.Date(9999, 12, 31, 23, 59, 59, 999999000, time.UTC), + } { + want := ChannelCursor{LastSeen: at, ID: 2147483647} + got, err := ParseChannelCursor(want.String()) + if err != nil || got.ID != want.ID || !got.LastSeen.Equal(want.LastSeen) { + t.Fatalf("round trip lost timestamp precision or ID: %v", err) + } + } +} + +func TestChannelCursorInvalid(t *testing.T) { + for _, raw := range []string{"", "1", "v2:1:1", "v1:1", "v1:1:1:1", "v1:x:1", "v1:1.5:1", + "v1:1:0", "v1:1:-1", "v1:1:2147483648", "v1:9223372036854775808:1", + "v1:-9223372036854775808:1", "v1:9223372036854775807:1", strings.Repeat("x", 65)} { + if _, err := ParseChannelCursor(raw); err == nil { + t.Errorf("accepted invalid cursor %q", raw) + } + } +} diff --git a/internal/api/handlers/accounts.go b/internal/api/handlers/accounts.go new file mode 100644 index 00000000..3f4fb287 --- /dev/null +++ b/internal/api/handlers/accounts.go @@ -0,0 +1,175 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "encoding/json" + "errors" + "io" + "log/slog" + "mime" + "net/http" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" + "github.com/google/uuid" +) + +// AccountsRouter must be mounted inside the authenticated admin subtree. +func AccountsRouter(store api.AccountStore) http.Handler { + if store == nil { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { respondError(w, 503, "account storage is unavailable") }) + } + r := chi.NewRouter() + r.Get("/", listAccounts(store)) + r.Post("/", createAccount(store)) + r.Get("/{id}", getAccount(store)) + r.Delete("/{id}", deactivateAccount(store)) + return r +} + +// listAccounts godoc +// @Summary List operator accounts +// @Description Includes active and inactive accounts, newest first. These records are not login users. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Success 200 {object} api.AccountList +// @Failure 401,503,500 {object} map[string]APIError +// @Router /admin/accounts [get] +func listAccounts(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + items, err := store.ListAccounts(r.Context()) + if err != nil { + accountError(w, err) + return + } + if items == nil { + items = []api.Account{} + } + respond(w, 200, api.AccountList{Items: items}) + } +} + +// createAccount godoc +// @Summary Create an operator account +// @Description Trims the name; names are case-sensitive and unique among active accounts. No credential or login is created. +// @Tags Admin +// @Accept json +// @Produce json +// @Security AdminKey +// @Param account body api.CreateAccountRequest true "Account name (1-128 Unicode characters, no control characters); body at most 4 KiB" +// @Success 201 {object} api.Account +// @Failure 400,401,409,413,415,503,500 {object} map[string]APIError +// @Router /admin/accounts [post] +func createAccount(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/json" { + respondError(w, 415, "Content-Type must be application/json") + return + } + r.Body = http.MaxBytesReader(w, r.Body, 4096) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + var input api.CreateAccountRequest + err = decoder.Decode(&input) + if err == nil { + var extra any + err = decoder.Decode(&extra) + if errors.Is(err, io.EOF) { + err = nil + } else if err == nil { + err = errors.New("multiple JSON values") + } + } + if err != nil { + var sizeError *http.MaxBytesError + if errors.As(err, &sizeError) { + respondError(w, 413, "request body exceeds 4 KiB") + } else { + respondError(w, 400, "invalid account JSON") + } + return + } + name, err := api.NormalizeAccountName(input.Name) + if err != nil { + accountError(w, err) + return + } + account, err := store.CreateAccount(r.Context(), name) + if err != nil { + accountError(w, err) + return + } + respond(w, 201, account) + } +} + +// getAccount godoc +// @Summary Get an operator account +// @Description Returns active or inactive account records. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Param id path string true "Account UUID" format(uuid) +// @Success 200 {object} api.Account +// @Failure 400,401,404,503,500 {object} map[string]APIError +// @Router /admin/accounts/{id} [get] +func getAccount(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id, err := uuid.Parse(chi.URLParam(r, "id")) + if err != nil { + respondError(w, 400, "invalid account ID") + return + } + account, err := store.GetAccount(r.Context(), id) + if err != nil { + accountError(w, err) + return + } + respond(w, 200, account) + } +} + +// deactivateAccount godoc +// @Summary Deactivate an operator account +// @Description Soft deactivation preserves the record. Its name may be reused by a new account. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Param id path string true "Account UUID" format(uuid) +// @Success 204 "Deactivated" +// @Failure 400,401,404,409,503,500 {object} map[string]APIError +// @Router /admin/accounts/{id} [delete] +func deactivateAccount(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id, err := uuid.Parse(chi.URLParam(r, "id")) + if err != nil { + respondError(w, 400, "invalid account ID") + return + } + if err := store.DeactivateAccount(r.Context(), id); err != nil { + accountError(w, err) + return + } + w.WriteHeader(204) + } +} + +func accountError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, api.ErrAccountNameInvalid): + respondError(w, 400, api.ErrAccountNameInvalid.Error()) + case errors.Is(err, api.ErrAccountNameConflict): + respondError(w, 409, api.ErrAccountNameConflict.Error()) + case errors.Is(err, api.ErrAccountNotFound): + respondError(w, 404, api.ErrAccountNotFound.Error()) + case errors.Is(err, api.ErrAccountInactive): + respondError(w, 409, api.ErrAccountInactive.Error()) + default: + slog.Error("account request failed", "component", "api", "error", err) + respondError(w, 500, "internal server error") + } +} diff --git a/internal/api/handlers/accounts_test.go b/internal/api/handlers/accounts_test.go new file mode 100644 index 00000000..50aaa090 --- /dev/null +++ b/internal/api/handlers/accounts_test.go @@ -0,0 +1,164 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + mw "github.com/MeshCore-Beacon/beacon-server/internal/api/middleware" + "github.com/google/uuid" +) + +type accountStub struct { + account api.Account + items []api.Account + err error + calls int + name string + id uuid.UUID +} + +func (s *accountStub) CreateAccount(_ context.Context, name string) (api.Account, error) { + s.calls++ + s.name = name + return s.account, s.err +} +func (s *accountStub) ListAccounts(context.Context) ([]api.Account, error) { + s.calls++ + return s.items, s.err +} +func (s *accountStub) GetAccount(_ context.Context, id uuid.UUID) (api.Account, error) { + s.calls++ + s.id = id + return s.account, s.err +} +func (s *accountStub) DeactivateAccount(_ context.Context, id uuid.UUID) error { + s.calls++ + s.id = id + return s.err +} + +func accountRequest(handler http.Handler, method, path, body, contentType, token string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + return w +} + +func TestAccountInputAndErrors(t *testing.T) { + id := uuid.New() + for _, tc := range []struct { + name, method, path, body, media string + err error + status, calls int + }{ + {"create", "POST", "/", `{"name":" Alice "}`, "application/json", nil, 201, 1}, + {"media parameters", "POST", "/", `{"name":"Alice"}`, "application/json; charset=utf-8", nil, 201, 1}, + {"empty", "POST", "/", `{}`, "application/json", nil, 400, 0}, + {"null", "POST", "/", `null`, "application/json", nil, 400, 0}, + {"wrong field", "POST", "/", `{"name":"Alice","admin":true}`, "application/json", nil, 400, 0}, + {"trailing JSON", "POST", "/", `{"name":"Alice"} {}`, "application/json", nil, 400, 0}, + {"malformed", "POST", "/", `{"name":`, "application/json", nil, 400, 0}, + {"wrong name type", "POST", "/", `{"name":3}`, "application/json", nil, 400, 0}, + {"control", "POST", "/", `{"name":"x\u0000y"}`, "application/json", nil, 400, 0}, + {"oversized", "POST", "/", `{"name":"` + strings.Repeat("a", 5000) + `"}`, "application/json", nil, 413, 0}, + {"oversized trailing space", "POST", "/", `{"name":"Alice"}` + strings.Repeat(" ", 5000), "application/json", nil, 413, 0}, + {"missing media", "POST", "/", `{"name":"Alice"}`, "", nil, 415, 0}, + {"wrong media", "POST", "/", `{"name":"Alice"}`, "text/plain", nil, 415, 0}, + {"conflict", "POST", "/", `{"name":"Alice"}`, "application/json", fmt.Errorf("private-error: %w", api.ErrAccountNameConflict), 409, 1}, + {"get", "GET", "/" + id.String(), "", "", nil, 200, 1}, + {"bad ID", "GET", "/invalid", "", "", nil, 400, 0}, + {"bad delete ID", "DELETE", "/invalid", "", "", nil, 400, 0}, + {"not found", "GET", "/" + id.String(), "", "", api.ErrAccountNotFound, 404, 1}, + {"deactivate", "DELETE", "/" + id.String(), "", "", nil, 204, 1}, + {"inactive", "DELETE", "/" + id.String(), "", "", api.ErrAccountInactive, 409, 1}, + {"delete absent", "DELETE", "/" + id.String(), "", "", api.ErrAccountNotFound, 404, 1}, + {"list database failure", "GET", "/", "", "", errors.New("private-error"), 500, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + store := &accountStub{account: api.Account{ID: id, Name: "Alice", CreatedAt: time.Unix(1, 0).UTC(), Active: true}, err: tc.err} + handler := mw.BearerAuth("test-key", AccountsRouter(store)) + w := accountRequest(handler, tc.method, tc.path, tc.body, tc.media, "test-key") + if w.Code != tc.status || store.calls != tc.calls { + t.Fatalf("status=%d calls=%d", w.Code, store.calls) + } + if w.Header().Get("Cache-Control") != "no-store" || strings.Contains(w.Body.String(), "private-error") { + t.Fatal("cache policy or error disclosure") + } + if tc.status == 204 { + if w.Body.Len() != 0 { + t.Fatal("204 body") + } + return + } + if w.Header().Get("Content-Type") != "application/json" || !json.Valid(w.Body.Bytes()) { + t.Fatal("invalid JSON response") + } + if tc.status == 201 && store.name != "Alice" { + t.Fatal("untrimmed name reached storage") + } + if (tc.method == "DELETE" || tc.method == "GET") && tc.calls > 0 && tc.path != "/" && store.id != id { + t.Fatal("wrong account ID") + } + }) + } +} + +func TestAccountListAndAuthorization(t *testing.T) { + for _, items := range [][]api.Account{nil, {{ID: uuid.New(), Name: "inactive", Active: false}}} { + store := &accountStub{items: items} + handler := mw.BearerAuth("test-key", AccountsRouter(store)) + w := accountRequest(handler, "GET", "/", "", "", "test-key") + var body api.AccountList + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &body) != nil || body.Items == nil || len(body.Items) != len(items) { + t.Fatal("account list shape") + } + } + for _, key := range []string{"", "test-key"} { + store := &accountStub{} + handler := mw.BearerAuth(key, AccountsRouter(store)) + for _, method := range []string{"GET", "POST", "DELETE"} { + w := accountRequest(handler, method, "/", "{}", "application/json", "") + want := 401 + if key == "" { + want = 503 + } + if w.Code != want || store.calls != 0 { + t.Fatal("unauthorized store access") + } + } + } +} + +func TestAccountErrorLogsCause(t *testing.T) { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + rec := httptest.NewRecorder() + accountError(rec, errors.New("private-error")) + if rec.Code != 500 || strings.Contains(rec.Body.String(), "private-error") { + t.Fatalf("response leaked or wrong status: %d %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(buf.String(), "private-error") { + t.Fatalf("cause not logged: %s", buf.String()) + } +} diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go new file mode 100644 index 00000000..cf4f0d6a --- /dev/null +++ b/internal/api/handlers/admin.go @@ -0,0 +1,99 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "encoding/json" + "errors" + "io" + "mime" + "net/http" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + mw "github.com/MeshCore-Beacon/beacon-server/internal/api/middleware" + "github.com/go-chi/chi/v5" +) + +// AdminRouter mounts operator endpoints. Its caller must wrap the +// entire subrouter with BearerAuth, including unknown paths and methods. +func AdminRouter(runtime *mw.RuntimeConfig, routes map[string]http.Handler) http.Handler { + r := chi.NewRouter() + r.Get("/config", getAdminConfig(runtime)) + r.Put("/config", updateAdminConfig(runtime)) + for path, handler := range routes { + r.Mount(path, handler) + } + return r +} + +// getAdminConfig godoc +// +// @Summary Inspect selected running configuration +// @Description Returns current CORS options, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Runtime origin updates are lost on restart. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Success 200 {object} api.AdminConfig +// @Failure 401 {object} map[string]APIError +// @Failure 503 {object} map[string]APIError +// @Router /admin/config [get] +func getAdminConfig(runtime *mw.RuntimeConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + respond(w, http.StatusOK, runtime.Snapshot()) + } +} + +// updateAdminConfig godoc +// @Summary Update runtime CORS origins +// @Description Replaces only cors.allowed_origins immediately. Updates are serialized; concurrent valid updates are applied one at a time. Already-running requests may finish with the previous policy. Nothing is persisted; restart reloads saved configuration. +// @Tags Admin +// @Accept json +// @Produce json +// @Security AdminKey +// @Param config body api.UpdateAdminConfigRequest true "1-32 ASCII HTTP(S) origins, at most 512 bytes each; one hostname wildcard is supported, or a sole *. Empty/null lists and unsupported fields are rejected. Body at most 16 KiB." +// @Success 200 {object} api.UpdateAdminConfigResponse +// @Failure 400,401,413,415,503 {object} map[string]APIError +// @Router /admin/config [put] +func updateAdminConfig(runtime *mw.RuntimeConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + media, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || media != "application/json" { + respondError(w, 415, "Content-Type must be application/json") + return + } + r.Body = http.MaxBytesReader(w, r.Body, 16*1024) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + var input api.UpdateAdminConfigRequest + err = decoder.Decode(&input) + if err == nil { + var extra any + err = decoder.Decode(&extra) + if errors.Is(err, io.EOF) { + err = nil + } else if err == nil { + err = errors.New("multiple JSON values") + } + } + if err != nil { + var sizeError *http.MaxBytesError + if errors.As(err, &sizeError) { + respondError(w, 413, "request body exceeds 16 KiB") + } else { + respondError(w, 400, "invalid configuration JSON") + } + return + } + if input.CORS == nil || input.CORS.AllowedOrigins == nil { + respondError(w, 400, "cors.allowed_origins is required") + return + } + config, err := runtime.UpdateOrigins(*input.CORS.AllowedOrigins) + if err != nil { + respondError(w, 400, mw.ErrInvalidOrigins.Error()) + return + } + respond(w, 200, api.UpdateAdminConfigResponse{Config: config, Persisted: false, RequiresRestart: false}) + } +} diff --git a/internal/api/handlers/backup.go b/internal/api/handlers/backup.go new file mode 100644 index 00000000..e9a85fd9 --- /dev/null +++ b/internal/api/handlers/backup.go @@ -0,0 +1,125 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strconv" + "sync/atomic" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/go-chi/chi/v5" +) + +// BackupRouter returns an operator-only subrouter with one bounded export or +// transfer at a time. Its caller must apply the admin authentication middleware. +// The options are startup-owned; the request selects no targets or paths. +// shutdown aborts an in-flight export so its staging files are removed. +func BackupRouter(opts backup.Options, shutdown context.Context) http.Handler { + sweepStaleStaging() + r := chi.NewRouter() + r.Get("/", backupDownload(opts, backup.Export, shutdown)) + return r +} + +// sweepStaleStaging removes staging dirs an earlier process left behind (SIGKILL, crash). +func sweepStaleStaging() { + matches, _ := filepath.Glob(filepath.Join(os.TempDir(), "beacon-download-*")) + cutoff := time.Now().Add(-2 * backup.DefaultTimeout) + for _, dir := range matches { + if info, err := os.Stat(dir); err == nil && info.IsDir() && info.ModTime().Before(cutoff) { + _ = os.RemoveAll(dir) + } + } +} + +// backupDownload godoc +// +// @Summary Download a private database and saved-config backup +// @Description Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint. +// @Tags Admin +// @Produce application/gzip +// @Security AdminKey +// @Success 200 {file} binary +// @Header 200 {string} Content-Disposition "attachment; filename=beacon-backup.tar.gz" +// @Failure 400 {object} map[string]APIError +// @Failure 401 {object} map[string]APIError +// @Failure 409 {object} map[string]APIError +// @Failure 500 {object} map[string]APIError +// @Failure 503 {object} map[string]APIError +// @Failure 504 {object} map[string]APIError +// @Failure 507 {object} map[string]APIError +// @Router /admin/backup [get] +func backupDownload(opts backup.Options, export func(context.Context, backup.Options) error, shutdown context.Context) http.HandlerFunc { + var busy atomic.Bool + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if opts.ConnectionService == "" { + respondError(w, 503, "backup download is not configured") + return + } + if r.URL.RawQuery != "" || r.ContentLength != 0 || len(r.TransferEncoding) != 0 { + respondError(w, 400, "backup download accepts no query parameters or body") + return + } + if !busy.CompareAndSwap(false, true) { + respondError(w, 409, "a backup download is already in progress") + return + } + defer busy.Store(false) + dir, err := os.MkdirTemp("", "beacon-download-") + if err != nil { + respondError(w, 500, "cannot stage backup") + return + } + defer os.RemoveAll(dir) + requestOpts := opts + requestOpts.OutputPath = filepath.Join(dir, "backup.tar.gz") + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + defer context.AfterFunc(shutdown, cancel)() + if err := export(ctx, requestOpts); err != nil { + if errors.Is(err, backup.ErrTooLarge) { + respondError(w, http.StatusInsufficientStorage, "backup exceeds the configured export size limit") + } else if errors.Is(err, context.DeadlineExceeded) { + respondError(w, 504, "backup export timed out") + } else if shutdown.Err() != nil { + respondError(w, http.StatusServiceUnavailable, "server is shutting down") + } else if r.Context().Err() == nil { + // Export discards pg_dump stderr and never returns connection settings. + slog.Error("backup export failed", "component", "backup", "error", err) + respondError(w, 500, "backup export failed; check private client configuration and capacity") + } + return + } + file, err := os.Open(requestOpts.OutputPath) + if err != nil { + respondError(w, 500, "cannot read completed backup") + return + } + defer file.Close() + info, err := file.Stat() + if err != nil { + respondError(w, 500, "cannot inspect completed backup") + return + } + // Keep slow downloads from retaining the only slot and private files + // indefinitely. The native server supports ResponseController deadlines. + controller := http.NewResponseController(w) + _ = controller.SetWriteDeadline(time.Now().Add(backup.DefaultTimeout)) + defer controller.SetWriteDeadline(time.Time{}) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Disposition", `attachment; filename="beacon-backup.tar.gz"`) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Content-Length", strconv.FormatInt(info.Size(), 10)) + _, _ = io.Copy(w, file) + } +} diff --git a/internal/api/handlers/backup_test.go b/internal/api/handlers/backup_test.go new file mode 100644 index 00000000..0ad1e9cb --- /dev/null +++ b/internal/api/handlers/backup_test.go @@ -0,0 +1,158 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" +) + +func TestBackupDownload(t *testing.T) { + opts := backup.Options{ConnectionService: "synthetic-private-service", MaxBytes: 1024, Timeout: time.Second, ConfigPath: "saved.yaml"} + var output string + export := func(ctx context.Context, got backup.Options) error { + if got.ConfigPath != opts.ConfigPath || got.ConnectionService != opts.ConnectionService || got.MaxBytes != opts.MaxBytes || got.Timeout != opts.Timeout { + t.Fatal("startup options changed") + } + output = got.OutputPath + return os.WriteFile(output, []byte("completed fixture"), 0600) + } + handler := backupDownload(opts, export, context.Background()) + for _, query := range []string{"?database=other", "?path=/private", "?command=anything"} { + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup"+query, nil)) + if w.Code != 400 || output != "" { + t.Fatal("caller-selected arguments accepted") + } + } + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", strings.NewReader("body"))) + if w.Code != 400 || output != "" { + t.Fatal("request body accepted") + } + w = httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", nil)) + if w.Code != 200 || w.Body.String() != "completed fixture" || w.Header().Get("Content-Type") != "application/gzip" || + w.Header().Get("Cache-Control") != "no-store" || w.Header().Get("Content-Length") != "17" || w.Header().Get("Content-Disposition") != `attachment; filename="beacon-backup.tar.gz"` { + t.Fatalf("unexpected download: %d %v", w.Code, w.Header()) + } + if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { + t.Fatal("download staging retained") + } + for _, failure := range []error{errors.New("PRIVATE_CANARY"), context.DeadlineExceeded, backup.ErrTooLarge} { + handler = backupDownload(opts, func(_ context.Context, got backup.Options) error { + output = got.OutputPath + return failure + }, context.Background()) + w = httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", nil)) + want := 500 + if errors.Is(failure, context.DeadlineExceeded) { + want = 504 + } + if errors.Is(failure, backup.ErrTooLarge) { + want = 507 + } + if w.Code != want || strings.Contains(w.Body.String(), "PRIVATE_CANARY") || w.Header().Get("Content-Disposition") != "" { + t.Fatal("failure leaked diagnostics or published a download") + } + if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { + t.Fatal("failed staging retained") + } + } + w = httptest.NewRecorder() + BackupRouter(backup.Options{}, context.Background()).ServeHTTP(w, httptest.NewRequest("GET", "/", nil)) + if w.Code != 503 { + t.Fatal("unconfigured download enabled") + } +} + +func TestBackupDownloadConcurrencyAndCancellation(t *testing.T) { + started, finished := make(chan struct{}), make(chan struct{}) + var output string + handler := backupDownload(backup.Options{ConnectionService: "fixture"}, func(ctx context.Context, opts backup.Options) error { + output = opts.OutputPath + close(started) + <-ctx.Done() + return ctx.Err() + }, context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + defer close(finished) + handler(httptest.NewRecorder(), httptest.NewRequest("GET", "/backup", nil).WithContext(ctx)) + }() + <-started + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", nil)) + if w.Code != http.StatusConflict { + t.Fatal("concurrent export started") + } + cancel() + <-finished + if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { + t.Fatal("cancelled staging retained") + } +} + +func TestBackupDownloadAbortsOnShutdown(t *testing.T) { + shutdown, stop := context.WithCancel(context.Background()) + stagedCh := make(chan string, 1) + handler := backupDownload(backup.Options{ConnectionService: "fixture"}, func(ctx context.Context, opts backup.Options) error { + stagedCh <- filepath.Dir(opts.OutputPath) + <-ctx.Done() + return ctx.Err() + }, shutdown) + rec := httptest.NewRecorder() + done := make(chan struct{}) + go func() { + handler.ServeHTTP(rec, httptest.NewRequest("GET", "/", nil)) + close(done) + }() + staged := <-stagedCh + stop() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("export did not stop on shutdown") + } + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status %d, want 503", rec.Code) + } + if _, err := os.Stat(staged); !os.IsNotExist(err) { + t.Fatalf("staging dir survived shutdown: %v", err) + } +} + +func TestBackupRouterSweepsStaleStaging(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + stale, err := os.MkdirTemp("", "beacon-download-") + if err != nil { + t.Fatal(err) + } + old := time.Now().Add(-3 * backup.DefaultTimeout) + if err := os.Chtimes(stale, old, old); err != nil { + t.Fatal(err) + } + fresh, err := os.MkdirTemp("", "beacon-download-") + if err != nil { + t.Fatal(err) + } + BackupRouter(backup.Options{}, context.Background()) + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatal("stale staging dir not swept") + } + if _, err := os.Stat(fresh); err != nil { + t.Fatal("fresh staging dir swept") + } +} diff --git a/internal/api/handlers/channels.go b/internal/api/handlers/channels.go index f394058b..a43489aa 100644 --- a/internal/api/handlers/channels.go +++ b/internal/api/handlers/channels.go @@ -5,6 +5,7 @@ package handlers import ( "encoding/hex" + "errors" "net/http" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) // ChannelsRouter mounts all /channels routes onto a subrouter. @@ -35,24 +37,22 @@ func ChannelsRouter(reader api.Reader) http.Handler { // @Tags Channels // @Produce json // @Param hash query string false "Single-byte channel hash (hex)" -// @Param iata query string false "Filter by IATA code" -// @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ" -// @Param cursor query int false "last_seen epoch ms of last item for pagination" -// @Param limit query int false "Max results (default 50)" -// @Success 200 {object} api.Page[api.ChannelSummary] +// @Param iata query string false "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels" +// @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata" +// @Param keyKnown query bool false "Only channels Beacon can (true) or cannot (false) decrypt; omit for all" +// @Param cursor query int false "last_seen epoch ms of last item for pagination; 0 starts from the beginning" +// @Param pageCursor query string false "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed." +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) +// @Success 200 {object} api.ChannelPage // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /channels [get] func listChannels(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - var limit int64 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = l + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } iatas := parseIATAs(r) var cursor int64 @@ -64,6 +64,18 @@ func listChannels(reader api.Reader) http.HandlerFunc { } cursor = c } + var pageCursor *api.ChannelCursor + if raw := r.URL.Query().Get("pageCursor"); raw != "" { + if cursor > 0 { + respondError(w, http.StatusBadRequest, "pageCursor and a positive cursor cannot be combined") + return + } + pageCursor, err = api.ParseChannelCursor(raw) + if err != nil { + respondError(w, http.StatusBadRequest, "invalid pageCursor") + return + } + } var hashHex []byte if hash := strings.ToLower(r.URL.Query().Get("hash")); hash != "" { h, decodeErr := hex.DecodeString(hash) @@ -77,7 +89,19 @@ func listChannels(reader api.Reader) http.HandlerFunc { } hashHex = h } - channels, err := reader.ListChannels(r.Context(), int32(limit), hashHex, iatas, cursor) + var keyKnown *bool + if q := r.URL.Query(); q.Has("keyKnown") { + switch q.Get("keyKnown") { + case "true": + keyKnown = new(true) + case "false": + keyKnown = new(false) + default: + respondError(w, http.StatusBadRequest, "keyKnown must be true or false") + return + } + } + channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, keyKnown, cursor, pageCursor) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return @@ -95,6 +119,7 @@ func listChannels(reader api.Reader) http.HandlerFunc { // @Success 200 {object} api.Channel // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /channels/{channelID} [get] func getChannel(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -108,9 +133,13 @@ func getChannel(reader api.Reader) http.HandlerFunc { id = i } channel, err := reader.GetChannel(r.Context(), int32(id)) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && channel == nil: respondError(w, http.StatusNotFound, "channel not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, channel) } @@ -128,7 +157,7 @@ func getChannel(reader api.Reader) http.HandlerFunc { // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param scope query string false "Filter by transport scope name e.g. %23bc (URL-encoded #bc)" // @Param cursor query int false "Message ID of last item for pagination (results ordered newest first)" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} object // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -144,14 +173,10 @@ func listChannelMessages(reader api.Reader) http.HandlerFunc { } id = i } - var limit int64 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = l + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var since time.Time if sinceParam := r.URL.Query().Get("since"); sinceParam != "" { @@ -174,7 +199,7 @@ func listChannelMessages(reader api.Reader) http.HandlerFunc { } scope := r.URL.Query().Get("scope") chanID := int32(id) - messages, err := reader.ListChannelMessages(r.Context(), &chanID, since, int32(limit), iatas, scope, cursor) + messages, err := reader.ListChannelMessages(r.Context(), &chanID, since, limit, iatas, scope, cursor) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/channels_test.go b/internal/api/handlers/channels_test.go index c014dc4e..00bc5d79 100644 --- a/internal/api/handlers/channels_test.go +++ b/internal/api/handlers/channels_test.go @@ -8,7 +8,9 @@ import ( "net/http" "net/http/httptest" "reflect" + "strings" "testing" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" @@ -36,6 +38,34 @@ func TestListChannels_InvalidCursor(t *testing.T) { } } +func TestListChannels_PageCursor(t *testing.T) { + called := false + r := ChannelsRouter(stubReader{listChannels: func(_ context.Context, limit int32, hash []byte, iatas []string, _ *bool, legacy int64, cursor *api.ChannelCursor) (api.ChannelPage, error) { + called = true + if limit != 2 || legacy != 0 || cursor == nil || cursor.ID != 9 || !cursor.LastSeen.Equal(time.UnixMicro(1700000000000123)) || + !reflect.DeepEqual(hash, []byte{0xaa}) || !reflect.DeepEqual(iatas, []string{"YOW"}) { + t.Error("page cursor or filters did not reach the reader intact") + } + return api.ChannelPage{}, nil + }}) + for _, legacy := range []string{"", "&cursor=0"} { + called = false + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/?pageCursor=v1:1700000000000123:9&limit=2&hash=AA&iata=yow"+legacy, nil)) + if w.Code != http.StatusOK || !called { + t.Fatalf("legacy %q: HTTP %d, reader called=%v", legacy, w.Code, called) + } + } + for _, query := range []string{"pageCursor=bad", "cursor=1&pageCursor=v1:1700000000000123:9", "cursor=bad&pageCursor=v1:1700000000000123:9"} { + called = false + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/?"+query, nil)) + if w.Code != http.StatusBadRequest || called { + t.Errorf("HTTP %d, invalid request reached reader=%v", w.Code, called) + } + } +} + func TestListChannels_InvalidHash(t *testing.T) { r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{})) @@ -116,8 +146,8 @@ func TestListChannelMessages_InvalidCursor(t *testing.T) { func TestListChannels_OK(t *testing.T) { r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{ - listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ int64) (api.Page[api.ChannelSummary], error) { - return api.Page[api.ChannelSummary]{Items: []api.ChannelSummary{{ID: 1, ChannelHash: "ab"}}}, nil + listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + return api.ChannelPage{Page: api.Page[api.ChannelSummary]{Items: []api.ChannelSummary{{ID: 1, ChannelHash: "ab"}}}}, nil }, })) req := httptest.NewRequest(http.MethodGet, "/channels", nil) @@ -143,9 +173,9 @@ func TestListChannels_IATAParsing(t *testing.T) { var got []string r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{ - listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ int64) (api.Page[api.ChannelSummary], error) { + listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { got = iatas - return api.Page[api.ChannelSummary]{}, nil + return api.ChannelPage{}, nil }, })) req := httptest.NewRequest(http.MethodGet, "/channels"+tc.query, nil) @@ -161,6 +191,54 @@ func TestListChannels_IATAParsing(t *testing.T) { } } +func TestListChannels_KeyKnownParsing(t *testing.T) { + yes, no := true, false + cases := []struct { + name string + query string + code int + want *bool + }{ + {"true", "?keyKnown=true", http.StatusOK, &yes}, + {"false", "?keyKnown=false", http.StatusOK, &no}, + {"absent", "", http.StatusOK, nil}, + {"uppercase", "?keyKnown=TRUE", http.StatusBadRequest, nil}, + {"numeric", "?keyKnown=1", http.StatusBadRequest, nil}, + {"empty", "?keyKnown=", http.StatusBadRequest, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var got *bool + called := false + r := chi.NewRouter() + r.Get("/channels", listChannels(stubReader{ + listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, keyKnown *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + called = true + got = keyKnown + return api.ChannelPage{}, nil + }, + })) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/channels"+tc.query, nil)) + if w.Code != tc.code { + t.Fatalf("expected %d, got %d", tc.code, w.Code) + } + if tc.code != http.StatusOK { + if called { + t.Error("reader called for invalid keyKnown") + } + if !strings.Contains(w.Body.String(), "keyKnown must be true or false") { + t.Errorf("unexpected error body %s", w.Body.String()) + } + return + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("expected keyKnown %v, got %v", tc.want, got) + } + }) + } +} + func TestGetChannel_OK(t *testing.T) { r := chi.NewRouter() r.Get("/channels/{channelID}", getChannel(stubReader{ diff --git a/internal/api/handlers/iatas.go b/internal/api/handlers/iatas.go index 8524611c..19d2375b 100644 --- a/internal/api/handlers/iatas.go +++ b/internal/api/handlers/iatas.go @@ -4,10 +4,13 @@ package handlers import ( + "errors" "net/http" + "strings" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) // IATAsRouter mounts all /iatas routes onto a subrouter. @@ -23,21 +26,39 @@ func IATAsRouter(reader api.Reader) http.Handler { return r } +// pathIATA uppercases the {iata} param and accepts only 3 ASCII letters, which also keeps it +// from reaching into other cache keys. +func pathIATA(r *http.Request) (string, bool) { + iata := strings.ToUpper(chi.URLParam(r, "iata")) + if len(iata) != 3 { + return "", false + } + for i := range len(iata) { + if iata[i] < 'A' || iata[i] > 'Z' { + return "", false + } + } + return iata, true +} + // listIATAs godoc // // @Summary List all IATA codes // @Tags IATAs // @Produce json // @Success 200 {array} api.IATA -// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /iatas [get] func listIATAs(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { iatas, err := reader.ListIATAs(r.Context()) if err != nil { - respondError(w, http.StatusNotFound, "no IATAs found") + respondError(w, http.StatusInternalServerError, "internal server error") return } + if iatas == nil { + iatas = []api.IATA{} + } respond(w, http.StatusOK, iatas) } } @@ -50,14 +71,23 @@ func listIATAs(reader api.Reader) http.HandlerFunc { // @Param iata path string true "3-letter IATA code" // @Success 200 {object} api.IATA // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /iatas/{iata} [get] func getIATA(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - iata := chi.URLParam(r, "iata") + iata, ok := pathIATA(r) + if !ok { + respondError(w, http.StatusNotFound, "IATA not found") + return + } result, err := reader.GetIATA(r.Context(), iata) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && result == nil: respondError(w, http.StatusNotFound, "IATA not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, result) } @@ -72,14 +102,23 @@ func getIATA(reader api.Reader) http.HandlerFunc { // @Success 200 {object} object "GeoJSON Feature (Polygon or MultiPolygon geometry, with bbox)" // @Success 204 "IATA exists but has no border configured" // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /iatas/{iata}/border [get] func getIATABorder(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - iata := chi.URLParam(r, "iata") + iata, ok := pathIATA(r) + if !ok { + respondError(w, http.StatusNotFound, "IATA not found") + return + } border, err := reader.GetIATABorder(r.Context(), iata) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows): respondError(w, http.StatusNotFound, "IATA not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } // A cache round-trip re-encodes a nil json.RawMessage as the literal 4-byte JSON // "null" rather than leaving it empty, so both must be treated as "no border set". diff --git a/internal/api/handlers/iatas_test.go b/internal/api/handlers/iatas_test.go index 5c7ff31d..f6e26dff 100644 --- a/internal/api/handlers/iatas_test.go +++ b/internal/api/handlers/iatas_test.go @@ -6,13 +6,13 @@ package handlers import ( "context" "encoding/json" - "errors" "net/http" "net/http/httptest" "testing" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) func TestListIATAs_OK(t *testing.T) { @@ -106,11 +106,49 @@ func TestGetIATABorder_NoContentOnCacheRoundTrippedNull(t *testing.T) { } } +func TestIATAPathValidation(t *testing.T) { + var got []string + reader := stubReader{ + getIATA: func(_ context.Context, iata string) (*api.IATA, error) { + got = append(got, iata) + return &api.IATA{IATA: iata}, nil + }, + getIATABorder: func(_ context.Context, iata string) (json.RawMessage, error) { + got = append(got, iata) + return json.RawMessage(`{"type":"Feature"}`), nil + }, + } + r := chi.NewRouter() + r.Mount("/iatas", IATAsRouter(reader)) + for _, tc := range []struct { + path string + want int + }{ + {"/iatas/border:YOW", http.StatusNotFound}, + {"/iatas/border:YOW/border", http.StatusNotFound}, + {"/iatas/YO", http.StatusNotFound}, + {"/iatas/YOWW", http.StatusNotFound}, + {"/iatas/Y%C3%96W", http.StatusNotFound}, + {"/iatas/Y1W/border", http.StatusNotFound}, + {"/iatas/yow", http.StatusOK}, + {"/iatas/yow/border", http.StatusOK}, + } { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) + if w.Code != tc.want { + t.Errorf("%s: status = %d, want %d", tc.path, w.Code, tc.want) + } + } + if len(got) != 2 || got[0] != "YOW" || got[1] != "YOW" { + t.Errorf("reader saw %q, want only uppercased valid codes", got) + } +} + func TestGetIATABorder_NotFound(t *testing.T) { r := chi.NewRouter() r.Get("/iatas/{iata}/border", getIATABorder(stubReader{ getIATABorder: func(_ context.Context, _ string) (json.RawMessage, error) { - return nil, errors.New("not found") + return nil, pgx.ErrNoRows }, })) req := httptest.NewRequest(http.MethodGet, "/iatas/ZZZ/border", nil) diff --git a/internal/api/handlers/limits.go b/internal/api/handlers/limits.go new file mode 100644 index 00000000..80bda95b --- /dev/null +++ b/internal/api/handlers/limits.go @@ -0,0 +1,36 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "errors" + "log/slog" + "net/http" + "strconv" + "time" +) + +const maxListLimit int32 = 200 + +const clockSkewTolerance = 5 * time.Minute // client clocks run ahead; accept a small future 'until' + +// parseLimit caps the requested result size while retaining each endpoint's default. +func parseLimit(r *http.Request, defaultLimit int32) (int32, error) { + value := r.URL.Query().Get("limit") + if value == "" { + return defaultLimit, nil + } + limit, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return 0, errors.New("limit must be an integer") + } + if limit <= 0 { + return 0, errors.New("limit must be positive") + } + if limit > int64(maxListLimit) { + slog.Info("api: clamped list limit", "component", "api", "requested_limit", limit, "limit", maxListLimit) + return maxListLimit, nil + } + return int32(limit), nil +} diff --git a/internal/api/handlers/limits_logging_test.go b/internal/api/handlers/limits_logging_test.go new file mode 100644 index 00000000..54bfa147 --- /dev/null +++ b/internal/api/handlers/limits_logging_test.go @@ -0,0 +1,37 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "bytes" + "encoding/json" + "log" + "log/slog" + "net/http/httptest" + "strings" + "testing" +) + +func TestLimitLogOmitsRequestValues(t *testing.T) { + var out bytes.Buffer + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, nil))) + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) + req := httptest.NewRequest("GET", "/private-path%0Aforged?limit=1000", nil) + req.RemoteAddr = "private-peer" + limit, err := parseLimit(req, 50) + if err != nil || limit != 200 { + t.Fatalf("limit=%d error=%v", limit, err) + } + if strings.Contains(out.String(), "private-") || strings.Contains(out.String(), "forged") { + t.Fatal("limit log retained request identity") + } + var record map[string]any + if err := json.Unmarshal(out.Bytes(), &record); err != nil { + t.Fatal(err) + } + if record["requested_limit"] != float64(1000) || record["limit"] != float64(200) || record["component"] != "api" { + t.Fatalf("clamp diagnostics missing: %v", record) + } +} diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go new file mode 100644 index 00000000..e904e5c3 --- /dev/null +++ b/internal/api/handlers/limits_test.go @@ -0,0 +1,173 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" + "github.com/google/uuid" +) + +func TestListLimits(t *testing.T) { + var got int32 + var calls int + reader := stubReader{ + listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { + calls++ + got = limit + return api.ChannelPage{}, nil + }, + listChannelMessages: func(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) { + calls++ + got = limit + return api.Page[api.ChannelMessage]{}, nil + }, + listChannelMessagesByHash: func(ctx context.Context, hash []byte, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) { + calls++ + got = limit + return api.Page[api.ChannelMessage]{}, nil + }, + listMessagesAfterID: func(ctx context.Context, afterID int64, iatas []string, scope string, limit int32) ([]api.ChannelMessage, error) { + calls++ + got = limit + return nil, nil + }, + listObservers: func(ctx context.Context, iatas []string, observerType, broker, status, name, scope string, cursor int64, limit int32) (api.Page[api.ObserverSummary], error) { + calls++ + got = limit + return api.Page[api.ObserverSummary]{}, nil + }, + listObserverAdverts: func(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) { + calls++ + got = limit + return api.Page[api.AdvertObservation]{}, nil + }, + listNodes: func(ctx context.Context, nodeType int16, iatas []string, supportsMultibytePaths, supportsMultibyteTraces *bool, pubkey []byte, pubkeyPrefix, name, scope string, cursor int64, limit int32, includeNeighbors bool) (api.Page[api.NodeSummary], error) { + calls++ + got = limit + return api.Page[api.NodeSummary]{}, nil + }, + listNodeObservations: func(ctx context.Context, nodeID uuid.UUID, cursor int64, limit int32) (api.Page[api.PacketObservationSummary], error) { + calls++ + got = limit + return api.Page[api.PacketObservationSummary]{}, nil + }, + listPackets: func(ctx context.Context, payloadTypes, routeTypes []int16, iatas []string, scopes []string, since, until time.Time, cursor int64, limit int32) (api.Page[api.PacketSummary], error) { + calls++ + got = limit + return api.Page[api.PacketSummary]{}, nil + }, + listPacketsAfterID: func(ctx context.Context, afterObservationID int64, payloadType, routeType int16, iatas []string, scope string, limit int32) ([]api.PacketSummary, error) { + calls++ + got = limit + return nil, nil + }, + getStatsTopNodes: func(ctx context.Context, iatas []string, _ time.Time, limit int32) ([]api.TopNode, error) { + calls++ + got = limit + return nil, nil + }, + getStatsTopObservers: func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { + calls++ + got = limit + return nil, nil + }, + getStatsTopAdvertisers: func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { + calls++ + got = limit + return nil, nil + }, + getStatsClockDrift: func(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { + calls++ + got = limit + return nil, nil + }, + getStatsTopTalkers: func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { + calls++ + got = limit + return nil, nil + }, + listTraceTags: func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { + calls++ + got = limit + return nil, nil + }, + listKnownRoutes: func(ctx context.Context, iata string, hopCount int32, cursor time.Time, _ int64, limit int32) ([]api.KnownRoute, error) { + calls++ + got = limit + return nil, nil + }, + } + router := chi.NewRouter() + router.Mount("/nodes", NodesRouter(reader)) + router.Mount("/observers", ObserversRouter(reader)) + router.Mount("/packets", PacketsRouter(reader)) + router.Mount("/channels", ChannelsRouter(reader)) + router.Mount("/messages", MessagesRouter(reader)) + router.Mount("/routes", RoutesRouter(reader)) + router.Mount("/traces", TracesRouter(reader)) + router.Mount("/stats", StatsRouter(reader, StatsOptions{})) + for _, endpoint := range []struct { + path string + defaultLimit int32 + }{ + {"/nodes", 50}, + {"/nodes/00000000-0000-0000-0000-000000000001/observations", 50}, + {"/observers", 50}, + {"/observers/00000000-0000-0000-0000-000000000001/adverts", 50}, + {"/packets", 50}, + {"/packets/backfill?afterObservationId=1", 100}, + {"/channels", 50}, + {"/channels/1/messages", 50}, + {"/messages", 50}, + {"/messages?channelHash=11", 50}, + {"/messages/backfill?afterId=1", 100}, + {"/routes", 50}, + {"/traces", 50}, + {"/stats/top-nodes", 10}, + {"/stats/top-observers", 10}, + {"/stats/top-advertisers", 10}, + {"/stats/clock-drift", 10}, + {"/stats/top-talkers", 10}, + } { + t.Run(endpoint.path, func(t *testing.T) { + for _, tc := range []struct { + value string + want int32 + status int + }{ + {"", endpoint.defaultLimit, 200}, {"1", 1, 200}, {"200", 200, 200}, + {"201", 200, 200}, {"1000000", 200, 200}, {"2147483647", 200, 200}, + {"0", 0, 400}, {"-1", 0, 400}, {"bad", 0, 400}, {"1.5", 0, 400}, + {"2147483648", 0, 400}, + } { + t.Run(tc.value, func(t *testing.T) { + got, calls = 0, 0 + req := httptest.NewRequest(http.MethodGet, endpoint.path, nil) + query := req.URL.Query() + query.Set("limit", tc.value) + req.URL.RawQuery = query.Encode() + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != tc.status { + t.Fatalf("status %d, want %d: %s", rec.Code, tc.status, rec.Body.String()) + } + if tc.status == http.StatusBadRequest { + if calls != 0 { + t.Fatal("invalid limit reached the reader") + } + } else if calls != 1 || got != tc.want { + t.Fatalf("reader calls=%d limit=%d, want 1 and %d", calls, got, tc.want) + } + }) + } + }) + } +} diff --git a/internal/api/handlers/lookup_errors_test.go b/internal/api/handlers/lookup_errors_test.go new file mode 100644 index 00000000..4c2431d5 --- /dev/null +++ b/internal/api/handlers/lookup_errors_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// Lookups answer 404 only for a missing row; any other failure is a retryable 500. +func TestLookupErrors(t *testing.T) { + id := uuid.NewString() + for _, tc := range []struct { + name string + router func(error) http.Handler + path string + missing int // status for pgx.ErrNoRows + }{ + {"region", func(err error) http.Handler { + return RegionsRouter(stubReader{getRegion: func(context.Context, int32) (*api.Region, error) { return nil, err }}) + }, "/1", http.StatusNotFound}, + {"regions", func(err error) http.Handler { + return RegionsRouter(stubReader{listRegions: func(context.Context) ([]api.RegionSummary, error) { return nil, err }}) + }, "/", http.StatusInternalServerError}, + {"iata", func(err error) http.Handler { + return IATAsRouter(stubReader{getIATA: func(context.Context, string) (*api.IATA, error) { return nil, err }}) + }, "/YOW", http.StatusNotFound}, + {"iata border", func(err error) http.Handler { + return IATAsRouter(stubReader{getIATABorder: func(context.Context, string) (json.RawMessage, error) { return nil, err }}) + }, "/YOW/border", http.StatusNotFound}, + {"iatas", func(err error) http.Handler { + return IATAsRouter(stubReader{listIATAs: func(context.Context) ([]api.IATA, error) { return nil, err }}) + }, "/", http.StatusInternalServerError}, + {"channel", func(err error) http.Handler { + return ChannelsRouter(stubReader{getChannel: func(context.Context, int32) (*api.Channel, error) { return nil, err }}) + }, "/7", http.StatusNotFound}, + {"observer", func(err error) http.Handler { + return ObserversRouter(stubReader{getObserver: func(context.Context, uuid.UUID) (*api.Observer, error) { return nil, err }}) + }, "/" + id, http.StatusNotFound}, + {"node", func(err error) http.Handler { + return NodesRouter(stubReader{getNode: func(context.Context, uuid.UUID) (*api.Node, error) { return nil, err }}) + }, "/" + id, http.StatusNotFound}, + {"packet", func(err error) http.Handler { + return PacketsRouter(stubReader{getPacket: func(context.Context, []byte) (*api.Packet, error) { return nil, err }}) + }, "/abcd", http.StatusNotFound}, + } { + for _, c := range []struct { + err error + want int + }{ + {pgx.ErrNoRows, tc.missing}, + {context.DeadlineExceeded, http.StatusInternalServerError}, + } { + t.Run(tc.name+"/"+c.err.Error(), func(t *testing.T) { + w := httptest.NewRecorder() + tc.router(c.err).ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) + if w.Code != c.want { + t.Fatalf("status = %d, want %d", w.Code, c.want) + } + var body map[string]APIError + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || body["error"].Message == "" { + t.Fatalf("not a JSON error: %s", w.Body.String()) + } + }) + } + } +} + +// An unknown region filter is the caller's mistake; a failed lookup is retryable. +func TestRegionFilterErrors(t *testing.T) { + for _, tc := range []struct { + err error + want int + }{ + {pgx.ErrNoRows, http.StatusBadRequest}, + {context.DeadlineExceeded, http.StatusInternalServerError}, + } { + reader := stubReader{getRegionBySlug: func(context.Context, string) (*api.Region, error) { return nil, tc.err }} + for name, h := range map[string]http.Handler{ + "packets": PacketsRouter(reader), + "nodes": NodesRouter(reader), + "traces": TracesRouter(reader), + } { + t.Run(name+"/"+tc.err.Error(), func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/?region=nowhere", nil)) + if w.Code != tc.want { + t.Fatalf("status = %d, want %d", w.Code, tc.want) + } + }) + } + } +} + +func TestEmptyListsAreArrays(t *testing.T) { + for name, h := range map[string]http.Handler{ + "regions": RegionsRouter(stubReader{}), + "iatas": IATAsRouter(stubReader{}), + } { + t.Run(name, func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/", nil)) + if w.Code != http.StatusOK || w.Body.String() != "[]\n" { + t.Fatalf("got %d %q, want 200 []", w.Code, w.Body.String()) + } + }) + } +} diff --git a/internal/api/handlers/messages.go b/internal/api/handlers/messages.go index da5f8eee..2bd627bf 100644 --- a/internal/api/handlers/messages.go +++ b/internal/api/handlers/messages.go @@ -37,7 +37,7 @@ func MessagesRouter(reader api.Reader) http.Handler { // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param scope query string false "Filter by transport scope name e.g. %23bc (URL-encoded #bc)" // @Param cursor query int false "Message ID of last item for pagination (results ordered newest first)" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} object // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -60,14 +60,10 @@ func listMessages(reader api.Reader) http.HandlerFunc { } id = i } - var limit int64 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = l + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var since time.Time if sinceParam := r.URL.Query().Get("since"); sinceParam != "" { @@ -90,7 +86,6 @@ func listMessages(reader api.Reader) http.HandlerFunc { iatas := parseIATAs(r) scope := r.URL.Query().Get("scope") var messages api.Page[api.ChannelMessage] - var err error if channelHashParam != "" { hashHex, decodeErr := hex.DecodeString(channelHashParam) if decodeErr != nil { @@ -101,12 +96,12 @@ func listMessages(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "channel hash must be a single hex byte") return } - messages, err = reader.ListChannelMessagesByHash(r.Context(), hashHex, since, int32(limit), iatas, scope, cursor) + messages, err = reader.ListChannelMessagesByHash(r.Context(), hashHex, since, limit, iatas, scope, cursor) } else if channelIDParam != "" { chanID := int32(id) - messages, err = reader.ListChannelMessages(r.Context(), &chanID, since, int32(limit), iatas, scope, cursor) + messages, err = reader.ListChannelMessages(r.Context(), &chanID, since, limit, iatas, scope, cursor) } else { - messages, err = reader.ListChannelMessages(r.Context(), nil, since, int32(limit), iatas, scope, cursor) + messages, err = reader.ListChannelMessages(r.Context(), nil, since, limit, iatas, scope, cursor) } if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") @@ -126,7 +121,7 @@ func listMessages(reader api.Reader) http.HandlerFunc { // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" // @Param scope query string false "Filter by transport scope name" -// @Param limit query int false "Max results (default 100)" +// @Param limit query int false "Max results (default 100); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} []api.ChannelMessage // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -143,20 +138,16 @@ func listMessagesBackfill(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "afterId must be an integer") return } - var limit int32 = 100 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 100) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } iatas := parseIATAs(r) if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 65f4329a..1d2f09aa 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -5,6 +5,7 @@ package handlers import ( "encoding/hex" + "errors" "net/http" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" "github.com/google/uuid" + "github.com/jackc/pgx/v5" ) // NodesRouter mounts all /nodes routes onto a subrouter. @@ -49,7 +51,7 @@ func NodesRouter(reader api.Reader) http.Handler { // @Param supportsMultibyteTraces query bool false "Filter by multibyte trace support (true/false); omit for no filter" // @Param neighbors query bool false "Include each node's known neighbor IDs (neighborIds field). Bare ?neighbors or ?neighbors=true enables it; omit/false for none" // @Param cursor query int false "last_seen epoch ms of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} api.Page[api.NodeSummary] // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -67,14 +69,10 @@ func listNodes(reader api.Reader) http.HandlerFunc { } else if typeName := r.URL.Query().Get("typeName"); typeName != "" { nodeType = api.NodeTypeFromString(typeName) } - var limit int32 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var cursor int64 if cursorParam := r.URL.Query().Get("cursor"); cursorParam != "" { @@ -103,7 +101,7 @@ func listNodes(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -160,6 +158,7 @@ func listNodes(reader api.Reader) http.HandlerFunc { // @Success 200 {object} api.Node // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /nodes/{nodeId} [get] func getNode(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -169,9 +168,13 @@ func getNode(reader api.Reader) http.HandlerFunc { return } node, err := reader.GetNode(r.Context(), nodeID) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && node == nil: respondError(w, http.StatusNotFound, "node not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, node) } @@ -184,7 +187,7 @@ func getNode(reader api.Reader) http.HandlerFunc { // @Produce json // @Param nodeId path string true "Node UUID" // @Param cursor query int false "Observation ID of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} api.Page[api.PacketObservationSummary] // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -205,14 +208,10 @@ func listNodeObservations(reader api.Reader) http.HandlerFunc { } cursor = c } - var limit int32 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } observations, err := reader.ListNodeObservations(r.Context(), nodeID, cursor, limit) if err != nil { diff --git a/internal/api/handlers/observer_comparison.go b/internal/api/handlers/observer_comparison.go new file mode 100644 index 00000000..ff03db42 --- /dev/null +++ b/internal/api/handlers/observer_comparison.go @@ -0,0 +1,92 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strconv" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// getObserverComparison godoc +// +// @Summary Compare flood packets reported by two observers +// @Description Counts distinct retained packet hashes with route type 0 or 1, using reception time in [since, until). onlyA, onlyB and both partition totalPackets (the union). Repeated receptions count once. These are reported receptions, not a measure of radio packet loss or continuous observer coverage. +// @Tags Stats +// @Produce json +// @Param observerA query string true "First observer UUID" +// @Param observerB query string true "Second, distinct observer UUID" +// @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" +// @Param until query int true "Exclusive end, epoch milliseconds; later than since" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.ObserverComparison +// @Failure 400 {object} handlers.APIError +// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/observer-comparison [get] +func getObserverComparison(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + for _, key := range []string{"observerA", "observerB", "since", "until"} { + if len(q[key]) != 1 || q.Get(key) == "" { + respondError(w, http.StatusBadRequest, key+" must be supplied exactly once") + return + } + } + a, errA := uuid.Parse(q.Get("observerA")) + b, errB := uuid.Parse(q.Get("observerB")) + if errA != nil || errB != nil || a == uuid.Nil || b == uuid.Nil || a == b { + respondError(w, http.StatusBadRequest, "observerA and observerB must be distinct, nonzero UUIDs") + return + } + since, errSince := strconv.ParseInt(q.Get("since"), 10, 64) + until, errUntil := strconv.ParseInt(q.Get("until"), 10, 64) + if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 { + respondError(w, http.StatusBadRequest, "since and until must be epoch milliseconds between 0 and 253402300799999, with until later than since") + return + } + // Bound database work even for arbitrary historical windows. Cancellation + // also propagates when a user changes filters or leaves the page. + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondRegionError(w, err) + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + // An empty selected region must not silently become a global query. + iatas = []string{""} + } + } + comparison, err := reader.GetObserverComparison(ctx, a, b, time.UnixMilli(since), time.UnixMilli(until), iatas) + switch { + case errors.Is(err, pgx.ErrNoRows): + respondError(w, http.StatusNotFound, "observer not found") + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "comparison timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("observer comparison failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, comparison) + } + } +} diff --git a/internal/api/handlers/observer_comparison_test.go b/internal/api/handlers/observer_comparison_test.go new file mode 100644 index 00000000..78f7b539 --- /dev/null +++ b/internal/api/handlers/observer_comparison_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestObserverComparisonRejectsInvalidQuery(t *testing.T) { + for _, query := range []string{ + "", + "observerA=bad&observerB=bad&since=0&until=1", + "observerA=11111111-1111-1111-1111-111111111111&observerB=11111111-1111-1111-1111-111111111111&since=0&until=1", + "observerA=11111111-1111-1111-1111-111111111111&observerB=22222222-2222-2222-2222-222222222222&since=2&until=1", + "observerA=11111111-1111-1111-1111-111111111111&observerB=22222222-2222-2222-2222-222222222222&since=no&until=1", + "observerA=11111111-1111-1111-1111-111111111111&observerB=22222222-2222-2222-2222-222222222222&since=-1&until=1", + } { + t.Run(query, func(t *testing.T) { + w := httptest.NewRecorder() + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) + if w.Code != 400 { + t.Fatalf("status %d; wanted validation error", w.Code) + } + }) + } +} + +func TestObserverComparisonHTTP(t *testing.T) { + a := uuid.MustParse("11111111-1111-1111-1111-111111111111") + b := uuid.MustParse("22222222-2222-2222-2222-222222222222") + query := "observerA=" + a.String() + "&observerB=" + b.String() + "&since=0&until=1000" + for _, suffix := range []string{"&since=2", "&observerA=" + b.String(), "&until=253402300800000", "&observerB=00000000-0000-0000-0000-000000000000"} { + w := httptest.NewRecorder() + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) + if w.Code != 400 { + t.Fatalf("invalid query returned %d", w.Code) + } + } + for _, tc := range []struct { + name, suffix string + err error + status int + iatas []string + }{ + {"global", "", nil, 200, nil}, + {"IATA", "&iatas=yvr,yyj", nil, 200, []string{"YVR", "YYJ"}}, + {"region", "®ion=coast", nil, 200, []string{"YVR"}}, + {"empty region", "®ion=empty", nil, 200, []string{""}}, + {"unknown observer", "", pgx.ErrNoRows, 404, nil}, + {"timeout", "", context.DeadlineExceeded, 503, nil}, + {"database failure", "", errors.New("private database detail"), 500, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := stubReader{ + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }, + getObserverComparison: func(ctx context.Context, first, second uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { + t.Fatal("query has no bounded deadline") + } + if first != a || second != b || since.UnixMilli() != 0 || until.UnixMilli() != 1000 || !reflect.DeepEqual(iatas, tc.iatas) { + t.Fatalf("unexpected arguments: %s %s %v %v %v", first, second, since, until, iatas) + } + return &api.ObserverComparison{ObserverA: a, ObserverB: b, Until: 1000, TotalPackets: 4, OnlyA: 1, OnlyB: 1, Both: 2}, tc.err + }, + } + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) + if w.Code != tc.status { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + if tc.status == 200 { + var got api.ObserverComparison + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.TotalPackets != 4 || got.ObserverA != a || got.Both != 2 { + t.Fatalf("response = %+v", got) + } + } + }) + } +} diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 34ca2c61..089c6e4b 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -4,6 +4,7 @@ package handlers import ( + "errors" "net/http" "strconv" "time" @@ -11,6 +12,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" "github.com/google/uuid" + "github.com/jackc/pgx/v5" ) // ObserversRouter mounts all /observers routes onto a subrouter. @@ -18,6 +20,7 @@ import ( // GET /observers → listObservers // GET /observers/{observerId} → getObserver // GET /observers/{observerId}/telemetry → getObserverTelemetry +// GET /observers/{observerId}/activity → getObserverActivity // GET /observers/{observerId}/adverts → listObserverAdverts func ObserversRouter(reader api.Reader) http.Handler { r := chi.NewRouter() @@ -26,6 +29,7 @@ func ObserversRouter(reader api.Reader) http.Handler { r.Get("/", getObserver(reader)) r.Get("/adverts", listObserverAdverts(reader)) r.Get("/telemetry", getObserverTelemetry(reader)) + r.Get("/activity", getObserverActivity(reader)) }) return r } @@ -45,7 +49,7 @@ func ObserversRouter(reader api.Reader) http.Handler { // @Param name query string false "Partial case-insensitive display name match" // @Param scope query string false "Filter by transport scope name e.g. %23bc (URL-encoded #bc)" // @Param cursor query int false "last_seen epoch ms of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} api.Page[api.ObserverSummary] // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -66,20 +70,16 @@ func listObservers(reader api.Reader) http.HandlerFunc { } cursor = c } - var limit int32 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } iatas := parseIATAs(r) if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -102,6 +102,7 @@ func listObservers(reader api.Reader) http.HandlerFunc { // @Success 200 {object} api.Observer // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /observers/{observerId} [get] func getObserver(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -112,9 +113,13 @@ func getObserver(reader api.Reader) http.HandlerFunc { return } obs, err := reader.GetObserver(r.Context(), id) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && obs == nil: respondError(w, http.StatusNotFound, "observer not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, obs) } @@ -127,7 +132,7 @@ func getObserver(reader api.Reader) http.HandlerFunc { // @Produce json // @Param observerId path string true "Observer UUID" // @Param cursor query int false "Observation ID of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} api.Page[api.AdvertObservation] // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -148,14 +153,10 @@ func listObserverAdverts(reader api.Reader) http.HandlerFunc { } cursor = c } - var limit int32 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } adverts, err := reader.ListObserverAdverts(r.Context(), observerID, cursor, limit) if err != nil { @@ -226,7 +227,8 @@ func getObserverTelemetry(reader api.Reader) http.HandlerFunc { if bucketHours == 0 { telemetry, err = reader.GetObserverTelemetry(r.Context(), observerID, since, until, afterID) } else { - points, err := reader.GetObserverTelemetryBucketed(r.Context(), observerID, since, until, bucketHours) + var points []api.ObserverTelemetryPoint + points, err = reader.GetObserverTelemetryBucketed(r.Context(), observerID, since, until, bucketHours) if err == nil { telemetry = &api.ObserverTelemetry{Points: points} } @@ -240,3 +242,86 @@ func getObserverTelemetry(reader api.Reader) http.HandlerFunc { respond(w, http.StatusOK, telemetry) } } + +// activityIntervals is the fixed bucket set; every value divides 24h so bucket +// starts stay aligned to the clock regardless of when the window began. +var activityIntervals = map[string]time.Duration{ + "5m": 5 * time.Minute, + "15m": 15 * time.Minute, + "1h": time.Hour, + "6h": 6 * time.Hour, + "24h": 24 * time.Hour, +} + +// getObserverActivity godoc +// +// @Summary Get observer heard-activity history +// @Tags Observers +// @Produce json +// @Param observerId path string true "Observer UUID" +// @Param range query string false "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h" +// @Param interval query string false "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)" +// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until." +// @Success 200 {object} api.ObserverActivity +// @Failure 400 {object} handlers.APIError +// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Router /observers/{observerId}/activity [get] +func getObserverActivity(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + observerID, err := uuid.Parse(chi.URLParam(r, "observerId")) + if err != nil { + respondError(w, http.StatusBadRequest, "invalid observer ID") + return + } + rangeParam := r.URL.Query().Get("range") + if rangeParam == "" { + rangeParam = "24h" + } + window, err := time.ParseDuration(rangeParam) + if err != nil || window <= 0 || window > 720*time.Hour { + respondError(w, http.StatusBadRequest, "invalid range, use a duration up to 720h e.g. 24h, 168h, 720h") + return + } + intervalParam := r.URL.Query().Get("interval") + if intervalParam == "" { + intervalParam = "15m" + } + interval, ok := activityIntervals[intervalParam] + if !ok { + respondError(w, http.StatusBadRequest, "invalid interval, use 5m, 15m, 1h, 6h or 24h") + return + } + // the raw path scans live observations, so keep sub-hour windows short + if interval < time.Hour && window > 48*time.Hour { + respondError(w, http.StatusBadRequest, "range must be 48h or less for intervals under 1h") + return + } + if window/interval > 1000 { + respondError(w, http.StatusBadRequest, "range/interval exceeds 1000 buckets") + return + } + until := time.Time{} + if values, ok := r.URL.Query()["until"]; ok { + n, err := strconv.ParseInt(values[0], 10, 64) + now := time.Now() + if len(values) != 1 || err != nil || n < 0 || n > now.Add(clockSkewTolerance).UnixMilli() || n < now.Add(-720*time.Hour).UnixMilli() { + respondError(w, http.StatusBadRequest, "until must be one past epoch-millisecond timestamp within 30 days") + return + } + until = time.UnixMilli(n) + } + activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval, until) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + respondError(w, http.StatusNotFound, "observer not found") + return + } + respondError(w, http.StatusInternalServerError, "internal server error") + return + } + activity.Range = rangeParam + activity.Interval = intervalParam + respond(w, http.StatusOK, activity) + } +} diff --git a/internal/api/handlers/observers_test.go b/internal/api/handlers/observers_test.go index 148a06f8..7344526e 100644 --- a/internal/api/handlers/observers_test.go +++ b/internal/api/handlers/observers_test.go @@ -5,6 +5,9 @@ package handlers import ( "context" + "encoding/json" + "errors" + "fmt" "net/http" "net/http/httptest" "testing" @@ -13,6 +16,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" "github.com/google/uuid" + "github.com/jackc/pgx/v5" ) func TestGetObserverTelemetry_InvalidUUID(t *testing.T) { @@ -142,6 +146,29 @@ func TestGetObserverTelemetry_Bucketed_OK(t *testing.T) { } } +func TestGetObserverTelemetry_Bucketed_StoreError(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + r := chi.NewRouter() + r.Get("/observers/{observerId}/telemetry", getObserverTelemetry(stubReader{ + getObserverTelemetryBucketed: func(_ context.Context, _ uuid.UUID, _, _ time.Time, _ int32) ([]api.ObserverTelemetryPoint, error) { + return nil, errors.New("boom") + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/telemetry?interval=6h", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d", w.Code) + } + var body map[string]APIError + if err := json.NewDecoder(w.Body).Decode(&body); err != nil { + t.Fatalf("expected JSON error body, got %q: %v", w.Body.String(), err) + } + if body["error"].Code != "internal_server_error" { + t.Errorf("expected internal_server_error, got %q", body["error"].Code) + } +} + func TestListObserverAdverts_OK(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() @@ -168,3 +195,240 @@ func TestListObserverAdverts_InvalidUUID(t *testing.T) { t.Errorf("expected 400, got %d", w.Code) } } + +func TestGetObserverActivity_Defaults(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + var gotWindow, gotInterval time.Duration + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { + gotWindow, gotInterval = window, interval + return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", w.Code) + } + if gotWindow != 24*time.Hour { + t.Errorf("expected window 24h, got %s", gotWindow) + } + if gotInterval != 15*time.Minute { + t.Errorf("expected interval 15m, got %s", gotInterval) + } + var body api.ObserverActivity + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Range != "24h" || body.Interval != "15m" { + t.Errorf("expected range 24h interval 15m, got %q/%q", body.Range, body.Interval) + } + + skewed := time.Now().Add(time.Minute).UnixMilli() + req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/observers/%s/activity?until=%d", observerID.String(), skewed), nil) + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 for until within clock skew, got %d (%s)", w.Code, w.Body.String()) + } +} + +func TestGetObserverActivity_CustomRangeAndInterval(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + var gotWindow, gotInterval time.Duration + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { + gotWindow, gotInterval = window, interval + return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity?range=168h&interval=1h", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", w.Code) + } + if gotWindow != 168*time.Hour { + t.Errorf("expected window 168h, got %s", gotWindow) + } + if gotInterval != time.Hour { + t.Errorf("expected interval 1h, got %s", gotInterval) + } + var body api.ObserverActivity + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Range != "168h" || body.Interval != "1h" { + t.Errorf("expected range 168h interval 1h, got %q/%q", body.Range, body.Interval) + } +} + +func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { + t.Fatal("reader should not be called") + return nil, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity?range=168h&interval=15m", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d (%s)", w.Code, w.Body.String()) + } + var errBody map[string]APIError + if err := json.Unmarshal(w.Body.Bytes(), &errBody); err != nil { + t.Fatalf("decode body: %v", err) + } + if errBody["error"].Message != "range must be 48h or less for intervals under 1h" { + t.Errorf("unexpected error message %q", errBody["error"].Message) + } + + var gotWindow, gotInterval time.Duration + ok := chi.NewRouter() + ok.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { + gotWindow, gotInterval = window, interval + return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil + }, + })) + req = httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity?range=48h&interval=15m", nil) + w = httptest.NewRecorder() + ok.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 at 48h/15m, got %d (%s)", w.Code, w.Body.String()) + } + if gotWindow != 48*time.Hour || gotInterval != 15*time.Minute { + t.Errorf("expected 48h/15m, got %s/%s", gotWindow, gotInterval) + } +} + +func TestGetObserverActivity_BadRequests(t *testing.T) { + tests := []struct { + name string + path string + query string + }{ + {"invalid uuid", "not-a-uuid", ""}, + {"bad end", "00000000-0000-0000-0000-000000000001", "?until=banana"}, + {"negative end", "00000000-0000-0000-0000-000000000001", "?until=-1"}, + {"duplicate end", "00000000-0000-0000-0000-000000000001", "?until=1&until=2"}, + {"future end", "00000000-0000-0000-0000-000000000001", "?until=253402300799999"}, + {"unparseable range", "00000000-0000-0000-0000-000000000001", "?range=banana"}, + {"range too long", "00000000-0000-0000-0000-000000000001", "?range=721h"}, + {"zero range", "00000000-0000-0000-0000-000000000001", "?range=0"}, + {"invalid interval", "00000000-0000-0000-0000-000000000001", "?interval=2h"}, + {"too many buckets", "00000000-0000-0000-0000-000000000001", "?range=720h&interval=5m"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { + t.Fatal("reader should not be called") + return nil, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+tt.path+"/activity"+tt.query, nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Errorf("expected 400, got %d (%s)", w.Code, w.Body.String()) + } + }) + } +} + +func TestGetObserverActivity_NotFound(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { + return nil, pgx.ErrNoRows + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusNotFound { + t.Fatalf("expected 404, got %d", w.Code) + } + var body struct { + Error APIError `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Error.Code != "not_found" { + t.Errorf("expected code not_found, got %q", body.Error.Code) + } +} + +func TestGetObserverActivity_ReaderError(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { + return nil, errors.New("boom") + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusInternalServerError { + t.Errorf("expected 500, got %d", w.Code) + } +} + +func TestGetObserverActivity_NullsSerialise(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { + return &api.ObserverActivity{ + PayloadTypes: []api.PayloadBreakdownItem{}, + Points: []api.ObserverActivityPoint{{T: 1757376000000, Observations: 3}}, + }, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", w.Code) + } + var body map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("decode body: %v", err) + } + radio, ok := body["radio"] + if !ok { + t.Fatal("expected radio key in body") + } + if radio != nil { + t.Errorf("expected radio null, got %v", radio) + } + points, ok := body["points"].([]any) + if !ok || len(points) != 1 { + t.Fatalf("expected 1 point, got %v", body["points"]) + } + point := points[0].(map[string]any) + airtime, ok := point["airtimeMs"] + if !ok { + t.Fatal("expected airtimeMs key in point") + } + if airtime != nil { + t.Errorf("expected airtimeMs null, got %v", airtime) + } +} diff --git a/internal/api/handlers/packets.go b/internal/api/handlers/packets.go index 43e6bb71..82f10262 100644 --- a/internal/api/handlers/packets.go +++ b/internal/api/handlers/packets.go @@ -5,6 +5,7 @@ package handlers import ( "encoding/hex" + "errors" "net/http" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) // PacketsRouter mounts all /packets routes onto a subrouter. @@ -45,7 +47,7 @@ func PacketsRouter(reader api.Reader) http.Handler { // @Param since query int false "Filter by first_heard_at >= since (epoch ms)" // @Param until query int false "Filter by first_heard_at <= until (epoch ms)" // @Param cursor query int false "epoch ms of last item for pagination; last_heard_at, or site-local heard_at when iatas is set" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} object // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -93,20 +95,16 @@ func listPackets(reader api.Reader) http.HandlerFunc { } cursor = c } - var limit int32 = 50 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } iatas := parseIATAs(r) if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -134,7 +132,7 @@ func listPackets(reader api.Reader) http.HandlerFunc { // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" // @Param scope query string false "Filter by transport scope name" -// @Param limit query int false "Max results (default 100)" +// @Param limit query int false "Max results (default 100); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} []api.PacketSummary // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -151,14 +149,10 @@ func listPacketsBackfill(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "afterObservationId must be an integer") return } - var limit int32 = 100 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 100) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var payloadType int16 = -1 if v := r.URL.Query().Get("payloadType"); v != "" { @@ -180,7 +174,7 @@ func listPacketsBackfill(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -215,9 +209,13 @@ func getPacket(reader api.Reader) http.HandlerFunc { return } packet, err := reader.GetPacket(r.Context(), hash) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && packet == nil: respondError(w, http.StatusNotFound, "packet not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, packet) } diff --git a/internal/api/handlers/paths.go b/internal/api/handlers/paths.go new file mode 100644 index 00000000..7f25169a --- /dev/null +++ b/internal/api/handlers/paths.go @@ -0,0 +1,68 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// getPathStats godoc +// +// @Summary Received path-entry and hash-width distributions +// @Description Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly path rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted. +// @Tags Stats +// @Produce json +// @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" +// @Param until query int true "Exclusive end, epoch milliseconds; at most 30 days after since" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.PathStats +// @Failure 400 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/paths [get] +func getPathStats(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + since, until, err := parseStatsWindow(r, distributionWindow) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondRegionError(w, err) + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} + } + } + stats, err := reader.GetPathStats(ctx, since, until, iatas) + switch { + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "path query timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("path stats failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, stats) + } + } +} diff --git a/internal/api/handlers/paths_test.go b/internal/api/handlers/paths_test.go new file mode 100644 index 00000000..b7be4205 --- /dev/null +++ b/internal/api/handlers/paths_test.go @@ -0,0 +1,94 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +type pathReader struct { + stubReader + read func(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) +} + +func (s pathReader) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + return s.read(ctx, since, until, iatas) +} + +func TestPathsRejectInvalidWindow(t *testing.T) { + for _, query := range []string{"", "since=0", "since=x&until=1", "since=-1&until=1", "since=1&until=1", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { + w := httptest.NewRecorder() + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) + if w.Code != 400 { + t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) + } + } +} + +func TestPathsFiltersErrorsAndCancellation(t *testing.T) { + for _, tc := range []struct { + name, query string + iatas []string + err error + status int + }{ + {"global", "", nil, nil, 200}, {"IATAs", "&iatas=yvr,yyj", []string{"YVR", "YYJ"}, nil, 200}, + {"region", "®ion=west", []string{"YVR"}, nil, 200}, {"empty region", "®ion=empty", []string{""}, nil, 200}, + {"missing region", "®ion=missing", nil, nil, 400}, {"timeout", "", nil, context.DeadlineExceeded, 503}, + {"database error", "", nil, errors.New("private detail"), 500}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := pathReader{stubReader: stubReader{getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + if slug == "missing" { + return nil, pgx.ErrNoRows + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }}, read: func(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + if d, ok := ctx.Deadline(); !ok || time.Until(d) > 15*time.Second { + t.Fatal("query has no bounded deadline") + } + if since.UnixMilli() != 0 || until.UnixMilli() != 2592000000 || !reflect.DeepEqual(iatas, tc.iatas) { + t.Fatalf("unexpected arguments: %v %v %v", since, until, iatas) + } + return &api.PathStats{Until: until.UnixMilli(), Receptions: 4, Hashed: 1, Empty: 1, Trace: 1, Unclassified: 1}, tc.err + }} + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) + if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + if tc.status == 200 { + var got api.PathStats + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || got.Receptions != 4 || got.Trace != 1 { + t.Fatalf("response: %+v %v", got, err) + } + } + }) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + called := false + r := pathReader{read: func(ctx context.Context, _ time.Time, _ time.Time, _ []string) (*api.PathStats, error) { + called = true + return nil, ctx.Err() + }} + w := httptest.NewRecorder() + StatsRouter(r, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) + if !called || w.Body.Len() != 0 { + t.Fatal("cancellation did not reach reader") + } +} diff --git a/internal/api/handlers/regions.go b/internal/api/handlers/regions.go index b9bfe5c5..8437f41b 100644 --- a/internal/api/handlers/regions.go +++ b/internal/api/handlers/regions.go @@ -5,13 +5,16 @@ package handlers import ( "context" + "errors" "fmt" "net/http" "strconv" "strings" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) // RegionsRouter mounts all /regions routes onto a subrouter. @@ -34,15 +37,18 @@ func RegionsRouter(reader api.Reader) http.Handler { // @Tags Regions // @Produce json // @Success 200 {array} api.RegionSummary -// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /regions [get] func listRegions(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { regions, err := reader.ListRegions(r.Context()) if err != nil { - respondError(w, http.StatusNotFound, "no regions found") + respondError(w, http.StatusInternalServerError, "internal server error") return } + if regions == nil { + regions = []api.RegionSummary{} + } respond(w, http.StatusOK, regions) } } @@ -56,6 +62,7 @@ func listRegions(reader api.Reader) http.HandlerFunc { // @Success 200 {object} api.Region // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /regions/{regionId} [get] func getRegion(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -66,14 +73,39 @@ func getRegion(reader api.Reader) http.HandlerFunc { return } region, err := reader.GetRegion(r.Context(), int32(regionInt)) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && region == nil: respondError(w, http.StatusNotFound, "region not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, region) } } +// distributionWindow caps /stats/signal and /stats/paths, which sum every rolled hour per request. +const distributionWindow = 30 * 24 * time.Hour + +// parseStatsWindow rounds both endpoints down to UTC hours. Polls within an +// hour share a cache key; the response reports the effective window. A valid +// sub-hour request can contain no complete buckets and return an empty result. +func parseStatsWindow(r *http.Request, maxWindow time.Duration) (time.Time, time.Time, error) { + q := r.URL.Query() + for _, key := range []string{"since", "until"} { + if len(q[key]) != 1 || q.Get(key) == "" { + return time.Time{}, time.Time{}, fmt.Errorf("%s must be supplied exactly once", key) + } + } + since, errSince := strconv.ParseInt(q.Get("since"), 10, 64) + until, errUntil := strconv.ParseInt(q.Get("until"), 10, 64) + if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 || until-since > maxWindow.Milliseconds() { + return time.Time{}, time.Time{}, fmt.Errorf("since and until must be epoch milliseconds between 0 and 253402300799999, with a positive window of at most %d days", int(maxWindow/(24*time.Hour))) + } + return time.UnixMilli(since).UTC().Truncate(time.Hour), time.UnixMilli(until).UTC().Truncate(time.Hour), nil +} + // parseIATAs splits a comma-separated iatas query param and uppercases each value. func parseIATAs(r *http.Request) []string { raw := r.URL.Query().Get("iatas") @@ -169,3 +201,13 @@ func resolveRegionIATAs(ctx context.Context, regionID, regionSlug string, reader } return region.IATAs, nil } + +// respondRegionError reports an unknown or malformed region filter as 400 and a +// failed lookup as a retryable 500. +func respondRegionError(w http.ResponseWriter, err error) { + if errors.Is(err, pgx.ErrNoRows) || errors.Is(err, strconv.ErrSyntax) || errors.Is(err, strconv.ErrRange) { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + respondError(w, http.StatusInternalServerError, "internal server error") +} diff --git a/internal/api/handlers/responses.go b/internal/api/handlers/responses.go index 3a12ec12..e1ef3b28 100644 --- a/internal/api/handlers/responses.go +++ b/internal/api/handlers/responses.go @@ -6,7 +6,8 @@ package handlers import ( "encoding/json" - "log" + "fmt" + "log/slog" "net/http" "strings" ) @@ -25,7 +26,7 @@ func respond(w http.ResponseWriter, status int, data any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) if err := json.NewEncoder(w).Encode(data); err != nil { - log.Printf("api: failed to encode response: %v", err) + slog.Error("api: failed to encode response", "component", "api", "error", err) } } @@ -33,7 +34,7 @@ func respond(w http.ResponseWriter, status int, data any) { // The error code is derived automatically from the HTTP status text. func respondError(w http.ResponseWriter, status int, message string) { if status >= 500 { - log.Printf("api: error %d: %s", status, message) + slog.Error(fmt.Sprintf("api: error %d: %s", status, message), "component", "api") } code := strings.ToLower(strings.ReplaceAll(http.StatusText(status), " ", "_")) respond(w, status, map[string]APIError{"error": {Code: code, Message: message}}) diff --git a/internal/api/handlers/route_evidence.go b/internal/api/handlers/route_evidence.go new file mode 100644 index 00000000..a194306c --- /dev/null +++ b/internal/api/handlers/route_evidence.go @@ -0,0 +1,118 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" +) + +func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.RouteEvidenceQuery, error) { + q := api.RouteEvidenceQuery{} + values := r.URL.Query() + for _, name := range []string{"pageCursor", "since", "until", "range", "limit"} { + if len(values[name]) > 1 { + return q, api.ErrRouteEvidenceInput + } + } + if values.Has("cursor") { + return q, api.ErrRouteEvidenceInput + } + limit, err := parseLimit(r, 50) + if err != nil { + return q, err + } + q.Limit = limit + if values.Has("pageCursor") { + c, err := api.ParseRouteEvidenceCursor(values.Get("pageCursor")) + if err != nil || c.IATA != iata || c.PathKey != key { + return q, api.ErrRouteEvidenceInput + } + q.Cursor = c + q.Since = c.Since + q.Until = c.Until + } else { + q.Until = now.UTC().Truncate(time.Millisecond) + q.Since = q.Until.Add(-24 * time.Hour) + } + if values.Has("range") { + if values.Has("pageCursor") || values.Has("since") || values.Has("until") { + return q, api.ErrRouteEvidenceInput + } + duration, err := time.ParseDuration(values.Get("range")) + if err != nil || duration < time.Millisecond || duration > api.MaxRouteEvidenceWindow || duration%time.Millisecond != 0 { + return q, api.ErrRouteEvidenceInput + } + q.Since = q.Until.Add(-duration) + } + if values.Has("since") || values.Has("until") { + if !values.Has("since") || !values.Has("until") { + return q, api.ErrRouteEvidenceInput + } + since, e1 := strconv.ParseInt(values.Get("since"), 10, 64) + until, e2 := strconv.ParseInt(values.Get("until"), 10, 64) + if e1 != nil || e2 != nil || since < 0 || until < 0 || since > 253402300799999 || until > 253402300799999 { + return q, api.ErrRouteEvidenceInput + } + q.Since = time.UnixMilli(since).UTC() + q.Until = time.UnixMilli(until).UTC() + if q.Cursor != nil && (!q.Since.Equal(q.Cursor.Since) || !q.Until.Equal(q.Cursor.Until)) { + return q, api.ErrRouteEvidenceInput + } + } + if !api.ValidRouteEvidenceWindow(q.Since, q.Until) || q.Until.After(now.Add(clockSkewTolerance)) { + return q, api.ErrRouteEvidenceInput + } + return q, nil +} + +// getRouteEvidence godoc +// +// @Summary Get retained reports matching a full saved route prefix sequence +// @Description Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope. +// @Tags Routes +// @Produce json +// @Param iata path string true "Three-character IATA code" +// @Param pathKey path string true "Stable 32-hex pathKey from a known-route response" +// @Param range query string false "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor" +// @Param since query int false "Window start epoch ms; provide with until (default last 24h)" +// @Param until query int false "Exclusive window end epoch ms; maximum span 30d, end may be up to 5 minutes ahead of server time (clock skew tolerance)" +// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; window is pinned" +// @Param limit query int false "Default 50; positive, capped at 200" minimum(1) maximum(200) +// @Success 200 {object} api.RouteEvidence +// @Failure 400 {object} handlers.APIError +// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Router /routes/{iata}/{pathKey}/observations [get] +func getRouteEvidence(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + iata, key := strings.ToUpper(chi.URLParam(r, "iata")), strings.ToLower(chi.URLParam(r, "pathKey")) + if !api.ValidRouteEvidenceKey(iata, key) { + respondError(w, 400, "invalid IATA or route pathKey") + return + } + query, err := routeEvidenceQuery(r, iata, key, time.Now()) + if err != nil { + respondError(w, 400, "invalid route evidence window, limit or page cursor") + return + } + result, err := reader.GetRouteEvidence(r.Context(), iata, key, query) + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && result == nil: + respondError(w, 404, "saved route not found") + case errors.Is(err, api.ErrRouteEvidenceInput): + respondError(w, 400, "invalid route evidence request") + case err != nil: + respondError(w, 500, "internal server error") + default: + respond(w, 200, result) + } + } +} diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go new file mode 100644 index 00000000..f2268d86 --- /dev/null +++ b/internal/api/handlers/route_evidence_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "context" + "errors" + "fmt" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +const evidenceKey = "f097439148601d9f3291c474f82fa64c" + +func TestRouteEvidenceQuery(t *testing.T) { + now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC) + q, err := routeEvidenceQuery(httptest.NewRequest("GET", "/", nil), "YOW", evidenceKey, now) + if err != nil || q.Limit != 50 || q.Until != now || q.Until.Sub(q.Since) != 24*time.Hour { + t.Fatalf("defaults: %+v %v", q, err) + } + week, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?range=168h", nil), "YOW", evidenceKey, now) + if err != nil || !week.Until.Equal(now) || week.Until.Sub(week.Since) != 7*24*time.Hour { + t.Fatalf("server anchored range: %+v %v", week, err) + } + c := api.RouteEvidenceCursor{IATA: "YOW", PathKey: evidenceKey, Since: q.Since, Until: q.Until, HeardAt: q.Since.Add(time.Hour + 123*time.Microsecond), ID: 3} + next, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String())+"&limit=999", nil), "YOW", evidenceKey, now.Add(time.Hour)) + if err != nil || next.Limit != 200 || !next.Until.Equal(now) || !next.Cursor.HeardAt.Equal(c.HeardAt) { + t.Fatalf("pinned cursor: %+v %v", next, err) + } + for _, query := range []string{"?limit=0", "?limit=-1", "?limit=a", "?limit=1&limit=2", "?since=0", "?until=10", "?since=2&until=1", "?since=0&until=2678400000", "?since=0&until=99999999999999999", "?pageCursor=bad", "?pageCursor=", "?cursor=123", "?pageCursor=" + url.QueryEscape(c.String()) + "&since=0&until=10"} { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String()), nil), "YVR", evidenceKey, now); err == nil { + t.Fatal("cursor crossed IATA") + } + for _, query := range []string{"range=0h", "range=-1h", "range=721h", "range=1ns", "range=7d", "range=24h&range=168h", "range=24h&since=0&until=10", "range=24h&pageCursor=" + url.QueryEscape(c.String())} { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } + ahead := now.Add(time.Minute) + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", fmt.Sprintf("/?since=%d&until=%d", ahead.Add(-time.Hour).UnixMilli(), ahead.UnixMilli()), nil), "YOW", evidenceKey, now); err != nil { + t.Fatalf("rejected until within clock skew: %v", err) + } + far := now.Add(10 * time.Minute) + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", fmt.Sprintf("/?since=%d&until=%d", far.Add(-time.Hour).UnixMilli(), far.UnixMilli()), nil), "YOW", evidenceKey, now); err == nil { + t.Fatal("accepted until far in the future") + } +} + +func TestRouteEvidenceHandler(t *testing.T) { + for _, tc := range []struct { + path string + err error + status int + }{ + {"/yow/" + evidenceKey + "/observations", nil, 200}, + {"/YOW/bad/observations", nil, 400}, + {"/LONG/" + evidenceKey + "/observations", nil, 400}, + {"/YOW/" + evidenceKey + "/observations", pgx.ErrNoRows, 404}, + {"/YOW/" + evidenceKey + "/observations", errors.New("database unavailable"), 500}, + } { + called := false + router := RoutesRouter(stubReader{getRouteEvidence: func(_ context.Context, iata, key string, q api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + called = true + if iata != "YOW" || key != evidenceKey || q.Limit != 50 { + t.Fatal("wrong reader scope") + } + return &api.RouteEvidence{Items: []api.RouteObservation{}}, tc.err + }}) + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", tc.path, nil)) + if w.Code != tc.status { + t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String()) + } + if tc.status == 400 && called { + t.Fatal("invalid request reached database") + } + } +} diff --git a/internal/api/handlers/routes.go b/internal/api/handlers/routes.go index 9e819ac2..86a7c861 100644 --- a/internal/api/handlers/routes.go +++ b/internal/api/handlers/routes.go @@ -22,6 +22,7 @@ func RoutesRouter(reader api.Reader) http.Handler { r.Get("/", listKnownRoutes(reader)) r.Get("/cross", searchCrossIATARoutes(reader)) r.Get("/search", searchKnownRoutes(reader)) + r.Get("/{iata}/{pathKey}/observations", getRouteEvidence(reader)) return r } @@ -33,7 +34,9 @@ func RoutesRouter(reader api.Reader) http.Handler { // @Param iata query string false "Filter by IATA code" // @Param hopCount query int false "Filter by exact hop count" // @Param cursor query int false "Epoch ms timestamp of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param cursorId query int false "id of the last item; with cursor, also returns later routes sharing that millisecond" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) +// @Failure 400 {object} handlers.APIError // @Success 200 {object} []api.KnownRoute // @Failure 500 {object} handlers.APIError // @Router /routes [get] @@ -52,13 +55,26 @@ func listKnownRoutes(reader api.Reader) http.HandlerFunc { cursor = time.UnixMilli(ms) } } - var limit int32 = 50 - if v := r.URL.Query().Get("limit"); v != "" { - if l, err := strconv.ParseInt(v, 10, 32); err == nil { - limit = int32(l) + // cursorId breaks ties within the cursor's millisecond. + var cursorID int64 + if v := r.URL.Query().Get("cursorId"); v != "" { + id, err := strconv.ParseInt(v, 10, 64) + if err != nil || id <= 0 { + respondError(w, http.StatusBadRequest, "cursorId must be a positive integer") + return + } + if cursor.IsZero() { + respondError(w, http.StatusBadRequest, "cursorId requires cursor") + return } + cursorID = id + } + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } - routes, err := reader.ListKnownRoutes(r.Context(), iata, hopCount, cursor, limit) + routes, err := reader.ListKnownRoutes(r.Context(), iata, hopCount, cursor, cursorID, limit) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/routes_test.go b/internal/api/handlers/routes_test.go index 699d5455..9860d6d6 100644 --- a/internal/api/handlers/routes_test.go +++ b/internal/api/handlers/routes_test.go @@ -60,10 +60,44 @@ func TestSearchCrossIATARoutes_MissingParams(t *testing.T) { } } +func TestListKnownRoutes_CursorID(t *testing.T) { + var gotCursor time.Time + var gotID int64 + r := chi.NewRouter() + r.Get("/routes", listKnownRoutes(stubReader{ + listKnownRoutes: func(_ context.Context, _ string, _ int32, cursor time.Time, cursorID int64, _ int32) ([]api.KnownRoute, error) { + gotCursor, gotID = cursor, cursorID + return nil, nil + }, + })) + for _, tc := range []struct { + query string + code int + id int64 + }{ + {"cursor=1700000000000&cursorId=42", http.StatusOK, 42}, + {"cursor=1700000000000", http.StatusOK, 0}, + {"cursor=1700000000000&cursorId=0", http.StatusBadRequest, 0}, + {"cursor=1700000000000&cursorId=x", http.StatusBadRequest, 0}, + {"cursorId=42", http.StatusBadRequest, 0}, + } { + gotID = 0 + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/routes?"+tc.query, nil)) + if w.Code != tc.code { + t.Errorf("%s: code %d, want %d", tc.query, w.Code, tc.code) + continue + } + if tc.code == http.StatusOK && (gotID != tc.id || gotCursor.UnixMilli() != 1700000000000) { + t.Errorf("%s: cursor %v id %d", tc.query, gotCursor, gotID) + } + } +} + func TestListKnownRoutes_OK(t *testing.T) { r := chi.NewRouter() r.Get("/routes", listKnownRoutes(stubReader{ - listKnownRoutes: func(_ context.Context, _ string, _ int32, _ time.Time, _ int32) ([]api.KnownRoute, error) { + listKnownRoutes: func(_ context.Context, _ string, _ int32, _ time.Time, _ int64, _ int32) ([]api.KnownRoute, error) { return []api.KnownRoute{{IATA: "YVR"}}, nil }, })) diff --git a/internal/api/handlers/scope_stats_test.go b/internal/api/handlers/scope_stats_test.go new file mode 100644 index 00000000..9d30e3f0 --- /dev/null +++ b/internal/api/handlers/scope_stats_test.go @@ -0,0 +1,152 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "slices" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" +) + +func TestGetStatsScopes_RegionErrors(t *testing.T) { + for _, query := range []string{"regionId=invalid", "regionId=2147483648", "regionId=999", "region=missing"} { + t.Run(query, func(t *testing.T) { + reader := stubReader{ + getRegion: func(context.Context, int32) (*api.Region, error) { + return nil, pgx.ErrNoRows + }, + getRegionBySlug: func(context.Context, string) (*api.Region, error) { + return nil, pgx.ErrNoRows + }, + } + w := httptest.NewRecorder() + getStatsScopes(reader, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+query, nil)) + if w.Code != http.StatusBadRequest { + t.Fatalf("got status %d, want 400", w.Code) + } + }) + } +} + +func TestGetStatsScopes_Filters(t *testing.T) { + for _, tc := range []struct { + query string + want []string + empty bool + }{ + {"", nil, false}, + {"iata=yvr", []string{"YVR"}, false}, + {"iatas=yyj,%20yvr,yyj", []string{"YYJ", "YVR", "YYJ"}, false}, + {"iatas=YVR&iata=YYZ", []string{"YVR"}, false}, + {"regionId=7", []string{"YVR", "YYJ"}, false}, + {"region=west", []string{"YVR", "YYJ"}, false}, + {"regionId=7®ion=missing", []string{"YVR", "YYJ"}, false}, + {"iatas=YYZ,YVR®ion=west", []string{"YYZ", "YVR", "YVR", "YYJ"}, false}, + {"region=empty", nil, true}, + {"region=empty&iata=YVR", []string{"YVR"}, false}, + } { + t.Run(tc.query, func(t *testing.T) { + called := false + reader := stubReader{ + getRegion: func(_ context.Context, id int32) (*api.Region, error) { + if id != 7 { + t.Fatalf("unexpected region ID %d", id) + } + return &api.Region{IATAs: []string{"YVR", "YYJ"}}, nil + }, + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + if slug != "west" { + t.Fatalf("unexpected region slug %s", slug) + } + return &api.Region{IATAs: []string{"YVR", "YYJ"}}, nil + }, + getScopeStats: func(_ context.Context, iatas []string, _ time.Time) ([]api.ScopeStats, error) { + called = true + if !slices.Equal(iatas, tc.want) { + t.Fatalf("IATAs = %v, want %v", iatas, tc.want) + } + return []api.ScopeStats{{Name: "#test", PacketCount: 2, ObserverCount: 1, NodeCount: 3}}, nil + }, + } + w := httptest.NewRecorder() + getStatsScopes(reader, testScopes(map[string][]string{"YVR": {"#test"}, "YYJ": {"#test"}}))(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + if called == tc.empty { + t.Fatalf("reader called = %v, empty region = %v", called, tc.empty) + } + var rows []api.ScopeStats + if err := json.Unmarshal(w.Body.Bytes(), &rows); err != nil { + t.Fatal(err) + } + if tc.empty { + if rows == nil || len(rows) != 0 { + t.Fatalf("empty region returned %s", w.Body.String()) + } + } else if len(rows) != 1 || rows[0].PacketCount != 2 || rows[0].ObserverCount != 1 || rows[0].NodeCount != 3 { + t.Fatalf("response changed: %+v", rows) + } + }) + } +} + +func TestGetStatsScopes_ReaderError(t *testing.T) { + w := httptest.NewRecorder() + getStatsScopes(stubReader{getScopeStats: func(context.Context, []string, time.Time) ([]api.ScopeStats, error) { + return nil, errors.New("database unavailable") + }}, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil)) + if w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", w.Code) + } +} + +func testScopes(byIATA map[string][]string) *scopestore.ScopeStore { + s := scopestore.New() + s.SetManualMembers(byIATA) + return s +} + +func TestGetStatsScopes_FiltersToRegionMembers(t *testing.T) { + reader := stubReader{getScopeStats: func(context.Context, []string, time.Time) ([]api.ScopeStats, error) { + return []api.ScopeStats{{Name: "#elsewhere", PacketCount: 9}, {Name: "#on"}, {Name: "#yow", PacketCount: 4}}, nil + }} + members := testScopes(map[string][]string{"YOW": {"#yow", "#on"}}) + for _, tc := range []struct { + query string + members ScopeMembership + want []string + }{ + {"", members, []string{"#elsewhere", "#on", "#yow"}}, + {"iata=YOW", members, []string{"#on", "#yow"}}, + {"iata=YUL", members, []string{}}, + {"iata=YOW", nil, []string{}}, + } { + w := httptest.NewRecorder() + getStatsScopes(reader, tc.members)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil)) + var rows []api.ScopeStats + if err := json.Unmarshal(w.Body.Bytes(), &rows); err != nil || rows == nil { + t.Fatalf("%q: body %s, err %v", tc.query, w.Body.String(), err) + } + got := []string{} + for _, row := range rows { + got = append(got, row.Name) + } + if !slices.Equal(got, tc.want) { + t.Fatalf("%q: got %v, want %v", tc.query, got, tc.want) + } + } +} diff --git a/internal/api/handlers/scopes.go b/internal/api/handlers/scopes.go index 5267ed1a..d6293234 100644 --- a/internal/api/handlers/scopes.go +++ b/internal/api/handlers/scopes.go @@ -4,19 +4,33 @@ package handlers import ( + "errors" "net/http" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) +// ScopeMembership lists scopes by configured region and imported catalogue membership. +type ScopeMembership interface { + NamesForIATAs(iatas []string) []string // sorted +} + +func scopeNamesFor(scopes ScopeMembership, iatas []string) []string { + if scopes == nil { + return []string{} + } + return scopes.NamesForIATAs(iatas) +} + // ScopesRouter mounts all /scopes routes onto a subrouter. // // GET /scopes → listScopes // GET /scopes/{name} → getScope -func ScopesRouter(reader api.Reader) http.Handler { +func ScopesRouter(reader api.Reader, scopes ScopeMembership) http.Handler { r := chi.NewRouter() - r.Get("/", listScopes(reader)) + r.Get("/", listScopes(reader, scopes)) r.Get("/{name}", getScope(reader)) return r } @@ -24,26 +38,28 @@ func ScopesRouter(reader api.Reader) http.Handler { // listScopes godoc // // @Summary List transport scopes +// @Description Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes. // @Tags Scopes // @Produce json // @Param iatas query string false "Filter by IATA code(s), comma-separated" // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" -// @Success 200 {object} object +// @Success 200 {array} string +// @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /scopes [get] -func listScopes(reader api.Reader) http.HandlerFunc { +func listScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { iatas := parseIATAs(r) - if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { - regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) + regionID, region := r.URL.Query().Get("regionId"), r.URL.Query().Get("region") + if regionID != "" || region != "" { + regionIATAs, err := resolveRegionIATAs(r.Context(), regionID, region, reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) - } - if len(iatas) == 0 { + } else if len(iatas) == 0 { names, err := reader.GetScopeNames(r.Context()) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") @@ -52,18 +68,14 @@ func listScopes(reader api.Reader) http.HandlerFunc { respond(w, http.StatusOK, names) return } - scopes, err := reader.GetScopesByIATAs(r.Context(), iatas) - if err != nil { - respondError(w, http.StatusInternalServerError, "internal server error") - return - } - respond(w, http.StatusOK, scopes) + respond(w, http.StatusOK, scopeNamesFor(scopes, iatas)) } } // getScope godoc // // @Summary Get scope detail by name +// @Description packetCount sums packets per hour across every retained rollup hour; iatas are the IATAs that heard the scope's packets. Observer and node counts are current memberships. // @Tags Scopes // @Produce json // @Param name path string true "Scope name e.g. %23bc (URL-encoded #bc)" @@ -75,9 +87,13 @@ func getScope(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") scope, err := reader.GetScopeByName(r.Context(), name) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && scope == nil: respondError(w, http.StatusNotFound, "scope not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "failed to load scope") + return } respond(w, http.StatusOK, scope) } diff --git a/internal/api/handlers/scopes_test.go b/internal/api/handlers/scopes_test.go index 29abb392..e23bbc57 100644 --- a/internal/api/handlers/scopes_test.go +++ b/internal/api/handlers/scopes_test.go @@ -5,12 +5,15 @@ package handlers import ( "context" + "encoding/json" "net/http" "net/http/httptest" + "slices" "testing" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) func TestListScopes_NoIATAs_OK(t *testing.T) { @@ -19,7 +22,7 @@ func TestListScopes_NoIATAs_OK(t *testing.T) { getScopeNames: func(_ context.Context) ([]string, error) { return []string{"#bc", "#west"}, nil }, - })) + }, nil)) req := httptest.NewRequest(http.MethodGet, "/scopes", nil) w := httptest.NewRecorder() r.ServeHTTP(w, req) @@ -42,3 +45,69 @@ func TestGetScope_OK(t *testing.T) { t.Errorf("expected 200, got %d", w.Code) } } + +func TestGetScope_Errors(t *testing.T) { + for _, tc := range []struct { + name string + scope *api.ScopeDetail + err error + want int + }{ + {"no rows", nil, pgx.ErrNoRows, http.StatusNotFound}, + {"nil scope", nil, nil, http.StatusNotFound}, + {"database failure", nil, context.DeadlineExceeded, http.StatusInternalServerError}, + } { + t.Run(tc.name, func(t *testing.T) { + r := chi.NewRouter() + r.Get("/scopes/{name}", getScope(stubReader{ + getScopeByName: func(context.Context, string) (*api.ScopeDetail, error) { return tc.scope, tc.err }, + })) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/scopes/%23bc", nil)) + if w.Code != tc.want { + t.Fatalf("status = %d, want %d", w.Code, tc.want) + } + var body map[string]APIError + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || body["error"].Message == "" { + t.Fatalf("not a JSON error: %s", w.Body.String()) + } + }) + } +} + +func TestListScopes_FiltersByMembership(t *testing.T) { + reader := stubReader{ + getScopeNames: func(context.Context) ([]string, error) { + t.Fatal("filtered list must not fall back to every stored name") + return nil, nil + }, + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + return &api.Region{IATAs: []string{"YOW", "YGK"}}, nil + }, + } + members := testScopes(map[string][]string{"YOW": {"#on", "#yow"}, "YGK": {"#on"}, "YUL": {"#qc"}}) + for _, tc := range []struct { + query string + members ScopeMembership + want []string + }{ + {"iatas=yow", members, []string{"#on", "#yow"}}, + {"iatas=YUL,YYZ", members, []string{"#qc"}}, + {"region=ottawa", members, []string{"#on", "#yow"}}, + {"region=empty", members, []string{}}, + {"iatas=YOW", nil, []string{}}, + } { + w := httptest.NewRecorder() + listScopes(reader, tc.members)(w, httptest.NewRequest(http.MethodGet, "/scopes?"+tc.query, nil)) + var got []string + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || w.Code != http.StatusOK || got == nil { + t.Fatalf("%q: status %d body %s", tc.query, w.Code, w.Body.String()) + } + if !slices.Equal(got, tc.want) { + t.Fatalf("%q: got %v, want %v", tc.query, got, tc.want) + } + } +} diff --git a/internal/api/handlers/signal.go b/internal/api/handlers/signal.go new file mode 100644 index 00000000..1a2563dc --- /dev/null +++ b/internal/api/handlers/signal.go @@ -0,0 +1,68 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// getSignalStats godoc +// +// @Summary Reception signal distributions and hourly trends +// @Description Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly signal rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted. These last-hop readings do not measure end-to-end quality or packet loss. +// @Tags Stats +// @Produce json +// @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" +// @Param until query int true "Exclusive end, epoch milliseconds; at most 30 days after since" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.SignalStats +// @Failure 400 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/signal [get] +func getSignalStats(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + since, until, err := parseStatsWindow(r, distributionWindow) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondRegionError(w, err) + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} + } + } + stats, err := reader.GetSignalStats(ctx, since, until, iatas) + switch { + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "signal query timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("signal stats failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, stats) + } + } +} diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go new file mode 100644 index 00000000..5cc234bb --- /dev/null +++ b/internal/api/handlers/signal_test.go @@ -0,0 +1,114 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +type signalReader struct { + stubReader + read func(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) +} + +func (s signalReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + return s.read(ctx, since, until, iatas) +} + +func TestSignalRejectsInvalidWindow(t *testing.T) { + for _, query := range []string{"", "since=0", "until=1", "since=x&until=1", "since=-1&until=1", "since=0&until=0", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { + w := httptest.NewRecorder() + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) + if w.Code != 400 { + t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) + } + } +} + +func TestSignalFiltersAndErrors(t *testing.T) { + for _, tc := range []struct { + name, query string + iatas []string + err error + status int + }{ + {"global", "", nil, nil, 200}, + {"IATAs", "&iatas=yvr,yyj", []string{"YVR", "YYJ"}, nil, 200}, + {"region", "®ion=west", []string{"YVR"}, nil, 200}, + {"empty region", "®ion=empty", []string{""}, nil, 200}, + {"missing region", "®ion=missing", nil, nil, 400}, + {"timeout", "", nil, context.DeadlineExceeded, 503}, + {"database error", "", nil, errors.New("private detail"), 500}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := signalReader{stubReader: stubReader{getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + switch slug { + case "empty": + return &api.Region{}, nil + case "missing": + return nil, pgx.ErrNoRows + default: + return &api.Region{IATAs: []string{"YVR"}}, nil + } + }}, read: func(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { + t.Fatal("missing bounded query deadline") + } + if since.UnixMilli() != 0 || until.UnixMilli() != 2592000000 || !reflect.DeepEqual(iatas, tc.iatas) { + t.Fatalf("unexpected arguments: %v %v %v", since, until, iatas) + } + return &api.SignalStats{Until: until.UnixMilli(), Receptions: 5, Hourly: []api.SignalHour{}}, tc.err + }} + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) + if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + if tc.status == 200 { + var got api.SignalStats + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || got.Receptions != 5 || got.SNR.Average != nil { + t.Fatalf("invalid response: %+v %v", got, err) + } + } + }) + } +} + +func TestSignalCancellationReachesReader(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + called := false + reader := signalReader{read: func(ctx context.Context, _ time.Time, _ time.Time, _ []string) (*api.SignalStats, error) { + called = true + return nil, ctx.Err() + }} + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) + if !called || w.Body.Len() != 0 { + t.Fatalf("cancellation: called=%v body=%s", called, w.Body.String()) + } +} + +func TestStatsWindowSnapsPollingTimes(t *testing.T) { + for _, query := range []string{"since=123&until=86400123", "since=3599999&until=89999999"} { + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?"+query, nil), distributionWindow) + if err != nil || since.UnixMilli() != 0 || until.UnixMilli() != 86400000 { + t.Fatalf("%v %v %v", since, until, err) + } + } + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?since=1&until=2", nil), distributionWindow) + if err != nil || !since.Equal(until) { + t.Fatal("sub-hour normalization") + } +} diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index d9bc8177..426ad836 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -4,8 +4,10 @@ package handlers import ( - "log" + "errors" + "log/slog" "net/http" + "slices" "strconv" "time" @@ -26,24 +28,50 @@ import ( // GET /stats/scopes → GetStatsScopes // // All endpoints accept an optional iata= filter (case-insensitive). -func StatsRouter(reader api.Reader) http.Handler { +// StatsOptions configures StatsRouter. +type StatsOptions struct { + Scopes ScopeMembership // region-filtered scope lists; nil lists none + SeriesWindow time.Duration // longest /stats/series window, the rollup retention; 0 means 90 days +} + +// parseSince reads an optional epoch-ms since; zero means the endpoint's default window. +func parseSince(r *http.Request) (time.Time, error) { + p := r.URL.Query().Get("since") + if p == "" { + return time.Time{}, nil + } + ms, err := strconv.ParseInt(p, 10, 64) + if err != nil { + return time.Time{}, errors.New("since must be epoch milliseconds") + } + return time.UnixMilli(ms), nil +} + +func StatsRouter(reader api.Reader, opts StatsOptions) http.Handler { + scopes := opts.Scopes r := chi.NewRouter() + r.Get("/series", getStatsSeries(reader, opts.SeriesWindow)) r.Get("/overview", getStatsOverview(reader)) r.Get("/observations", getStatsObservations(reader)) + r.Get("/signal", getSignalStats(reader)) + r.Get("/paths", getPathStats(reader)) + r.Get("/observer-comparison", getObserverComparison(reader)) r.Get("/payload-breakdown", getStatsPayloadBreakdown(reader)) r.Get("/top-nodes", getStatsTopNodes(reader)) r.Get("/top-observers", getStatsTopObservers(reader)) r.Get("/top-advertisers", getStatsTopAdvertisers(reader)) + r.Get("/clock-drift", getStatsClockDrift(reader)) r.Get("/top-talkers", getStatsTopTalkers(reader)) r.Get("/radio-presets", getStatsRadioPresets(reader)) - r.Get("/scopes", getStatsScopes(reader)) + r.Get("/scopes", getStatsScopes(reader, scopes)) r.Get("/node-types", getStatsNodeTypes(reader)) return r } // getStatsOverview godoc // -// @Summary Network overview stats (last 24h) +// @Summary Network overview stats (last 24 rolled hours) +// @Description Summarises the 24 most recent hours that can have been rolled (an hour rolls about 95 minutes after it closes); since/until report that window. Packets count once per hour heard. Use /stats/series for sparklines. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -58,14 +86,17 @@ func getStatsOverview(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } overview, err := reader.GetStatsOverview(r.Context(), iatas) if err != nil { - log.Printf("api: GetStatsOverview failed: %v", err) + slog.Error("api: GetStatsOverview failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -75,7 +106,8 @@ func getStatsOverview(reader api.Reader) http.HandlerFunc { // getStatsObservations godoc // -// @Summary Hourly observation time series +// @Summary Hourly observation counts per IATA +// @Description Observation counts only; distinct packet and observer counts don't sum across IATAs, so read them from /stats/series. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -91,23 +123,22 @@ func getStatsObservations(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } points, err := reader.GetStatsObservations(r.Context(), iatas, since) if err != nil { - log.Printf("api: GetStatsObservations failed: %v", err) + slog.Error("api: GetStatsObservations failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -133,23 +164,22 @@ func getStatsPayloadBreakdown(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } breakdown, err := reader.GetStatsPayloadBreakdown(r.Context(), iatas, since) if err != nil { - log.Printf("api: GetStatsPayloadBreakdown failed: %v", err) + slog.Error("api: GetStatsPayloadBreakdown failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -159,13 +189,15 @@ func getStatsPayloadBreakdown(reader api.Reader) http.HandlerFunc { // getStatsTopNodes godoc // -// @Summary Top N nodes by observation count (from materialized view) +// @Summary Top N nodes by advert hearings (last 7 days by default) +// @Description nodeId is null when the node row has been deleted; publicKey always identifies it. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" -// @Param limit query int false "Max results (default 10)" +// @Param since query int false "Start of window epoch ms, rounded down to the hour (default 7 days ago)" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopNode // @Failure 500 {object} handlers.APIError // @Router /stats/top-nodes [get] @@ -175,23 +207,27 @@ func getStatsTopNodes(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) - } - var limit int32 = 10 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - limit = int32(l) } - nodes, err := reader.GetStatsTopNodes(r.Context(), iatas, limit) + limit, err := parseLimit(r, 10) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + since, err := parseSince(r) if err != nil { - log.Printf("api: GetStatsTopNodes failed: %v", err) + respondError(w, http.StatusBadRequest, err.Error()) + return + } + nodes, err := reader.GetStatsTopNodes(r.Context(), iatas, since, limit) + if err != nil { + slog.Error("api: GetStatsTopNodes failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -208,7 +244,7 @@ func getStatsTopNodes(reader api.Reader) http.HandlerFunc { // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param since query int false "Start of window epoch ms (default last 24h)" -// @Param limit query int false "Max results (default 10)" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopObserver // @Failure 500 {object} handlers.APIError // @Router /stats/top-observers [get] @@ -218,32 +254,27 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) } - var limit int32 = 10 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + limit, err := parseLimit(r, 10) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } observers, err := reader.GetStatsTopObservers(r.Context(), iatas, since, limit) if err != nil { - log.Printf("api: GetStatsTopObservers failed: %v", err) + slog.Error("api: GetStatsTopObservers failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -254,13 +285,14 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { // getStatsTopAdvertisers godoc // // @Summary Top N nodes by distinct ADVERT packet count (last 24h by default) +// @Description Each advert counts once per hour heard, however many requested IATAs heard it. nodeId is null when the node row has been deleted. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param since query int false "Start of window epoch ms (default last 24h)" -// @Param limit query int false "Max results (default 10)" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopAdvertiser // @Failure 500 {object} handlers.APIError // @Router /stats/top-advertisers [get] @@ -270,36 +302,72 @@ func getStatsTopAdvertisers(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) } - var limit int32 = 10 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + limit, err := parseLimit(r, 10) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + advertisers, err := reader.GetStatsTopAdvertisers(r.Context(), iatas, since, limit) + if err != nil { + slog.Error("api: GetStatsTopAdvertisers failed", "component", "api", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + return + } + respond(w, http.StatusOK, advertisers) + } +} + +// getStatsClockDrift godoc +// +// @Summary Repeaters/room servers whose clock has drifted beyond the configured threshold, worst first +// @Tags Stats +// @Produce json +// @Param iatas query string false "Comma-separated IATA codes" +// @Param regionId query int false "Filter by region ID, expands to member IATAs" +// @Param region query string false "Filter by region slug, expands to member IATAs" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) +// @Success 200 {array} api.ClockDriftEntry +// @Failure 500 {object} handlers.APIError +// @Router /stats/clock-drift [get] +func getStatsClockDrift(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + iatas := parseIATAs(r) + if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") + respondRegionError(w, err) return } - limit = int32(l) + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } - advertisers, err := reader.GetStatsTopAdvertisers(r.Context(), iatas, since, limit) + limit, err := parseLimit(r, 10) if err != nil { - log.Printf("api: GetStatsTopAdvertisers failed: %v", err) + respondError(w, http.StatusBadRequest, err.Error()) + return + } + entries, err := reader.GetStatsClockDrift(r.Context(), iatas, limit) + if err != nil { + slog.Error("api: GetStatsClockDrift failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } - respond(w, http.StatusOK, advertisers) + respond(w, http.StatusOK, entries) } } @@ -312,7 +380,7 @@ func getStatsTopAdvertisers(reader api.Reader) http.HandlerFunc { // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param since query int false "Start of window epoch ms (default last 24h)" -// @Param limit query int false "Max results (default 10)" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopTalker // @Failure 500 {object} handlers.APIError // @Router /stats/top-talkers [get] @@ -322,32 +390,27 @@ func getStatsTopTalkers(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) } - var limit int32 = 10 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + limit, err := parseLimit(r, 10) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } talkers, err := reader.GetStatsTopTalkers(r.Context(), iatas, since, limit) if err != nil { - log.Printf("api: GetStatsTopTalkers failed: %v", err) + slog.Error("api: GetStatsTopTalkers failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -374,10 +437,13 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } presets, err := reader.GetRadioPresets(r.Context(), preset, iatas) if err != nil { @@ -391,18 +457,50 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // getStatsScopes godoc // // @Summary Scope statistics +// @Description Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour and adds the distinct observers and advertising nodes active in the scope each hour (empty hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. // @Tags Stats // @Produce json +// @Param iatas query string false "Comma-separated IATA codes" +// @Param iata query string false "Single IATA code; used when iatas is absent" +// @Param regionId query int false "Filter by region ID, expands to member IATAs" +// @Param region query string false "Filter by region slug, expands to member IATAs; combined with explicit IATAs" +// @Param since query int false "Start of packet window epoch ms, rounded down to the hour (default 7 days ago)" // @Success 200 {object} []api.ScopeStats +// @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /stats/scopes [get] -func getStatsScopes(reader api.Reader) http.HandlerFunc { +func getStatsScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - stats, err := reader.GetScopeStats(r.Context()) + iatas := parseIATAs(r) + if regionID := r.URL.Query().Get("regionId"); regionID != "" || r.URL.Query().Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(r.Context(), regionID, r.URL.Query().Get("region"), reader) + if err != nil { + respondRegionError(w, err) + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + respond(w, http.StatusOK, []api.ScopeStats{}) + return + } + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + stats, err := reader.GetScopeStats(r.Context(), iatas, since) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return } + if len(iatas) > 0 { + members := scopeNamesFor(scopes, iatas) + stats = slices.DeleteFunc(slices.Clone(stats), func(s api.ScopeStats) bool { + _, found := slices.BinarySearch(members, s.Name) + return !found + }) + } respond(w, http.StatusOK, stats) } } @@ -424,10 +522,13 @@ func getStatsNodeTypes(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } result, err := reader.GetStatsNodeTypes(r.Context(), iatas) if err != nil { diff --git a/internal/api/handlers/stats_region_test.go b/internal/api/handlers/stats_region_test.go new file mode 100644 index 00000000..4ed483f4 --- /dev/null +++ b/internal/api/handlers/stats_region_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// iataRecorder records the IATA filter each stats read receives. +type iataRecorder struct { + stubReader + got *[]string +} + +func (r iataRecorder) rec(iatas []string) { *r.got = append([]string{}, iatas...) } + +func (r iataRecorder) GetStatsOverview(_ context.Context, iatas []string) (*api.StatsOverview, error) { + r.rec(iatas) + return &api.StatsOverview{}, nil +} +func (r iataRecorder) GetStatsObservations(_ context.Context, iatas []string, _ time.Time) ([]api.ObservationPoint, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsPayloadBreakdown(_ context.Context, iatas []string, _ time.Time) ([]api.PayloadBreakdownItem, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopNodes(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopNode, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopObservers(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopObserver, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopAdvertisers(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopAdvertiser, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsClockDrift(_ context.Context, iatas []string, _ int32) ([]api.ClockDriftEntry, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopTalkers(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopTalker, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetRadioPresets(_ context.Context, _ string, iatas []string) ([]api.RadioPreset, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsNodeTypes(_ context.Context, iatas []string) ([]api.NodeTypeCount, error) { + r.rec(iatas) + return nil, nil +} + +// A region with no IATAs must filter to nothing, not fall back to every IATA. +func TestStatsEmptyRegionMatchesNothing(t *testing.T) { + for _, path := range []string{"overview", "observations", "payload-breakdown", "top-nodes", "top-observers", + "top-advertisers", "clock-drift", "top-talkers", "radio-presets", "node-types"} { + for _, tc := range []struct { + region string + want []string + }{ + {"empty", []string{""}}, + {"west", []string{"YVR"}}, + } { + var got []string + reader := iataRecorder{got: &got, stubReader: stubReader{getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }}} + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/"+path+"?region="+tc.region, nil)) + if w.Code != 200 || !reflect.DeepEqual(got, tc.want) { + t.Errorf("/stats/%s?region=%s: status %d, iatas %q; want 200, %q", path, tc.region, w.Code, got, tc.want) + } + } + } +} diff --git a/internal/api/handlers/stats_series.go b/internal/api/handlers/stats_series.go new file mode 100644 index 00000000..961d58ad --- /dev/null +++ b/internal/api/handlers/stats_series.go @@ -0,0 +1,79 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +const defaultSeriesWindow = 90 * 24 * time.Hour + +// getStatsSeries godoc +// +// @Summary Hourly network activity from the analytics rollups +// @Description One entry per UTC hour in [since, until) plus a summary over the complete hours. Both bounds round down to UTC hours and since is clamped to the rollup retention; the response reports the effective window. An hour is rolled about 95 minutes after it closes, so recent hours are "missing" until then; "partial" hours lost raw rows before they could be rolled and never get values. Counts (observations, packets) sum across hours, with a packet counted once in each hour it was heard; observers, IATAs and scopes are distinct across the whole window; averages are sum / samples. revision changes whenever any rolled hour changes. +// @Tags Stats +// @Produce json +// @Param since query int true "Inclusive start, epoch milliseconds" +// @Param until query int true "Exclusive end, epoch milliseconds; the window is at most the rollup retention (default 90 days)" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.StatsSeries +// @Failure 400 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/series [get] +func getStatsSeries(reader api.Reader, maxWindow time.Duration) http.HandlerFunc { + if maxWindow <= 0 { + maxWindow = defaultSeriesWindow + } + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + since, until, err := parseStatsWindow(r, maxWindow) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + if oldest := time.Now().UTC().Add(-maxWindow).Truncate(time.Hour); since.Before(oldest) { + since = oldest + } + if !since.Before(until) { + since = until + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondRegionError(w, err) + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } + } + series, err := reader.GetStatsSeries(ctx, since, until, iatas) + switch { + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "series query timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("stats series failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, series) + } + } +} diff --git a/internal/api/handlers/stats_series_test.go b/internal/api/handlers/stats_series_test.go new file mode 100644 index 00000000..68b34f80 --- /dev/null +++ b/internal/api/handlers/stats_series_test.go @@ -0,0 +1,77 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "fmt" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +func TestGetStatsSeries(t *testing.T) { + now := time.Now().UTC() + until := now.Truncate(time.Hour) + since := until.Add(-48 * time.Hour) + window := fmt.Sprintf("since=%d&until=%d", since.UnixMilli()+1234, until.UnixMilli()+1234) + for _, tc := range []struct { + name, query string + maxWindow time.Duration + wantSince time.Time + iatas []string + err error + status int + }{ + {"global", window, 0, since, nil, nil, 200}, + {"IATAs", window + "&iatas=yvr,yyj", 0, since, []string{"YVR", "YYJ"}, nil, 200}, + {"region", window + "®ion=west", 0, since, []string{"YVR"}, nil, 200}, + {"empty region matches nothing", window + "®ion=empty", 0, since, []string{""}, nil, 200}, + {"missing region", window + "®ion=missing", 0, since, nil, nil, 400}, + {"missing until", fmt.Sprintf("since=%d", since.UnixMilli()), 0, since, nil, nil, 400}, + {"window over retention", window, 24 * time.Hour, since, nil, nil, 400}, + // The window fits, but its start predates what the rollups still hold. + {"since clamped to retention", fmt.Sprintf("since=%d&until=%d", until.Add(-50*time.Hour).UnixMilli(), until.Add(-2*time.Hour).UnixMilli()), + 48 * time.Hour, until.Add(-48 * time.Hour), nil, nil, 200}, + {"timeout", window, 0, since, nil, context.DeadlineExceeded, 503}, + {"database error", window, 0, since, nil, errors.New("private detail"), 500}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := stubReader{ + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + switch slug { + case "empty": + return &api.Region{}, nil + case "missing": + return nil, pgx.ErrNoRows + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }, + getStatsSeries: func(ctx context.Context, gotSince, gotUntil time.Time, iatas []string) (*api.StatsSeries, error) { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { + t.Error("missing bounded query deadline") + } + if !gotSince.Equal(tc.wantSince) || !reflect.DeepEqual(iatas, tc.iatas) { + t.Errorf("args %s %v; want %s %v", gotSince, iatas, tc.wantSince, tc.iatas) + } + if gotUntil.Sub(gotSince) > 48*time.Hour || gotUntil.Minute() != 0 { + t.Errorf("until %s not snapped within the window", gotUntil) + } + return &api.StatsSeries{Since: gotSince.UnixMilli()}, tc.err + }, + } + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{SeriesWindow: tc.maxWindow}).ServeHTTP(w, httptest.NewRequest("GET", "/series?"+tc.query, nil)) + if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + }) + } +} diff --git a/internal/api/handlers/stats_test.go b/internal/api/handlers/stats_test.go index d4e572ea..d6019596 100644 --- a/internal/api/handlers/stats_test.go +++ b/internal/api/handlers/stats_test.go @@ -117,7 +117,7 @@ func TestGetStatsPayloadBreakdown_OK(t *testing.T) { func TestGetStatsTopNodes_OK(t *testing.T) { r := chi.NewRouter() r.Get("/stats/top-nodes", getStatsTopNodes(stubReader{ - getStatsTopNodes: func(_ context.Context, _ []string, _ int32) ([]api.TopNode, error) { + getStatsTopNodes: func(_ context.Context, _ []string, _ time.Time, _ int32) ([]api.TopNode, error) { return []api.TopNode{{IATA: "YVR", ObservationCount: 50}}, nil }, })) @@ -181,6 +181,52 @@ func TestGetStatsTopAdvertisers_OK(t *testing.T) { } } +func TestGetStatsClockDrift_OK(t *testing.T) { + r := chi.NewRouter() + r.Get("/stats/clock-drift", getStatsClockDrift(stubReader{ + getStatsClockDrift: func(_ context.Context, _ []string, _ int32) ([]api.ClockDriftEntry, error) { + return []api.ClockDriftEntry{{ClockDriftSeconds: -600}}, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/stats/clock-drift", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected 200, got %d", w.Code) + } +} + +func TestGetStatsClockDrift_MultiIATA_PassedThrough(t *testing.T) { + var gotIATAs []string + r := chi.NewRouter() + r.Get("/stats/clock-drift", getStatsClockDrift(stubReader{ + getStatsClockDrift: func(_ context.Context, iatas []string, _ int32) ([]api.ClockDriftEntry, error) { + gotIATAs = iatas + return nil, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/stats/clock-drift?iatas=YVR,YYJ", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected 200, got %d", w.Code) + } + if len(gotIATAs) != 2 || gotIATAs[0] != "YVR" || gotIATAs[1] != "YYJ" { + t.Errorf("expected [YVR YYJ], got %v", gotIATAs) + } +} + +func TestGetStatsClockDrift_InvalidLimit(t *testing.T) { + r := chi.NewRouter() + r.Get("/stats/clock-drift", getStatsClockDrift(stubReader{})) + req := httptest.NewRequest(http.MethodGet, "/stats/clock-drift?limit=notanint", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Errorf("expected 400, got %d", w.Code) + } +} + func TestGetStatsTopTalkers_InvalidSince(t *testing.T) { r := chi.NewRouter() r.Get("/stats/top-talkers", getStatsTopTalkers(stubReader{})) @@ -236,10 +282,10 @@ func TestGetStatsRadioPresets_OK(t *testing.T) { func TestGetStatsScopes_OK(t *testing.T) { r := chi.NewRouter() r.Get("/stats/scopes", getStatsScopes(stubReader{ - getScopeStats: func(_ context.Context) ([]api.ScopeStats, error) { + getScopeStats: func(_ context.Context, _ []string, _ time.Time) ([]api.ScopeStats, error) { return []api.ScopeStats{{Name: "#bc", PacketCount: 100}}, nil }, - })) + }, nil)) req := httptest.NewRequest(http.MethodGet, "/stats/scopes", nil) w := httptest.NewRecorder() r.ServeHTTP(w, req) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index ba720b30..efb5cb8f 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -16,13 +16,16 @@ import ( // Unset fields return zero values. Use it for both validation tests // (leave all fields nil) and happy path tests (set only what you need). type stubReader struct { + getStatsSeries func(context.Context, time.Time, time.Time, []string) (*api.StatsSeries, error) + getRouteEvidence func(context.Context, string, string, api.RouteEvidenceQuery) (*api.RouteEvidence, error) + getObserverComparison func(context.Context, uuid.UUID, uuid.UUID, time.Time, time.Time, []string) (*api.ObserverComparison, error) listIATAs func(ctx context.Context) ([]api.IATA, error) getIATA func(ctx context.Context, iata string) (*api.IATA, error) getIATABorder func(ctx context.Context, iata string) (json.RawMessage, error) listRegions func(ctx context.Context) ([]api.RegionSummary, error) getRegion func(ctx context.Context, regionID int32) (*api.Region, error) getRegionBySlug func(ctx context.Context, slug string) (*api.Region, error) - listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) + listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) getChannel func(ctx context.Context, channelID int32) (*api.Channel, error) listChannelMessages func(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) listChannelMessagesByHash func(ctx context.Context, hash []byte, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) @@ -31,6 +34,7 @@ type stubReader struct { getObserver func(ctx context.Context, observerID uuid.UUID) (*api.Observer, error) getObserverTelemetry func(ctx context.Context, observerID uuid.UUID, since, until time.Time, afterID int64) (*api.ObserverTelemetry, error) getObserverTelemetryBucketed func(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]api.ObserverTelemetryPoint, error) + getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) getObserverScopes func(ctx context.Context, observerID uuid.UUID) ([]string, error) listObserverAdverts func(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) listNodes func(ctx context.Context, nodeType int16, iatas []string, supportsMultibytePaths, supportsMultibyteTraces *bool, pubkey []byte, pubkeyPrefix, name, scope string, cursor int64, limit int32, includeNeighbors bool) (api.Page[api.NodeSummary], error) @@ -44,18 +48,18 @@ type stubReader struct { getStatsOverview func(ctx context.Context, iatas []string) (*api.StatsOverview, error) getStatsObservations func(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) getStatsPayloadBreakdown func(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) - getStatsTopNodes func(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) + getStatsTopNodes func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) getStatsTopObservers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) getStatsTopAdvertisers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) + getStatsClockDrift func(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) getStatsTopTalkers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) - getScopeStats func(ctx context.Context) ([]api.ScopeStats, error) + getScopeStats func(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) getStatsNodeTypes func(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) getScopeNames func(ctx context.Context) ([]string, error) - getScopesByIATAs func(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) getScopeByName func(ctx context.Context, name string) (*api.ScopeDetail, error) - listTraceTags func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) + listTraceTags func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) getTraceByTag func(ctx context.Context, tag string) (*api.TraceDetail, error) - listKnownRoutes func(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) + listKnownRoutes func(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]api.KnownRoute, error) searchKnownRoutes func(ctx context.Context, iata, fromHash, toHash string) ([]api.KnownRoute, error) getKnownRoutesByNode func(ctx context.Context, iata string, nodeID uuid.UUID) ([]api.KnownRoute, error) getCrossIATANeighbors func(ctx context.Context, nodeID uuid.UUID, iata string) ([]api.NodeNeighbor, error) @@ -63,6 +67,28 @@ type stubReader struct { getNodesByIDs func(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]*api.ResolvedNode, error) } +func (s stubReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { + return nil, nil +} + +func (s stubReader) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { + if s.getStatsSeries != nil { + return s.getStatsSeries(ctx, since, until, iatas) + } + return &api.StatsSeries{}, nil +} + +func (s stubReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { + return nil, nil +} + +func (s stubReader) GetObserverComparison(ctx context.Context, a, b uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { + if s.getObserverComparison != nil { + return s.getObserverComparison(ctx, a, b, since, until, iatas) + } + return nil, nil +} + func (s stubReader) ListIATAs(ctx context.Context) ([]api.IATA, error) { if s.listIATAs != nil { return s.listIATAs(ctx) @@ -105,11 +131,11 @@ func (s stubReader) GetRegionBySlug(ctx context.Context, slug string) (*api.Regi return nil, nil } -func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) { +func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { if s.listChannels != nil { - return s.listChannels(ctx, limit, hash, iatas, cursor) + return s.listChannels(ctx, limit, hash, iatas, keyKnown, cursor, pageCursor) } - return api.Page[api.ChannelSummary]{}, nil + return api.ChannelPage{}, nil } func (s stubReader) GetChannel(ctx context.Context, channelID int32) (*api.Channel, error) { @@ -168,6 +194,13 @@ func (s stubReader) GetObserverTelemetryBucketed(ctx context.Context, observerID return nil, nil } +func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { + if s.getObserverActivity != nil { + return s.getObserverActivity(ctx, observerID, window, interval, until) + } + return nil, nil +} + func (s stubReader) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) { if s.getObserverScopes != nil { return s.getObserverScopes(ctx, observerID) @@ -259,9 +292,9 @@ func (s stubReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string return nil, nil } -func (s stubReader) GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { +func (s stubReader) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { if s.getStatsTopNodes != nil { - return s.getStatsTopNodes(ctx, iatas, limit) + return s.getStatsTopNodes(ctx, iatas, since, limit) } return nil, nil } @@ -280,6 +313,13 @@ func (s stubReader) GetStatsTopAdvertisers(ctx context.Context, iatas []string, return nil, nil } +func (s stubReader) GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { + if s.getStatsClockDrift != nil { + return s.getStatsClockDrift(ctx, iatas, limit) + } + return nil, nil +} + func (s stubReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { if s.getStatsTopTalkers != nil { return s.getStatsTopTalkers(ctx, iatas, since, limit) @@ -287,9 +327,9 @@ func (s stubReader) GetStatsTopTalkers(ctx context.Context, iatas []string, sinc return nil, nil } -func (s stubReader) GetScopeStats(ctx context.Context) ([]api.ScopeStats, error) { +func (s stubReader) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { if s.getScopeStats != nil { - return s.getScopeStats(ctx) + return s.getScopeStats(ctx, iatas, since) } return nil, nil } @@ -308,13 +348,6 @@ func (s stubReader) GetScopeNames(ctx context.Context) ([]string, error) { return nil, nil } -func (s stubReader) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) { - if s.getScopesByIATAs != nil { - return s.getScopesByIATAs(ctx, iatas) - } - return nil, nil -} - func (s stubReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { if s.getScopeByName != nil { return s.getScopeByName(ctx, name) @@ -322,9 +355,9 @@ func (s stubReader) GetScopeByName(ctx context.Context, name string) (*api.Scope return nil, nil } -func (s stubReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) { +func (s stubReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { if s.listTraceTags != nil { - return s.listTraceTags(ctx, iatas, scope, traceType, since, until, cursor, limit) + return s.listTraceTags(ctx, iatas, scope, traceType, since, until, cursor, cursorTag, limit) } return nil, nil } @@ -336,9 +369,9 @@ func (s stubReader) GetTraceByTag(ctx context.Context, tag string) (*api.TraceDe return nil, nil } -func (s stubReader) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) { +func (s stubReader) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]api.KnownRoute, error) { if s.listKnownRoutes != nil { - return s.listKnownRoutes(ctx, iata, hopCount, cursor, limit) + return s.listKnownRoutes(ctx, iata, hopCount, cursor, cursorID, limit) } return nil, nil } @@ -377,3 +410,10 @@ func (s stubReader) GetNodesByIDs(ctx context.Context, ids []uuid.UUID) (map[uui } return nil, nil } + +func (s stubReader) GetRouteEvidence(ctx context.Context, iata, key string, q api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + if s.getRouteEvidence != nil { + return s.getRouteEvidence(ctx, iata, key, q) + } + return nil, nil +} diff --git a/internal/api/handlers/traces.go b/internal/api/handlers/traces.go index 3fd39ab5..9ce002db 100644 --- a/internal/api/handlers/traces.go +++ b/internal/api/handlers/traces.go @@ -4,6 +4,7 @@ package handlers import ( + "encoding/hex" "net/http" "strconv" "strings" @@ -32,22 +33,23 @@ func TracesRouter(reader api.Reader) http.Handler { // @Param iatas query string false "Filter by IATA code(s), comma-separated" // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" -// @Param scope query string false "Filter by transport scope name" -// @Param type query string false "Filter by type: TRACE or PING (default: all)" +// @Param scope query string false "Filter by the tag's first transport scope name" +// @Param type query string false "Filter by type: TRACE (any multi-hop packet) or PING (default: all)" // @Param since query int false "Filter by first_heard_at >= since (epoch ms)" // @Param until query int false "Filter by first_heard_at <= until (epoch ms)" -// @Param cursor query int false "last_heard_at epoch ms of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param cursor query int false "lastHeardAt (epoch ms) of the last item; returns older tags" +// @Param cursorTag query string false "traceTag (hex) of the last item; with cursor, also returns later tags sharing that millisecond" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) +// @Failure 400 {object} handlers.APIError // @Success 200 {object} []api.TraceTagSummary // @Failure 500 {object} handlers.APIError // @Router /traces [get] func listTraceTags(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - var limit int32 = 50 - if l := r.URL.Query().Get("limit"); l != "" { - if v, err := strconv.ParseInt(l, 10, 32); err == nil { - limit = int32(v) - } + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var since, until, cursor time.Time if v := r.URL.Query().Get("since"); v != "" { @@ -65,18 +67,30 @@ func listTraceTags(reader api.Reader) http.HandlerFunc { cursor = time.UnixMilli(ms) } } + // cursorTag breaks ties within the cursor's millisecond. + cursorTag := strings.ToLower(r.URL.Query().Get("cursorTag")) + if cursorTag != "" { + if b, err := hex.DecodeString(cursorTag); err != nil || len(b) == 0 { + respondError(w, http.StatusBadRequest, "cursorTag must be hex") + return + } + if cursor.IsZero() { + respondError(w, http.StatusBadRequest, "cursorTag requires cursor") + return + } + } iatas := parseIATAs(r) if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) } scope := r.URL.Query().Get("scope") traceType := strings.ToUpper(r.URL.Query().Get("type")) - tags, err := reader.ListTraceTags(r.Context(), iatas, scope, traceType, since, until, cursor, limit) + tags, err := reader.ListTraceTags(r.Context(), iatas, scope, traceType, since, until, cursor, cursorTag, limit) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/traces_test.go b/internal/api/handlers/traces_test.go index 722d8d3d..c69c169f 100644 --- a/internal/api/handlers/traces_test.go +++ b/internal/api/handlers/traces_test.go @@ -17,7 +17,7 @@ import ( func TestListTraceTags_OK(t *testing.T) { r := chi.NewRouter() r.Get("/traces", listTraceTags(stubReader{ - listTraceTags: func(_ context.Context, _ []string, _, _ string, _, _ time.Time, _ time.Time, _ int32) ([]api.TraceTagSummary, error) { + listTraceTags: func(_ context.Context, _ []string, _, _ string, _, _ time.Time, _ time.Time, _ string, _ int32) ([]api.TraceTagSummary, error) { return []api.TraceTagSummary{{TraceTag: "trace-001"}}, nil }, })) @@ -29,6 +29,39 @@ func TestListTraceTags_OK(t *testing.T) { } } +func TestListTraceTags_CursorTag(t *testing.T) { + var gotCursor time.Time + var gotTag string + r := chi.NewRouter() + r.Get("/traces", listTraceTags(stubReader{ + listTraceTags: func(_ context.Context, _ []string, _, _ string, _, _ time.Time, cursor time.Time, cursorTag string, _ int32) ([]api.TraceTagSummary, error) { + gotCursor, gotTag = cursor, cursorTag + return nil, nil + }, + })) + for _, tc := range []struct { + query string + code int + tag string + }{ + {"cursor=1700000000000&cursorTag=A1B2C3D4", http.StatusOK, "a1b2c3d4"}, + {"cursor=1700000000000", http.StatusOK, ""}, + {"cursor=1700000000000&cursorTag=zz", http.StatusBadRequest, ""}, + {"cursorTag=a1b2c3d4", http.StatusBadRequest, ""}, + } { + gotTag = "" + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/traces?"+tc.query, nil)) + if w.Code != tc.code { + t.Errorf("%s: code %d, want %d", tc.query, w.Code, tc.code) + continue + } + if tc.code == http.StatusOK && (gotTag != tc.tag || gotCursor.UnixMilli() != 1700000000000) { + t.Errorf("%s: cursor %v tag %q", tc.query, gotCursor, gotTag) + } + } +} + func TestGetTrace_OK(t *testing.T) { r := chi.NewRouter() r.Get("/traces/{tag}", getTrace(stubReader{ diff --git a/internal/api/local_borders.go b/internal/api/local_borders.go new file mode 100644 index 00000000..65e44a0d --- /dev/null +++ b/internal/api/local_borders.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "context" + "slices" + + "github.com/MeshCore-Beacon/beacon-server/internal/borders" + "github.com/google/uuid" +) + +// WithLocalBorders annotates node responses after cache reads. Historical rows +// use the current boundaries without extra SQL or cached-policy drift. +// Cached values are copied; the decorator never changes its inner reader's data. +func WithLocalBorders(reader Reader, local *borders.Live) Reader { + if local == nil { + return reader + } + return &localBorderReader{Reader: reader, local: local} +} + +type localBorderReader struct { + Reader + local *borders.Live +} + +func (r *localBorderReader) ListNodes(ctx context.Context, nodeType int16, iatas []string, paths, traces *bool, pubkey []byte, prefix, name, scope string, cursor int64, limit int32, neighbors bool) (Page[NodeSummary], error) { + page, err := r.Reader.ListNodes(ctx, nodeType, iatas, paths, traces, pubkey, prefix, name, scope, cursor, limit, neighbors) + if err != nil { + return page, err + } + page.Items = slices.Clone(page.Items) + for i := range page.Items { + node := &page.Items[i] + node.PossiblyForeign = r.local.PossiblyForeign(node.NodeType, node.Latitude, node.Longitude) + } + return page, nil +} + +func (r *localBorderReader) GetNode(ctx context.Context, id uuid.UUID) (*Node, error) { + node, err := r.Reader.GetNode(ctx, id) + if err != nil || node == nil { + return node, err + } + copy := *node + copy.PossiblyForeign = r.local.PossiblyForeign(node.NodeType, node.Latitude, node.Longitude) + return ©, nil +} diff --git a/internal/api/local_borders_test.go b/internal/api/local_borders_test.go new file mode 100644 index 00000000..d7f0b013 --- /dev/null +++ b/internal/api/local_borders_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api_test + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/borders" + "github.com/MeshCore-Beacon/beacon-server/internal/cache" + "github.com/alicebob/miniredis/v2" + "github.com/google/uuid" + "github.com/paulmach/orb" +) + +type borderReader struct { + api.Reader + node *api.Node + page api.Page[api.NodeSummary] + getCalls, listCalls int + err error +} + +func (r *borderReader) GetNode(context.Context, uuid.UUID) (*api.Node, error) { + r.getCalls++ + return r.node, r.err +} +func (r *borderReader) ListNodes(_ context.Context, _ int16, _ []string, _, _ *bool, _ []byte, _, _, _ string, _ int64, _ int32, _ bool) (api.Page[api.NodeSummary], error) { + r.listCalls++ + return r.page, r.err +} + +func TestLocalBordersAfterCache(t *testing.T) { + ctx := context.Background() + local, err := borders.New([]orb.Polygon{{{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}}}}) + if err != nil { + t.Fatal(err) + } + other, err := borders.New([]orb.Polygon{{{{30, 30}, {40, 30}, {40, 40}, {30, 40}, {30, 30}}}}) + if err != nil { + t.Fatal(err) + } + node := &api.Node{NodeSummary: api.NodeSummary{ID: uuid.New(), NodeType: 2, Latitude: new(15.0), Longitude: new(15.0), IATAs: []api.NodeIATA{{IATA: "ZZZ", LastHeard: 1}}}} + base := &borderReader{node: node, page: api.Page[api.NodeSummary]{Items: []api.NodeSummary{node.NodeSummary}, HasMore: true, NextCursor: new(int64(123))}} + mr := miniredis.RunT(t) + client := cache.NewClient(mr.Addr(), "", 0) + t.Cleanup(func() { client.Close() }) + cached := cache.NewCachedReader(base, client, cache.CacheTTLs{Nodes: time.Hour}) + first := api.WithLocalBorders(cached, borders.NewLive(local)) + second := api.WithLocalBorders(cached, borders.NewLive(other)) + if api.WithLocalBorders(cached, nil) != cached { + t.Fatal("disabled decorator changed reader") + } + for _, tc := range []struct { + reader api.Reader + want bool + }{{first, false}, {second, true}} { + got, err := tc.reader.GetNode(ctx, node.ID) + if err != nil { + t.Fatal(err) + } + if got.PossiblyForeign == nil || *got.PossiblyForeign != tc.want || got.IATAs[0].IATA != "ZZZ" { + t.Fatalf("detail: %+v", got) + } + page, err := tc.reader.ListNodes(ctx, 2, []string{"ZZZ"}, nil, nil, nil, "", "", "", 0, 50, false) + if err != nil { + t.Fatal(err) + } + if !page.HasMore || *page.NextCursor != 123 || page.Items[0].PossiblyForeign == nil || *page.Items[0].PossiblyForeign != tc.want { + t.Fatalf("page: %+v", page) + } + data, _ := json.Marshal(got) + if !strings.Contains(string(data), `"possiblyForeign":`) { + t.Fatal("known false was omitted") + } + } + if base.getCalls != 1 || base.listCalls != 2 { + t.Fatalf("extra reads: get=%d list=%d", base.getCalls, base.listCalls) + } + if node.PossiblyForeign != nil || base.page.Items[0].PossiblyForeign != nil { + t.Fatal("shared reader data was mutated") + } + disabled, err := cached.GetNode(ctx, node.ID) + if err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(disabled) + if strings.Contains(string(data), "possiblyForeign") { + t.Fatal("classification leaked into cached data") + } + // Existing ingest invalidation still supplies fresh coordinates to the wrapper. + base.node.Longitude = new(35.0) + cached.(*cache.CachedReader).InvalidateNode(ctx, node.ID) + updated, err := first.GetNode(ctx, node.ID) + if err != nil || updated.PossiblyForeign == nil || !*updated.PossiblyForeign || base.getCalls != 2 { + t.Fatal("updated coordinates not classified") + } +} + +func TestLocalBordersPreserveReaderErrors(t *testing.T) { + local, err := borders.New([]orb.Polygon{{{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}}}}) + if err != nil { + t.Fatal(err) + } + want := errors.New("reader unavailable") + base := &borderReader{err: want} + reader := api.WithLocalBorders(base, borders.NewLive(local)) + if _, err := reader.GetNode(context.Background(), uuid.New()); !errors.Is(err, want) { + t.Fatal(err) + } + if _, err := reader.ListNodes(context.Background(), 0, nil, nil, nil, nil, "", "", "", 0, 1, false); !errors.Is(err, want) { + t.Fatal(err) + } + base.err = nil + if node, err := reader.GetNode(context.Background(), uuid.New()); node != nil || err != nil { + t.Fatal("nil result changed") + } +} diff --git a/internal/api/middleware/auth.go b/internal/api/middleware/auth.go index efa49c69..3e775f3d 100644 --- a/internal/api/middleware/auth.go +++ b/internal/api/middleware/auth.go @@ -3,17 +3,36 @@ package middleware -import "net/http" +import ( + "crypto/subtle" + "io" + "net/http" + "strings" +) -// NoopAuth is a placeholder for the authentication middleware that will be -// wired onto the private route group when auth is implemented (see Future -// Features → Admin authentication in the design doc). -// -// Replace this with a real JWT/session validation middleware before shipping -// any write endpoints or admin functionality. -func NoopAuth(next http.Handler) http.Handler { +// BearerAuth protects admin routes with one operator key. An empty key disables +// access; public routes must be mounted outside this middleware. +func BearerAuth(apiKey string, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // TODO: validate bearer token, set user in context, return 401 on failure. - next.ServeHTTP(w, r) + w.Header().Set("Cache-Control", "no-store") + status := http.StatusServiceUnavailable + body := `{"error":{"code":"service_unavailable","message":"admin authentication is not configured"}}` + if apiKey != "" { + if values := r.Header.Values("Authorization"); len(values) == 1 { + scheme, token, found := strings.Cut(values[0], " ") + token = strings.TrimLeft(token, " ") // Bearer permits one or more spaces. + if found && strings.EqualFold(scheme, "Bearer") && token != "" && !strings.ContainsAny(token, " \t\r\n,") && + subtle.ConstantTimeCompare([]byte(token), []byte(apiKey)) == 1 { + next.ServeHTTP(w, r) + return + } + } + status = http.StatusUnauthorized + body = `{"error":{"code":"unauthorized","message":"valid bearer token required"}}` + w.Header().Set("WWW-Authenticate", "Bearer") + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, body+"\n") }) } diff --git a/internal/api/middleware/auth_test.go b/internal/api/middleware/auth_test.go new file mode 100644 index 00000000..e394d923 --- /dev/null +++ b/internal/api/middleware/auth_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestBearerAuth(t *testing.T) { + const key = "synthetic-test-key" + for _, tc := range []struct { + name, key string + headers []string + status int + }{ + {"disabled", "", nil, 503}, {"disabled with token", "", []string{"Bearer " + key}, 503}, + {"missing", key, nil, 401}, {"empty", key, []string{""}, 401}, + {"wrong scheme", key, []string{"Basic " + key}, 401}, {"wrong token", key, []string{"Bearer wrong"}, 401}, + {"empty token", key, []string{"Bearer "}, 401}, {"no separator", key, []string{"Bearer" + key}, 401}, + {"tab separator", key, []string{"Bearer\t" + key}, 401}, {"extra token", key, []string{"Bearer " + key + " extra"}, 401}, + {"line break", key, []string{"Bearer " + key + "\r\n"}, 401}, + {"duplicate", key, []string{"Bearer " + key, "Bearer " + key}, 401}, + {"combined", key, []string{"Bearer " + key + ", Bearer " + key}, 401}, + {"case sensitive token", key, []string{"Bearer SYNTHETIC-TEST-KEY"}, 401}, + {"valid", key, []string{"Bearer " + key}, 200}, + {"scheme case", key, []string{"bEaReR " + key}, 200}, + {"multiple spaces", key, []string{"Bearer " + key}, 200}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + handler := BearerAuth(tc.key, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.WriteHeader(200) + })) + // Query and form tokens alone must never grant access. + req := httptest.NewRequest("POST", "/admin?access_token="+key, strings.NewReader("access_token="+key)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + for _, value := range tc.headers { + req.Header.Add("Authorization", value) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != tc.status || (calls == 1) != (tc.status == 200) { + t.Fatalf("status=%d downstream calls=%d", w.Code, calls) + } + if w.Header().Get("Cache-Control") != "no-store" || strings.Contains(w.Body.String(), key) { + t.Fatal("cache policy missing or credential reflected") + } + if tc.status == 200 { + return + } + var body struct { + Error struct{ Code, Message string } + } + if w.Header().Get("Content-Type") != "application/json" || json.Unmarshal(w.Body.Bytes(), &body) != nil || body.Error.Message == "" { + t.Fatal("invalid JSON error contract") + } + wantCode := "service_unavailable" + if tc.status == 401 { + wantCode = "unauthorized" + if w.Header().Get("WWW-Authenticate") != "Bearer" { + t.Fatal("missing bearer challenge") + } + } + if body.Error.Code != wantCode { + t.Fatalf("error code=%s", body.Error.Code) + } + }) + } +} diff --git a/internal/api/middleware/logging.go b/internal/api/middleware/logging.go new file mode 100644 index 00000000..cfff07df --- /dev/null +++ b/internal/api/middleware/logging.go @@ -0,0 +1,63 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "fmt" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/go-chi/chi/v5" + chimw "github.com/go-chi/chi/v5/middleware" +) + +// RequestLogger uses chi's response wrapper to preserve streaming/hijacking support. +// Place it after TrustedProxyIP and RequestID, and before Recoverer. +func RequestLogger(next http.Handler) http.Handler { + return chimw.RequestLogger(requestLogFormatter{})(next) +} + +type requestLogFormatter struct{} +type requestLogEntry struct{ request *http.Request } + +func (requestLogFormatter) NewLogEntry(r *http.Request) chimw.LogEntry { + return &requestLogEntry{request: r} +} + +func (e *requestLogEntry) logger() *slog.Logger { + path := "unmatched" + if ctx := chi.RouteContext(e.request.Context()); ctx != nil && ctx.RoutePattern() != "" { + path = ctx.RoutePattern() + } + return slog.Default().With("component", "http", "method", singleLineLogValue(e.request.Method), "path", path, + "client_ip", singleLineLogValue(e.request.RemoteAddr), "request_id", singleLineLogValue(chimw.GetReqID(e.request.Context()))) +} + +// Strip line breaks before request fields reach a handler or downstream log consumer. +func singleLineLogValue(value string) string { + value = strings.ReplaceAll(value, "\r", "") + return strings.ReplaceAll(value, "\n", "") +} + +func (e *requestLogEntry) Write(status, bytes int, _ http.Header, elapsed time.Duration, _ any) { + if status == 0 { + status = http.StatusOK + } + level := slog.LevelInfo + if status >= 500 { + level = slog.LevelError + } else if status >= 400 { + level = slog.LevelWarn + } + if !slog.Default().Enabled(e.request.Context(), level) { + return + } + e.logger().Log(e.request.Context(), level, "request complete", "status", status, "bytes", bytes, "duration_ms", float64(elapsed)/float64(time.Millisecond)) +} + +func (e *requestLogEntry) Panic(value any, stack []byte) { + e.logger().ErrorContext(e.request.Context(), "request panic", "panic", fmt.Sprint(value), "stack", string(stack)) +} diff --git a/internal/api/middleware/logging_test.go b/internal/api/middleware/logging_test.go new file mode 100644 index 00000000..f949389a --- /dev/null +++ b/internal/api/middleware/logging_test.go @@ -0,0 +1,143 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "bytes" + "encoding/json" + "log" + "log/slog" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" + + "github.com/go-chi/chi/v5" + chimw "github.com/go-chi/chi/v5/middleware" +) + +func TestRequestLogging(t *testing.T) { + for _, tc := range []struct { + status int + panic bool + level slog.Level + wantLevel string + }{ + {200, false, slog.LevelInfo, "INFO"}, {429, false, slog.LevelWarn, "WARN"}, + {500, false, slog.LevelError, "ERROR"}, {200, false, slog.LevelWarn, ""}, {500, true, slog.LevelError, "ERROR"}, + } { + t.Run(tc.level.String()+"/"+http.StatusText(tc.status), func(t *testing.T) { + var out bytes.Buffer + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, &slog.HandlerOptions{Level: tc.level}))) + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) + r := chi.NewRouter() + r.Use(chimw.RequestID) + r.Use(TrustedProxyIP([]netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")})) + r.Use(RequestLogger) + r.Use(chimw.Recoverer) + r.Get("/items/{id}", func(w http.ResponseWriter, _ *http.Request) { + if tc.panic { + panic("test panic") + } + w.WriteHeader(tc.status) + }) + req := httptest.NewRequest(http.MethodGet, "/items/private-id?token=private-token", nil) + req.RemoteAddr = "192.0.2.1:4444" + req.Header.Set("X-Real-IP", "203.0.113.7") + req.Header.Set("True-Client-IP", "198.51.100.8") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != tc.status { + t.Fatalf("status=%d", w.Code) + } + if tc.wantLevel == "" { + if out.Len() != 0 { + t.Fatal("info request was not filtered") + } + return + } + if strings.Contains(out.String(), "private-id") || strings.Contains(out.String(), "private-token") || strings.Contains(out.String(), "198.51.100.8") { + t.Fatal("request values leaked or spoofed identity used") + } + var completed bool + for _, line := range bytes.Split(bytes.TrimSpace(out.Bytes()), []byte("\n")) { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal(err) + } + if record["level"] != tc.wantLevel || record["component"] != "http" || record["client_ip"] != "203.0.113.7" || record["path"] != "/items/{id}" { + t.Fatalf("bad record: %v", record) + } + if record["msg"] == "request complete" { + completed = true + if record["status"] != float64(tc.status) { + t.Fatal("wrong logged status") + } + } + } + if !completed { + t.Fatal("request completion missing") + } + }) + } +} + +func TestRequestLogInputBoundary(t *testing.T) { + for _, format := range []string{"text", "json"} { + for _, path := range []string{"/items/private-path", "/unmatched-private-path%0D%0Aforged"} { + t.Run(format+path, func(t *testing.T) { + var out bytes.Buffer + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + var handler slog.Handler = slog.NewTextHandler(&out, nil) + if format == "json" { + handler = slog.NewJSONHandler(&out, nil) + } + slog.SetDefault(slog.New(handler)) + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) + r := chi.NewRouter() + r.Use(chimw.RequestID, RequestLogger) + r.Get("/items/{id}", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(200) }) + req := httptest.NewRequest(http.MethodGet, path+"?token=private-token", nil) + req.RemoteAddr = "192.0.2.1:1234" + req.Header.Set("X-Request-ID", "client\r\nforged") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + wantPath, wantStatus := "/items/{id}", 200 + if strings.HasPrefix(path, "/unmatched") { + wantPath, wantStatus = "unmatched", 404 + } + if w.Code != wantStatus || strings.Count(out.String(), "\n") != 1 || strings.ContainsAny(strings.TrimSuffix(out.String(), "\n"), "\r\n") { + t.Fatalf("status or log boundary changed: %d %q", w.Code, out.String()) + } + if strings.Contains(out.String(), "private-path") || strings.Contains(out.String(), "private-token") { + t.Fatal("raw path or query was retained") + } + if format == "json" { + var record map[string]any + if err := json.Unmarshal(out.Bytes(), &record); err != nil { + t.Fatal(err) + } + if record["path"] != wantPath || record["request_id"] != "clientforged" || record["client_ip"] != "192.0.2.1:1234" { + t.Fatalf("unexpected request fields: %v", record) + } + } else if !strings.Contains(out.String(), "path="+wantPath) || !strings.Contains(out.String(), "request_id=clientforged") { + t.Fatalf("unexpected text fields: %s", out.String()) + } + }) + } + } +} + +func TestSingleLineLogValue(t *testing.T) { + for _, tc := range []struct{ input, want string }{ + {"", ""}, {"GET", "GET"}, {"[2001:db8::1]:80", "[2001:db8::1]:80"}, + {"client\r\nentry\n", "cliententry"}, {"café", "café"}, + } { + if got := singleLineLogValue(tc.input); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + } +} diff --git a/internal/api/middleware/rate_limit.go b/internal/api/middleware/rate_limit.go new file mode 100644 index 00000000..dcd3b444 --- /dev/null +++ b/internal/api/middleware/rate_limit.go @@ -0,0 +1,36 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "net/http" + "time" + + "github.com/go-chi/httprate" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +// RateLimit applies the minute budget and one-second burst cap to the API. +// TrustedProxyIP must run first; the access logger must wrap this middleware. +func RateLimit(cfg config.ResolvedRateLimitConfig) func(http.Handler) http.Handler { + if !cfg.Enabled { + return func(next http.Handler) http.Handler { return next } + } + options := []httprate.Option{ + httprate.WithLimitHandler(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":{"code":"rate_limited","message":"Request rate limit exceeded. Retry later."}}`)) + }), + // Only expose the backoff header, since the two windows have different budgets. + httprate.WithResponseHeaders(httprate.ResponseHeaders{RetryAfter: "Retry-After"}), + } + // RemoteAddr has already been resolved and forwarding headers removed. + minute := httprate.LimitBy(cfg.RequestsPerMinute, time.Minute, httprate.KeyByIP, options...) + burst := httprate.LimitBy(cfg.Burst, time.Second, httprate.KeyByIP, options...) + return func(next http.Handler) http.Handler { + return minute(burst(next)) + } +} diff --git a/internal/api/middleware/runtime_config.go b/internal/api/middleware/runtime_config.go new file mode 100644 index 00000000..cb6a9e60 --- /dev/null +++ b/internal/api/middleware/runtime_config.go @@ -0,0 +1,129 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "errors" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "unicode" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/cors" +) + +var ErrInvalidOrigins = errors.New("allowed_origins must contain 1-32 ASCII HTTP(S) origins (at most 512 bytes each), or a sole *; paths, credentials, queries and control characters are not allowed") + +type runtimePolicy struct { + config api.AdminConfig + handler http.Handler +} + +// RuntimeConfig owns one router's CORS handler and its nonsecret config view. +// Published policies are immutable. Updates are runtime-only, never persisted. +type RuntimeConfig struct { + mu sync.Mutex + current atomic.Pointer[runtimePolicy] + next http.Handler + exposedHeaders []string +} + +func NewRuntimeConfig(config api.AdminConfig, exposedHeaders []string) *RuntimeConfig { + s := &RuntimeConfig{exposedHeaders: slices.Clone(exposedHeaders)} + s.current.Store(&runtimePolicy{config: cloneAdminConfig(config)}) + return s +} + +// CORS is bound once as the outer middleware of its owning router. +func (s *RuntimeConfig) CORS(next http.Handler) http.Handler { + s.mu.Lock() + defer s.mu.Unlock() + if s.next != nil { + panic("runtime CORS already bound") + } + s.next = next + s.current.Store(s.policy(s.current.Load().config)) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.current.Load().handler.ServeHTTP(w, r) + }) +} + +func (s *RuntimeConfig) Snapshot() api.AdminConfig { + return cloneAdminConfig(s.current.Load().config) +} + +func (s *RuntimeConfig) UpdateOrigins(origins []string) (api.AdminConfig, error) { + validated, err := validateOrigins(origins) + if err != nil { + return api.AdminConfig{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + config := cloneAdminConfig(s.current.Load().config) + config.CORS.AllowedOrigins = validated + s.current.Store(s.policy(config)) + return cloneAdminConfig(config), nil +} + +func (s *RuntimeConfig) policy(config api.AdminConfig) *runtimePolicy { + p := &runtimePolicy{config: config} + if s.next != nil { + p.handler = cors.Handler(cors.Options{ + AllowedOrigins: slices.Clone(config.CORS.AllowedOrigins), + AllowedMethods: slices.Clone(config.CORS.AllowedMethods), + AllowedHeaders: slices.Clone(config.CORS.AllowedHeaders), + AllowCredentials: config.CORS.AllowCredentials, MaxAge: config.CORS.MaxAge, + ExposedHeaders: slices.Clone(s.exposedHeaders), + })(s.next) + } + return p +} + +func cloneAdminConfig(config api.AdminConfig) api.AdminConfig { + config.CORS.AllowedOrigins = slices.Clone(config.CORS.AllowedOrigins) + config.CORS.AllowedMethods = slices.Clone(config.CORS.AllowedMethods) + config.CORS.AllowedHeaders = slices.Clone(config.CORS.AllowedHeaders) + return config +} + +func validateOrigins(origins []string) ([]string, error) { + if len(origins) == 0 || len(origins) > 32 { + return nil, ErrInvalidOrigins + } + result := make([]string, len(origins)) + for i, origin := range origins { + if len(origin) > 512 || strings.IndexFunc(origin, unicode.IsControl) >= 0 { + return nil, ErrInvalidOrigins + } + origin = strings.TrimSpace(origin) + if strings.IndexFunc(origin, func(r rune) bool { return r > unicode.MaxASCII }) >= 0 { + return nil, ErrInvalidOrigins + } + origin = strings.ToLower(origin) + if origin == "*" { + if len(origins) != 1 { + return nil, ErrInvalidOrigins + } + result[i] = origin + continue + } + u, err := url.Parse(origin) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(origin, "#") || strings.Count(u.Host, "*") > 1 || strings.Contains(u.Host, "%") || strings.HasSuffix(u.Host, ":") { + return nil, ErrInvalidOrigins + } + if port := u.Port(); port != "" { + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 { + return nil, ErrInvalidOrigins + } + } + result[i] = origin + } + return result, nil +} diff --git a/internal/api/middleware/runtime_config_test.go b/internal/api/middleware/runtime_config_test.go new file mode 100644 index 00000000..402edd2b --- /dev/null +++ b/internal/api/middleware/runtime_config_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func TestValidateOrigins(t *testing.T) { + for _, tc := range []struct { + input []string + valid bool + }{ + {[]string{"*"}, true}, {[]string{" HTTPS://Example.test ", "https://*.example.test", "http://127.0.0.1:8080", "http://[::1]:8080"}, true}, + {nil, false}, {[]string{}, false}, {make([]string, 33), false}, {[]string{""}, false}, {[]string{"null"}, false}, + {[]string{"*", "https://example.test"}, false}, {[]string{"ftp://example.test"}, false}, {[]string{"https://"}, false}, + {[]string{"https://example.test/"}, false}, {[]string{"https://example.test?"}, false}, {[]string{"https://example.test#"}, false}, + {[]string{"https://user:secret@example.test"}, false}, {[]string{"https://example.test:65536"}, false}, {[]string{"https://example.test:"}, false}, + {[]string{"https://*.*.test"}, false}, {[]string{"https://example.test\n"}, false}, {[]string{"https://é.test"}, false}, {[]string{"https://K.test"}, false}, {[]string{strings.Repeat("x", 513)}, false}, + } { + _, err := validateOrigins(tc.input) + if (err == nil) != tc.valid { + t.Fatalf("origins=%q valid=%v error=%v", tc.input, tc.valid, err) + } + } + input := []string{" HTTPS://Example.test "} + got, err := validateOrigins(input) + if err != nil || got[0] != "https://example.test" || input[0] != " HTTPS://Example.test " { + t.Fatal("normalization changed caller state") + } +} + +func TestRuntimePolicyAndConcurrency(t *testing.T) { + initial := api.AdminConfig{Auth: api.AdminAuthConfig{Configured: true}, Ingest: api.AdminIngestConfig{BrokerCount: 2}, + CORS: api.AdminCORSConfig{AllowedOrigins: []string{"https://before.test"}, AllowedMethods: []string{"GET", "PUT"}, AllowedHeaders: []string{"Authorization"}, AllowCredentials: true, MaxAge: 600}} + state := NewRuntimeConfig(initial, []string{"Retry-After"}) + initial.CORS.AllowedOrigins[0] = "https://caller-change.test" + handler := state.CORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) })) + preflight := func(origin string) *httptest.ResponseRecorder { + r := httptest.NewRequest("OPTIONS", "/api/v1/brokers", nil) + r.Header.Set("Origin", origin) + r.Header.Set("Access-Control-Request-Method", "PUT") + r.Header.Set("Access-Control-Request-Headers", "Authorization") + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + return w + } + if preflight("https://before.test").Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("startup policy changed with caller slice") + } + origins := []string{"https://*.after.test"} + updated, err := state.UpdateOrigins(origins) + if err != nil { + t.Fatal(err) + } + origins[0] = "https://caller-change.test" + updated.CORS.AllowedOrigins[0] = "https://response-change.test" + if preflight("https://before.test").Header().Get("Access-Control-Allow-Origin") != "" { + t.Fatal("old policy remains active") + } + w := preflight("https://app.after.test") + if w.Code != 200 || w.Header().Get("Access-Control-Allow-Origin") != "https://app.after.test" || w.Header().Get("Access-Control-Max-Age") != "600" || w.Header().Get("Access-Control-Allow-Credentials") != "true" { + t.Fatal("new policy/options not enforced") + } + r := httptest.NewRequest("GET", "/", nil) + r.Header.Set("Origin", "https://app.after.test") + w = httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Header().Get("Access-Control-Expose-Headers") != "Retry-After" { + t.Fatal("exposed headers lost") + } + before := state.Snapshot() + if _, err := state.UpdateOrigins(nil); err == nil || !reflect.DeepEqual(state.Snapshot(), before) { + t.Fatal("invalid update changed config") + } + var group sync.WaitGroup + for i := 0; i < 8; i++ { + group.Add(1) + go func(i int) { + defer group.Done() + for j := 0; j < 40; j++ { + origin := "https://a.test" + if i%2 != 0 { + origin = "https://b.test" + } + if _, err := state.UpdateOrigins([]string{origin}); err != nil { + t.Error(err) + } + snapshot := state.Snapshot() + if len(snapshot.CORS.AllowedOrigins) != 1 || !snapshot.Auth.Configured || snapshot.Ingest.BrokerCount != 2 { + t.Error("torn config") + } + preflight(origin) + } + }(i) + } + group.Wait() + final := state.Snapshot() + if preflight(final.CORS.AllowedOrigins[0]).Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("reported/enforced final policy differs") + } +} diff --git a/internal/api/middleware/trusted_proxy.go b/internal/api/middleware/trusted_proxy.go new file mode 100644 index 00000000..96b1942a --- /dev/null +++ b/internal/api/middleware/trusted_proxy.go @@ -0,0 +1,60 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "log/slog" + "net/http" + "net/netip" + "sync" +) + +// TrustedProxyIP accepts X-Real-IP only from a configured direct proxy peer. +// Run it before the access logger and handlers that use RemoteAddr as a key. +// All other forwarding headers are ignored, as are invalid/ambiguous values. +func TrustedProxyIP(proxies []netip.Prefix) func(http.Handler) http.Handler { + // Misconfigured proxies key every visitor to one IP; say so once, not per request. + var untrustedOnce, noRealIPOnce sync.Once + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peer, err := netip.ParseAddrPort(r.RemoteAddr) + if err == nil { + trusted := false + for _, proxy := range proxies { + if !proxy.Contains(peer.Addr().Unmap()) { + continue + } + trusted = true + if values := r.Header.Values("X-Real-IP"); len(values) == 1 { + if ip, err := netip.ParseAddr(values[0]); err == nil && ip.Zone() == "" { + r.RemoteAddr = ip.Unmap().String() + } + } else if len(values) == 0 && r.Header.Get("X-Forwarded-For") != "" { + noRealIPOnce.Do(func() { + slog.Warn("trusted proxy sent X-Forwarded-For without X-Real-IP; clients share its rate limits until the proxy sets X-Real-IP (see server.trusted_proxies)", + "component", "http", "peer", peer.Addr().Unmap().String()) + }) + } + break + } + if !trusted && hasForwardingHeader(r.Header) { + untrustedOnce.Do(func() { + slog.Warn("forwarding headers from an untrusted peer were ignored; if it is your reverse proxy, add it to server.trusted_proxies and have it set X-Real-IP, or all clients share its rate limits", + "component", "http", "peer", peer.Addr().Unmap().String()) + }) + } + } + // Downstream client-IP keys must use the resolved RemoteAddr, not + // reinterpret forwarding headers (including those from trusted peers). + r.Header.Del("True-Client-IP") + r.Header.Del("X-Forwarded-For") + r.Header.Del("X-Real-IP") + next.ServeHTTP(w, r) + }) + } +} + +func hasForwardingHeader(h http.Header) bool { + return len(h.Values("X-Forwarded-For")) > 0 || len(h.Values("X-Real-IP")) > 0 || len(h.Values("True-Client-IP")) > 0 +} diff --git a/internal/api/middleware/trusted_proxy_test.go b/internal/api/middleware/trusted_proxy_test.go new file mode 100644 index 00000000..6ac14ba4 --- /dev/null +++ b/internal/api/middleware/trusted_proxy_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "bytes" + "log/slog" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" +) + +func TestTrustedProxyIP(t *testing.T) { + trusted := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("2001:db8:1::/48")} + for _, tc := range []struct { + name, peer string + proxies []netip.Prefix + realIP []string + want string + }{ + {"empty allowlist", "192.0.2.10:1234", nil, []string{"198.51.100.1"}, "192.0.2.10:1234"}, + {"untrusted peer", "192.0.3.10:1234", trusted, []string{"198.51.100.1"}, "192.0.3.10:1234"}, + {"trusted IPv4", "192.0.2.10:1234", trusted, []string{"198.51.100.1"}, "198.51.100.1"}, + {"trusted IPv6", "[2001:db8:1::10]:1234", trusted, []string{"2001:db8:2::1"}, "2001:db8:2::1"}, + {"mapped peer", "[::ffff:192.0.2.10]:1234", trusted, []string{"198.51.100.1"}, "198.51.100.1"}, + {"mapped client", "192.0.2.10:1234", trusted, []string{"::ffff:198.51.100.1"}, "198.51.100.1"}, + {"missing header", "192.0.2.10:1234", trusted, nil, "192.0.2.10:1234"}, + {"empty header", "192.0.2.10:1234", trusted, []string{""}, "192.0.2.10:1234"}, + {"invalid IP", "192.0.2.10:1234", trusted, []string{"not-an-ip"}, "192.0.2.10:1234"}, + {"IP with port", "192.0.2.10:1234", trusted, []string{"198.51.100.1:8080"}, "192.0.2.10:1234"}, + {"IP with zone", "192.0.2.10:1234", trusted, []string{"fe80::1%eth0"}, "192.0.2.10:1234"}, + {"comma list", "192.0.2.10:1234", trusted, []string{"198.51.100.1, 198.51.100.2"}, "192.0.2.10:1234"}, + {"repeated header", "192.0.2.10:1234", trusted, []string{"198.51.100.1", "198.51.100.2"}, "192.0.2.10:1234"}, + {"invalid peer", "unknown", trusted, []string{"198.51.100.1"}, "unknown"}, + } { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.RemoteAddr = tc.peer + for _, ip := range tc.realIP { + r.Header.Add("X-Real-IP", ip) + } + // These must never override the selected address, even for trusted peers. + r.Header.Set("True-Client-IP", "203.0.113.1") + r.Header.Set("X-Forwarded-For", "203.0.113.2") + r.Header.Set("X-Request-ID", "keep-me") + TrustedProxyIP(tc.proxies)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RemoteAddr != tc.want { + t.Errorf("RemoteAddr = %q, want %q", r.RemoteAddr, tc.want) + } + for _, header := range []string{"True-Client-IP", "X-Forwarded-For", "X-Real-IP"} { + if len(r.Header.Values(header)) != 0 { + t.Errorf("downstream middleware can still read %s", header) + } + } + if r.Header.Get("X-Request-ID") != "keep-me" { + t.Error("unrelated header was changed") + } + })).ServeHTTP(httptest.NewRecorder(), r) + }) + } +} + +func TestTrustedProxyIPWarnsOnce(t *testing.T) { + trusted := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")} + for _, tc := range []struct { + name, peer, header string + proxies []netip.Prefix + want string // in the single warning; "" means none + }{ + {"untrusted XFF", "198.51.100.7:1234", "X-Forwarded-For", nil, "198.51.100.7"}, + {"untrusted X-Real-IP", "198.51.100.7:1234", "X-Real-IP", trusted, "198.51.100.7"}, + {"untrusted True-Client-IP", "198.51.100.7:1234", "True-Client-IP", nil, "198.51.100.7"}, + {"trusted XFF only", "192.0.2.10:1234", "X-Forwarded-For", trusted, "X-Real-IP"}, + {"trusted with X-Real-IP", "192.0.2.10:1234", "X-Real-IP", trusted, ""}, + {"direct client", "198.51.100.7:1234", "", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + handler := TrustedProxyIP(tc.proxies)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + for range 3 { + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.RemoteAddr = tc.peer + if tc.header != "" { + r.Header.Set(tc.header, "203.0.113.9") + } + handler.ServeHTTP(httptest.NewRecorder(), r) + } + logs := out.String() + warnings := strings.Count(logs, `"level":"WARN"`) + if tc.want == "" { + if warnings != 0 { + t.Fatalf("unexpected warning: %s", logs) + } + return + } + if warnings != 1 || !strings.Contains(logs, tc.want) || !strings.Contains(logs, "trusted_proxies") { + t.Fatalf("want one warning mentioning %q and trusted_proxies, got: %s", tc.want, logs) + } + if strings.Contains(logs, "203.0.113.9") { + t.Fatal("warning echoes a client-supplied header value") + } + }) + } +} diff --git a/internal/api/nodes.go b/internal/api/nodes.go index a79280b7..9dce5579 100644 --- a/internal/api/nodes.go +++ b/internal/api/nodes.go @@ -53,6 +53,9 @@ type NodeSummary struct { // Applies to every node type, unlike ClockDriftSeconds/ClockOutOfSync on Node, which // are repeater/room-server only. Stale bool `json:"stale"` + // PossiblyForeign compares a repeater's stored position with this server's + // configured IATA border union. Omitted when disabled, unknown, or not repeater. + PossiblyForeign *bool `json:"possiblyForeign,omitempty"` } // Node is the full node representation including firmware capability flags, diff --git a/internal/api/observers.go b/internal/api/observers.go index 94d10789..995ef04d 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -3,7 +3,11 @@ package api -import "github.com/google/uuid" +import ( + "encoding/json" + + "github.com/google/uuid" +) // ObserverSummary is the minimal observer representation used in list responses. type ObserverSummary struct { @@ -40,20 +44,21 @@ type Observer struct { RadioCR *int16 `json:"radioCr,omitempty"` // coding rate denominator BatteryLevel *float32 `json:"batteryLevel,omitempty"` // volts, nil if mains powered UptimeSeconds *int64 `json:"uptimeSeconds,omitempty"` - StatusMetadata any `json:"statusMetadata,omitempty"` // raw /status JSON payload - LastStatusAt *int64 `json:"lastStatusAt,omitempty"` // epoch ms - FirstSeen int64 `json:"firstSeen"` // epoch ms - LastSeen int64 `json:"lastSeen"` // epoch ms - ObservationCount int64 `json:"observationCount"` - Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on + StatusMetadata json.RawMessage `json:"statusMetadata,omitempty" swaggertype:"object"` // raw /status JSON payload + LastStatusAt *int64 `json:"lastStatusAt,omitempty"` // epoch ms + FirstSeen int64 `json:"firstSeen"` // epoch ms + LastSeen int64 `json:"lastSeen"` // epoch ms + ObservationCount int64 `json:"observationCount"` // legacy cumulative presence counter; includes non-packet events + Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on } // ObserverTelemetryPoint is a single telemetry snapshot for an observer. +// Airtime is radio seconds: cumulative since boot on 1h points, per-bucket delta on 6h/24h. type ObserverTelemetryPoint struct { T int64 `json:"t"` // epoch ms BatteryMV *int32 `json:"batteryMv,omitempty"` - AirtimeTxPct *float32 `json:"airtimeTxPct,omitempty"` - AirtimeRxPct *float32 `json:"airtimeRxPct,omitempty"` + AirtimeTxSecs *float32 `json:"airtimeTxSecs,omitempty"` + AirtimeRxSecs *float32 `json:"airtimeRxSecs,omitempty"` NoiseFloorDB *float32 `json:"noiseFloorDb,omitempty"` UptimeSeconds *int64 `json:"uptimeSeconds,omitempty"` QueueLength *int32 `json:"queueLength,omitempty"` @@ -67,3 +72,52 @@ type ObserverTelemetry struct { Interval string `json:"interval"` Points []ObserverTelemetryPoint `json:"points"` } + +// ObserverActivityRadio echoes the observer's current radio params plus the preamble the airtime math assumes. +type ObserverActivityRadio struct { + FreqMHz *float32 `json:"freqMhz"` + SF int16 `json:"sf"` + BWKHz float32 `json:"bwKhz"` + CR int16 `json:"cr"` + PreambleSymbols int `json:"preambleSymbols"` +} + +// ObserverActivityPoint is one bucket of what an observer heard. +type ObserverActivityPoint struct { + T int64 `json:"t"` // bucket start, epoch ms + Observations int64 `json:"observations"` + AirtimeMs *float32 `json:"airtimeMs"` + SNRAvg *float32 `json:"snrAvg"` + SNRMin *float32 `json:"snrMin"` + RSSIAvg *float32 `json:"rssiAvg"` +} + +// ObserverActivitySummary describes stored packet records, never MQTT presence events. +// Freshness and lastCompleteHour are measured at generatedAt, even with an explicit until; +// only recordedPackets follows the selected activity window. +type ObserverActivitySummary struct { + RecordedPackets int64 `json:"recordedPackets"` // stored observations within windowStart/windowEnd + LastCompleteHour int64 `json:"lastCompleteHour"` + LastCompleteHourStart int64 `json:"lastCompleteHourStart"` + LastCompleteHourEnd int64 `json:"lastCompleteHourEnd"` + LatestRecordedAt *int64 `json:"latestRecordedAt"` +} + +// ObserverActivity is the per-observer heard-activity response. +type ObserverActivity struct { + WindowStart int64 `json:"windowStart"` // inclusive complete-bucket start, epoch ms + WindowEnd int64 `json:"windowEnd"` // exclusive end, epoch ms; live requests include the current partial bucket + GeneratedAt int64 `json:"generatedAt"` // response computation time, not proof of continuous coverage + Source string `json:"source"` // raw or hourly; missing records do not prove an outage + Summary *ObserverActivitySummary `json:"summary,omitempty"` + // Hourly only. Buckets before rolledUntil come from rollups and from rawFrom on from raw rows; + // when rawFrom is later, the hours between are uncovered, not quiet. + RolledUntil *int64 `json:"rolledUntil,omitempty"` // end of the newest complete rollup hour, epoch ms + RawFrom *int64 `json:"rawFrom,omitempty"` // start of the raw tail, epoch ms + + Range string `json:"range"` + Interval string `json:"interval"` + Radio *ObserverActivityRadio `json:"radio"` + PayloadTypes []PayloadBreakdownItem `json:"payloadTypes"` + Points []ObserverActivityPoint `json:"points"` +} diff --git a/internal/api/packets.go b/internal/api/packets.go index 516be02c..e5f47fe8 100644 --- a/internal/api/packets.go +++ b/internal/api/packets.go @@ -17,6 +17,14 @@ type PacketLatestObserver struct { ID uuid.UUID `json:"id"` DisplayName *string `json:"displayName,omitempty"` IATA string `json:"iata"` + // PathLength/PathBytes and captured endpoint resolutions are stored on the observation, + // so list/backfill reads need no per-hash resolution queries. Legacy observations have + // no endpoint snapshot. ResolvedPath remains a detail/opted-in WS feature. + PathLength *PacketPathLength `json:"pathLength,omitempty"` + PathBytes *string `json:"pathBytes,omitempty"` // hex-encoded accumulated path hashes + ResolvedPath []ResolvedHop `json:"resolvedPath,omitempty"` + ResolvedSource *ResolvedHop `json:"resolvedSource,omitempty"` + ResolvedDestination *ResolvedHop `json:"resolvedDestination,omitempty"` } // PacketSummary is the minimal packet representation used in list responses. @@ -32,7 +40,7 @@ type PacketSummary struct { LastHeardAt int64 `json:"lastHeardAt"` // epoch ms ObservationCount int32 `json:"observationCount"` LatestObserver *PacketLatestObserver `json:"latestObserver,omitempty"` - Summary *string `json:"summary,omitempty"` // human-readable payload summary + Summary *string `json:"summary,omitempty"` // advert name, or the ACK/TRACE/PING summary; omitted when unavailable } // PacketPathLength is the decoded path_length byte from a packet observation. @@ -59,7 +67,9 @@ type PacketObservationDetail struct { SourceBroker string `json:"sourceBroker"` ResolvedPath []ResolvedHop `json:"resolvedPath"` // per-observation resolved path hashes // ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type - // carries a resolvable one: an exact match for ADVERT's full pubkey, an ambiguous + // carries one. Prefer the snapshot captured at ingest; legacy observations without a + // snapshot use the current node registry. Endpoint matching itself is unchanged: + // an exact match for ADVERT's full pubkey, an ambiguous // hash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte // source/destination hashes. Nil when the payload type doesn't carry one at all (e.g. // GRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and diff --git a/internal/api/paths.go b/internal/api/paths.go new file mode 100644 index 00000000..dad76d10 --- /dev/null +++ b/internal/api/paths.go @@ -0,0 +1,47 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +// PathStats counts stored receptions in [Since, Until), not unique packets or nodes. +// Hashed + Empty + Trace + Unclassified partitions Receptions. Empty paths never +// vote for a hash width. TRACE header paths contain signal readings, not hashes. +// Since and Until are the effective UTC-hour boundaries of the materialized snapshot. +type PathStats struct { + Since int64 `json:"since"` + Until int64 `json:"until"` + Receptions int64 `json:"receptions"` + Hashed int64 `json:"hashed"` + Empty int64 `json:"empty"` + Trace int64 `json:"trace"` + Unclassified int64 `json:"unclassified"` + HashWidths []PathHashWidth `json:"hashWidths"` + PathLengths []PathLengthBin `json:"pathLengths"` + Hourly []PathHour `json:"hourly"` +} + +type PathHashWidth struct { + Bytes int32 `json:"bytes"` + Receptions int64 `json:"receptions"` +} + +// PathLengthBin counts entries in validated ordinary header paths, including empty. +// Flood paths accumulate entries; direct paths contain remaining route entries. +// These counts cannot establish distance or a complete end-to-end hop count. +type PathLengthBin struct { + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +// PathHour is a UTC hour clipped to the requested window. Missing hours are omitted. +type PathHour struct { + Hour int64 `json:"hour"` + Receptions int64 `json:"receptions"` + OneByte int64 `json:"oneByte"` + TwoByte int64 `json:"twoByte"` + ThreeByte int64 `json:"threeByte"` + Empty int64 `json:"empty"` + Trace int64 `json:"trace"` + Unclassified int64 `json:"unclassified"` + MaxEntries int32 `json:"maxEntries"` // longest path received that hour across the requested IATAs; 0 if only empty paths +} diff --git a/internal/api/reader.go b/internal/api/reader.go index 8d8cc2fe..dc71e63f 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -23,6 +23,8 @@ type Page[T any] struct { } type Reader interface { + // GetRouteEvidence returns the full saved route and exact-prefix retained ordinary reports. + GetRouteEvidence(ctx context.Context, iata, pathKey string, query RouteEvidenceQuery) (*RouteEvidence, error) // ListIATAs returns all known IATA codes with display name and coordinates. // IATAs are auto-created on first packet arrival from that location. ListIATAs(ctx context.Context) ([]IATA, error) @@ -51,9 +53,11 @@ type Reader interface { // ListChannels returns a paginated list of channels ordered by last seen. // Includes both hashtag-derived and explicit key channels. // Pass nil hash to skip hash filtering. Pass empty iatas to return all channels; - // IATAs must be uppercase. + // IATAs must be uppercase. Nil keyKnown skips the decryption-key filter. // cursor is last_seen epoch ms of the last item; pass 0 to start from the beginning. - ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (Page[ChannelSummary], error) + // pageCursor is the precise timestamp/ID boundary; nil preserves the legacy numeric cursor. + // When non-nil, pageCursor takes precedence over cursor. + ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *ChannelCursor) (ChannelPage, error) // GetChannel returns full detail for a single channel by its integer ID. // Returns nil, pgx.ErrNoRows if the channel is not found. @@ -95,6 +99,11 @@ type Reader interface { // GetObserverTelemetryBucketed returns telemetry points for an observer bucketed into N-hour intervals. GetObserverTelemetryBucketed(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]ObserverTelemetryPoint, error) + // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. + // interval >= 1h is served from the hourly rollup. Returns pgx.ErrNoRows for an unknown observer. + // Range and Interval on the result are left empty for the handler to fill. + GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*ObserverActivity, error) + // GetObserverScopes returns the names of all transport scopes an observer has // been seen forwarding packets for, ordered alphabetically. GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) @@ -141,6 +150,10 @@ type Reader interface { // Pass empty string for preset or nil iatas to skip those filters. GetRadioPresets(ctx context.Context, preset string, iatas []string) ([]RadioPreset, error) + // GetObserverComparison compares distinct flood packets reported by two observers + // during [since, until), optionally restricted to reception IATAs. + GetObserverComparison(ctx context.Context, observerA, observerB uuid.UUID, since, until time.Time, iatas []string) (*ObserverComparison, error) + // GetStatsOverview returns top-line network figures for the last 24 hours. // Pass nil iatas to return stats across all IATAs. GetStatsOverview(ctx context.Context, iatas []string) (*StatsOverview, error) @@ -150,6 +163,15 @@ type Reader interface { // since defines the start of the window; pass zero time for default (last 7 days). GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]ObservationPoint, error) + // GetStatsSeries returns hourly rollup metrics for [since, until), both on UTC hours. + GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*StatsSeries, error) + + // GetSignalStats aggregates retained reception readings in [since, until). + GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*SignalStats, error) + + // GetPathStats counts received path entries and validated ordinary hash widths. + GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*PathStats, error) + // GetStatsPayloadBreakdown returns observation counts grouped by payload type. // Pass nil for iatas to return stats across all IATAs. // since defines the start of the window; pass zero time for default (last 24h). @@ -157,7 +179,7 @@ type Reader interface { // GetStatsTopNodes returns the top N nodes by observation count. // Pass nil for iatas to return stats across all IATAs. - GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]TopNode, error) + GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]TopNode, error) // GetStatsTopObservers returns the top N observers by observation count. // Pass nil for iatas to return stats across all IATAs. @@ -169,13 +191,20 @@ type Reader interface { // since defines the start of the window; pass zero time for default (last 24h). GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]TopAdvertiser, error) + // GetStatsClockDrift returns repeaters/room servers whose most recent advert-derived + // clock drift exceeds the configured threshold (nodes.clock_drift_threshold), worst + // drift first. Pass nil for iatas to return across all IATAs. Unlike the other stats + // endpoints this isn't time-windowed -- it reflects each node's current drift state. + GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]ClockDriftEntry, error) + // GetStatsTopTalkers returns the top N companion names by decrypted channel message count. // Pass nil for iatas to return stats across all IATAs. // since defines the start of the window; pass zero time for default (last 24h). GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]TopTalker, error) // GetScopeStats returns aggregate packet, observer and node counts per transport scope. - GetScopeStats(ctx context.Context) ([]ScopeStats, error) + // Pass empty iatas for global totals. IATAs must be uppercase. + GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]ScopeStats, error) // GetStatsNodeTypes returns node counts grouped by type, optionally filtered by IATA. GetStatsNodeTypes(ctx context.Context, iatas []string) ([]NodeTypeCount, error) @@ -184,23 +213,20 @@ type Reader interface { // Use when no geographic filter is applied — returns names only for a lightweight response. GetScopeNames(ctx context.Context) ([]string, error) - // GetScopesByIATAs returns scope summaries filtered by the given IATA codes, - // including observer, node and IATA counts. Expands region/regionId to IATAs automatically. - GetScopesByIATAs(ctx context.Context, iatas []string) ([]ScopeSummary, error) - // GetScopeByName returns full detail for a single scope by its normalized name (e.g. "#bc"), // including packet count, observer count, node count, and the list of IATAs it is active in. - // Returns nil if the scope is not found. + // Returns nil, pgx.ErrNoRows if the scope is not found. GetScopeByName(ctx context.Context, name string) (*ScopeDetail, error) - // ListTraceTags returns a paginated list of trace tags with aggregate metadata. - ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]TraceTagSummary, error) + // ListTraceTags returns trace tags newest first, keyset-paged on (cursor ms, cursorTag hex). + ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]TraceTagSummary, error) // GetTraceByTag returns all packets for a given trace tag with resolved routes. GetTraceByTag(ctx context.Context, tag string) (*TraceDetail, error) // ListKnownRoutes returns known routes filtered by IATA and optional hop count. - ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]KnownRoute, error) + // A nonzero cursorID breaks ties within the cursor's millisecond. + ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]KnownRoute, error) // SearchKnownRoutes returns known routes containing a path from source to destination hash. SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash string) ([]KnownRoute, error) diff --git a/internal/api/route_evidence.go b/internal/api/route_evidence.go new file mode 100644 index 00000000..d320ade5 --- /dev/null +++ b/internal/api/route_evidence.go @@ -0,0 +1,111 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import ( + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +const MaxRouteEvidenceWindow = 30 * 24 * time.Hour + +var ErrRouteEvidenceInput = errors.New("invalid route evidence request") + +// RouteObservation is a retained report reference, not a packet body or proof of hop identity. +type RouteObservation struct { + ID int64 `json:"id"` + PacketHash string `json:"packetHash"` + ObserverID uuid.UUID `json:"observerId"` + ObserverName *string `json:"observerName,omitempty"` + HeardAt int64 `json:"heardAt"` + PayloadType int16 `json:"payloadType"` + PayloadTypeName string `json:"payloadTypeName"` + RSSI *int16 `json:"rssi,omitempty"` + SNR *float32 `json:"snr,omitempty"` +} + +// RouteEvidence pages by nextPageCursor, which keeps full timestamp precision. +type RouteEvidence struct { + Items []RouteObservation `json:"items"` + HasMore bool `json:"hasMore"` + Route KnownRoute `json:"route"` + WindowStart int64 `json:"windowStart"` + WindowEnd int64 `json:"windowEnd"` + GeneratedAt int64 `json:"generatedAt"` + MatchType string `json:"matchType"` + MatchAvailable bool `json:"matchAvailable"` + HashSize int16 `json:"hashSize"` + PathBytes string `json:"pathBytes"` + NextPageCursor *string `json:"nextPageCursor,omitempty"` +} + +type RouteEvidenceQuery struct { + Since time.Time + Until time.Time + Cursor *RouteEvidenceCursor + Limit int32 +} + +// RouteEvidenceCursor binds the precise (heard_at DESC,id DESC) boundary to its route/window. +type RouteEvidenceCursor struct { + IATA string + PathKey string + Since time.Time + Until time.Time + HeardAt time.Time + ID int64 +} + +func ValidRouteEvidenceKey(iata, key string) bool { + if len(iata) != 3 || len(key) != 32 { + return false + } + for _, c := range iata { + if !(c >= 'A' && c <= 'Z') && !(c >= '0' && c <= '9') { + return false + } + } + _, err := hex.DecodeString(key) + return err == nil && key == strings.ToLower(key) +} + +func ValidRouteEvidenceWindow(since, until time.Time) bool { + return !since.Before(time.Unix(0, 0)) && until.After(since) && until.Sub(since) <= MaxRouteEvidenceWindow +} + +func (c RouteEvidenceCursor) String() string { + return fmt.Sprintf("v1:%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) +} + +func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { + if len(raw) > 200 { + return nil, ErrRouteEvidenceInput + } + p := strings.Split(raw, ":") + if len(p) != 7 || p[0] != "v1" || !ValidRouteEvidenceKey(p[1], p[2]) { + return nil, ErrRouteEvidenceInput + } + values := [4]int64{} + for i, s := range p[3:] { + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || n < 0 || strconv.FormatInt(n, 10) != s { + return nil, ErrRouteEvidenceInput + } + values[i] = n + } + // Bound before converting; this also keeps millisecond/microsecond comparisons safe. + if values[0] > 253402300799999 || values[1] > 253402300799999 || values[2] > 253402300799999999 || values[3] <= 0 { + return nil, ErrRouteEvidenceInput + } + c := &RouteEvidenceCursor{IATA: p[1], PathKey: p[2], Since: time.UnixMilli(values[0]).UTC(), Until: time.UnixMilli(values[1]).UTC(), HeardAt: time.UnixMicro(values[2]).UTC(), ID: values[3]} + if !ValidRouteEvidenceWindow(c.Since, c.Until) || c.HeardAt.Before(c.Since) || !c.HeardAt.Before(c.Until) { + return nil, ErrRouteEvidenceInput + } + return c, nil +} diff --git a/internal/api/route_evidence_test.go b/internal/api/route_evidence_test.go new file mode 100644 index 00000000..42c9a73a --- /dev/null +++ b/internal/api/route_evidence_test.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import ( + "strings" + "testing" + "time" +) + +func TestRouteEvidenceCursor(t *testing.T) { + c := RouteEvidenceCursor{IATA: "YOW", PathKey: "f097439148601d9f3291c474f82fa64c", Since: time.UnixMilli(1000).UTC(), Until: time.UnixMilli(3000).UTC(), HeardAt: time.UnixMicro(2000123).UTC(), ID: 9223372036854775806} + got, err := ParseRouteEvidenceCursor(c.String()) + if err != nil || *got != c { + t.Fatalf("cursor round-trip: %+v %v", got, err) + } + if got.HeardAt.Nanosecond() != 123000 { + t.Fatal("lost microsecond precision") + } + for _, bad := range []string{"", strings.Repeat("x", 300), "v2:YOW:x:1:2:3:4", strings.Replace(c.String(), "YOW", "TOOLONG", 1), strings.Replace(c.String(), c.PathKey, "bad", 1), strings.Replace(c.String(), "2000123", "4000000", 1), strings.TrimSuffix(c.String(), "9223372036854775806") + "0"} { + if _, err := ParseRouteEvidenceCursor(bad); err == nil { + t.Errorf("accepted %q", bad) + } + } +} diff --git a/internal/api/router/accounts_test.go b/internal/api/router/accounts_test.go new file mode 100644 index 00000000..7cd7fd10 --- /dev/null +++ b/internal/api/router/accounts_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/google/uuid" +) + +type routedAccountStore struct { + api.AccountStore + calls int +} + +func (s *routedAccountStore) CreateAccount(_ context.Context, name string) (api.Account, error) { + s.calls++ + return api.Account{ID: uuid.New(), Name: name, Active: true}, nil +} +func TestAccountRouterUsesProtectedStore(t *testing.T) { + store := &routedAccountStore{} + r := New(nil, nil, nil, Options{Auth: config.AuthConfig{APIKey: "test-key"}, AdminRoutes: map[string]http.Handler{"/accounts": handlers.AccountsRouter(store)}}) + for _, token := range []string{"", "wrong", "test-key"} { + req := httptest.NewRequest("POST", "/api/v1/admin/accounts", strings.NewReader(`{"name":"test"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + want := 401 + if token == "test-key" { + want = 201 + } + if w.Code != want { + t.Fatalf("route status=%d want=%d", w.Code, want) + } + } + if store.calls != 1 { + t.Fatal("auth boundary was bypassed") + } + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest("POST", "/api/v1/accounts", nil)) + if w.Code != 404 || store.calls != 1 { + t.Fatal("accounts mounted publicly") + } +} diff --git a/internal/api/router/admin_config_test.go b/internal/api/router/admin_config_test.go new file mode 100644 index 00000000..9acd524c --- /dev/null +++ b/internal/api/router/admin_config_test.go @@ -0,0 +1,86 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" +) + +func TestAdminConfig(t *testing.T) { + const key = "admin-key-sentinel" + for _, variable := range []string{"POSTGRES_DSN", "MQTT_BROKER_1_URL", "MQTT_BROKER_1_USERNAME", "MQTT_BROKER_1_PASSWORD", "REDIS_PASSWORD"} { + t.Setenv(variable, "environment-secret-sentinel") + } + for _, custom := range []bool{false, true} { + cfg := config.CORSConfig{} + want := map[string]any{ + "allowed_origins": []any{"*"}, "allowed_methods": []any{"GET", "HEAD", "OPTIONS"}, + "allowed_headers": []any{"Accept", "Authorization", "Content-Type"}, "allow_credentials": false, "max_age": float64(300), + } + if custom { + cfg = config.CORSConfig{AllowedOrigins: []string{"https://example.test"}, AllowedMethods: []string{"GET"}, + AllowedHeaders: []string{"Authorization", "X-Operator"}, AllowCredentials: true, MaxAge: 600} + want = map[string]any{"allowed_origins": []any{"https://example.test"}, "allowed_methods": []any{"GET"}, + "allowed_headers": []any{"Authorization", "X-Operator"}, "allow_credentials": true, "max_age": float64(600)} + } + handler := New(nil, nil, []*ingest.Worker{{}, {}}, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, CORS: cfg, Auth: config.AuthConfig{APIKey: key}}) + if custom { + cfg.AllowedOrigins[0] = "https://changed.test" + cfg.AllowedMethods[0] = "DELETE" + cfg.AllowedHeaders[0] = "X-Changed" + } + request := func(method, path string, headers map[string]string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, nil) + for name, value := range headers { + req.Header.Set(name, value) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, req) + return response + } + headers := map[string]string{"Authorization": "Bearer " + key} + response := request("GET", "/api/v1/admin/config", headers) + if response.Code != 200 || response.Header().Get("Content-Type") != "application/json" || response.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("config read status/headers: %d %v", response.Code, response.Header()) + } + var body map[string]any + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + expected := map[string]any{"cors": want, "auth": map[string]any{"configured": true}, "ingest": map[string]any{"broker_count": float64(2)}} + if !reflect.DeepEqual(body, expected) { + t.Fatalf("unexpected config shape or values: %v", body) + } + if strings.Contains(response.Body.String(), "sentinel") { + t.Fatal("credential setting was exposed") + } + for _, auth := range []string{"", "Bearer wrong"} { + denied := request("GET", "/api/v1/admin/config", map[string]string{"Authorization": auth}) + if denied.Code != 401 || strings.Contains(denied.Body.String(), "allowed_origins") { + t.Fatal("config escaped authentication") + } + } + if request("PUT", "/api/v1/admin/config", headers).Code != http.StatusUnsupportedMediaType { + t.Fatal("untyped config write was accepted") + } + again := request("GET", "/api/v1/admin/config", headers) + if again.Body.String() != response.Body.String() { + t.Fatal("startup snapshot changed") + } + preflight := request("OPTIONS", "/api/v1/admin/config", map[string]string{ + "Origin": "https://example.test", "Access-Control-Request-Method": "GET", "Access-Control-Request-Headers": "Authorization", + }) + if preflight.Code != 200 || preflight.Header().Get("Access-Control-Allow-Origin") == "" || preflight.Header().Get("Access-Control-Max-Age") != map[bool]string{false: "300", true: "600"}[custom] { + t.Fatal("CORS behavior does not match reported options") + } + } +} diff --git a/internal/api/router/admin_routes_test.go b/internal/api/router/admin_routes_test.go new file mode 100644 index 00000000..c017236c --- /dev/null +++ b/internal/api/router/admin_routes_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestAdditionalAdminRoutesStayAuthenticated(t *testing.T) { + for _, key := range []string{"", "fixture-key"} { + calls := 0 + opts := Options{Auth: config.AuthConfig{APIKey: key}, AdminRoutes: map[string]http.Handler{ + "/feature": http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.WriteHeader(http.StatusNoContent) + }), + }} + handler := New(nil, nil, nil, opts) + // Registration happens at startup; later edits to the caller's map must + // not replace the registered handler or create a new unprotected route. + delete(opts.AdminRoutes, "/feature") + for _, path := range []string{"/api/v1/admin/feature", "/api/v1/admin/feature/nested"} { + for _, method := range []string{"GET", "POST", "DELETE"} { + for _, token := range []string{"", "wrong", "fixture-key"} { + request := httptest.NewRequest(method, path, nil) + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response := httptest.NewRecorder() + before := calls + handler.ServeHTTP(response, request) + want := http.StatusServiceUnavailable + if key != "" { + want = http.StatusUnauthorized + if token == key { + want = http.StatusNoContent + } + } + if response.Code != want || response.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("%s %s: status=%d want=%d", method, path, response.Code, want) + } + if (calls != before) != (want == http.StatusNoContent) { + t.Fatal("authentication boundary bypassed") + } + } + } + } + } +} diff --git a/internal/api/router/auth_test.go b/internal/api/router/auth_test.go new file mode 100644 index 00000000..d8796f79 --- /dev/null +++ b/internal/api/router/auth_test.go @@ -0,0 +1,65 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestAdminAuthBoundary(t *testing.T) { + for _, key := range []string{"", "synthetic-test-key"} { + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, Auth: config.AuthConfig{APIKey: key}}) + for _, path := range []string{"/api/v1/admin", "/api/v1/admin/", "/api/v1/admin/config", "/api/v1/admin//config"} { + for _, method := range []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"} { + for _, token := range []string{"", "wrong", "synthetic-test-key"} { + req := httptest.NewRequest(method, path, nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + want := http.StatusServiceUnavailable + if key != "" { + want = 401 + if token == key { + want = 404 + if path == "/api/v1/admin/config" || path == "/api/v1/admin//config" { + want = 405 + if method == "GET" { + want = 200 + } + if method == "PUT" { + want = 415 + } + } + } + } + if w.Code != want { + t.Fatalf("%s %s status=%d want=%d", method, path, w.Code, want) + } + } + } + } + for path, want := range map[string]int{"/api/v1/brokers": 200, "/api/v1/administrator": 404, "/swagger": 301} { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("GET", path, nil)) + if w.Code != want { + t.Fatalf("public %s status=%d want=%d", path, w.Code, want) + } + } + req := httptest.NewRequest("OPTIONS", "/api/v1/admin/config", nil) + req.Header.Set("Origin", "https://example.test") + req.Header.Set("Access-Control-Request-Method", "GET") + req.Header.Set("Access-Control-Request-Headers", "Authorization") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != 200 || w.Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("CORS preflight blocked") + } + } +} diff --git a/internal/api/router/backup_integration_test.go b/internal/api/router/backup_integration_test.go new file mode 100644 index 00000000..18f61830 --- /dev/null +++ b/internal/api/router/backup_integration_test.go @@ -0,0 +1,172 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// This opt-in test only creates/restores/drops its own randomly named databases. +func TestBackupDownloadPostgres(t *testing.T) { + if os.Getenv("BEACON_BACKUP_TEST_POSTGRES") != "1" { + t.Skip("requires pg_dump/psql and an explicitly selected private PostgreSQL test server") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, "") + if err != nil { + t.Fatal("cannot connect to private PostgreSQL") + } + defer admin.Close(context.Background()) + sourceName, targetName := "beacon_download_"+strings.ToLower(rand.Text()), "beacon_download_"+strings.ToLower(rand.Text()) + for _, name := range []string{sourceName, targetName} { + ident := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil { + t.Fatal("cannot create private fixture") + } + defer func() { + if _, err := admin.Exec(ctx, "DROP DATABASE "+ident+" WITH (FORCE)"); err != nil { + t.Error("cannot remove private fixture") + } + }() + } + cfg, err := pgxpool.ParseConfig("") + if err != nil { + t.Fatal("invalid fixture configuration") + } + cfg.ConnConfig.Database = sourceName + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal("cannot open source fixture") + } + defer pool.Close() + if err := db.RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, "CREATE TABLE download_fixture (id bigint GENERATED ALWAYS AS IDENTITY, message text); INSERT INTO download_fixture(message) VALUES ('café 雪')"); err != nil { + t.Fatal(err) + } + dsn := (&url.URL{Scheme: "postgres", Host: net.JoinHostPort(cfg.ConnConfig.Host, strconv.Itoa(int(cfg.ConnConfig.Port))), + User: url.UserPassword(cfg.ConnConfig.User, cfg.ConnConfig.Password), Path: "/" + sourceName, + RawQuery: "sslmode=disable&pool_max_conns=2"}).String() + service, err := backup.ConnectionService(dsn) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + configPath := filepath.Join(dir, "config.yaml") + saved := []byte("# verbatim saved file\nbackup:\n enabled: true\n") + if err := os.WriteFile(configPath, saved, 0600); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", dir) + server := httptest.NewServer(New(nil, nil, nil, Options{ + Auth: config.AuthConfig{APIKey: "fixture-key"}, + AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ + ConnectionService: service, ConfigPath: configPath, MaxBytes: 16 << 20, Timeout: 30 * time.Second, Version: "http-fixture", + }, context.Background())}, + })) + defer server.Close() + request, _ := http.NewRequestWithContext(ctx, "GET", server.URL+"/api/v1/admin/backup", nil) + request.Header.Set("Authorization", "Bearer fixture-key") + response, err := server.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != 200 || response.Header.Get("Cache-Control") != "no-store" { + t.Fatalf("download status=%d", response.StatusCode) + } + gz, err := gzip.NewReader(response.Body) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tarReader, members := tar.NewReader(gz), map[string][]byte{} + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil || header.Typeflag != tar.TypeReg || header.Mode != 0600 { + t.Fatal("invalid member") + } + members[header.Name], err = io.ReadAll(tarReader) + if err != nil { + t.Fatal(err) + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + t.Fatal(err) + } + var manifest backup.Manifest + if json.Unmarshal(members["manifest.json"], &manifest) != nil || len(members) != 3 || manifest.FormatVersion != 1 || !bytes.Equal(members["config.yaml"], saved) { + t.Fatal("invalid manifest/config") + } + for _, file := range manifest.Files { + sum := sha256.Sum256(members[file.Name]) + if file.Size != int64(len(members[file.Name])) || file.SHA256 != hex.EncodeToString(sum[:]) { + t.Fatal("checksum mismatch") + } + } + if bytes.Contains(members["database.sql"], []byte("beacon_backup]")) { + t.Fatal("connection settings leaked into bundle") + } + restore := exec.CommandContext(ctx, "psql", "-X", "--set=ON_ERROR_STOP=on", "--single-transaction") + restore.Env = append(os.Environ(), "PGDATABASE="+targetName) + restore.Stdin = bytes.NewReader(members["database.sql"]) + if err := restore.Run(); err != nil { + t.Fatal("cannot restore downloaded SQL into empty fixture") + } + targetCfg := admin.Config().Copy() + targetCfg.Database = targetName + target, err := pgx.ConnectConfig(ctx, targetCfg) + if err != nil { + t.Fatal("cannot connect to restored fixture") + } + defer target.Close(context.Background()) + const fingerprint = `SELECT md5(string_agg(table_name||column_name||data_type, ',' ORDER BY table_name,ordinal_position)) FROM information_schema.columns WHERE table_schema='public'` + var before, after, message string + if pool.QueryRow(ctx, fingerprint).Scan(&before) != nil || target.QueryRow(ctx, fingerprint).Scan(&after) != nil || before != after { + t.Fatal("schema differs after restore") + } + if target.QueryRow(ctx, "SELECT message FROM download_fixture").Scan(&message) != nil || message != "café 雪" { + t.Fatal("wrong database or changed data") + } + var nextID int + if target.QueryRow(ctx, "INSERT INTO download_fixture(message) VALUES ('next') RETURNING id").Scan(&nextID) != nil || nextID != 2 { + t.Fatal("identity sequence changed") + } + paths, _ := filepath.Glob(filepath.Join(dir, "beacon-download-*")) + if len(paths) != 0 { + t.Fatal("private HTTP staging retained") + } + t.Log("HTTP bundle restored current Beacon schema and Unicode data; native client ignored ambient PGDATABASE") +} diff --git a/internal/api/router/backup_test.go b/internal/api/router/backup_test.go new file mode 100644 index 00000000..2c026529 --- /dev/null +++ b/internal/api/router/backup_test.go @@ -0,0 +1,39 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestBackupAuthenticationBeforeExport(t *testing.T) { + for _, key := range []string{"", "fixture-key"} { + handler := New(nil, nil, nil, Options{Auth: config.AuthConfig{APIKey: key}, AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ConnectionService: "fixture"}, context.Background())}}) + for _, token := range []string{"", "wrong", "fixture-key"} { + r := httptest.NewRequest("GET", "/api/v1/admin/backup?database=other", nil) + if token != "" { + r.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + want := 503 + if key != "" { + want = 401 + if token == key { + want = 400 + } + } + if w.Code != want || w.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("status=%d want=%d", w.Code, want) + } + } + } +} diff --git a/internal/api/router/options.go b/internal/api/router/options.go new file mode 100644 index 00000000..e838834e --- /dev/null +++ b/internal/api/router/options.go @@ -0,0 +1,30 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +// Options names the startup settings used by the router. Adding an optional +// route does not change New's signature or require unrelated callers to change. +type Options struct { + MaxConnsPerIP int + MaxConnectsPerMinute int + WSAllowedOrigins []string + CORS config.CORSConfig + Server config.ServerConfig + Auth config.AuthConfig + RateLimit config.ResolvedRateLimitConfig + Scopes handlers.ScopeMembership // region-filtered scope lists; nil lists none + RollupRetention time.Duration // longest historical stats window + // AdminRoutes mounts operator-only subrouters at literal paths such as + // "/accounts". Each feature owns its dependencies, methods and handlers. + // The router applies authentication to every path and method in this group. + AdminRoutes map[string]http.Handler +} diff --git a/internal/api/router/rate_limit_test.go b/internal/api/router/rate_limit_test.go new file mode 100644 index 00000000..b0e1cd86 --- /dev/null +++ b/internal/api/router/rate_limit_test.go @@ -0,0 +1,193 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "bytes" + "encoding/json" + "log" + "log/slog" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestDefaultAPIRateLimit(t *testing.T) { + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, RateLimit: config.Resolve(&config.Config{}).RateLimit}) + for i := 0; i <= 300; i++ { + request := httptest.NewRequest(http.MethodGet, "/api/v1/brokers", nil) + request.RemoteAddr = "198.51.100.1:1234" + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + want := http.StatusOK + if i == 300 { + want = http.StatusTooManyRequests + } + if response.Code != want { + t.Fatalf("request %d: got %d, want %d", i+1, response.Code, want) + } + } +} + +func rateRequest(handler http.Handler, path, peer, headerIP string) *httptest.ResponseRecorder { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.RemoteAddr = peer + request.Header.Set("Origin", "https://example.test") + request.Header.Set("X-Real-IP", headerIP) + request.Header.Set("X-Forwarded-For", headerIP) + request.Header.Set("True-Client-IP", headerIP) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response +} + +func TestAPIRateLimitContractAndLogging(t *testing.T) { + var logs bytes.Buffer + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + slog.SetDefault(slog.New(slog.NewJSONHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) + proxy := config.ServerConfig{TrustedProxies: []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}} + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, Server: proxy, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 2, Burst: 10}}) + for _, path := range []string{"/api/v1/brokers", "/api/v1/packets?limit=0"} { + response := rateRequest(handler, path, "127.0.0.1:1234", "198.51.100.25") + if response.Code != http.StatusOK && response.Code != http.StatusBadRequest { + t.Fatalf("initial request: %d", response.Code) + } + } + response := rateRequest(handler, "/api/v1/brokers", "127.0.0.1:1234", "198.51.100.25") + var body struct { + Error struct{ Code, Message string } + } + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if response.Code != http.StatusTooManyRequests || body.Error.Code != "rate_limited" || body.Error.Message == "" { + t.Fatalf("wrong rate-limit response: %d %s", response.Code, response.Body.String()) + } + if response.Header().Get("Content-Type") != "application/json" || response.Header().Get("Retry-After") != "60" || !strings.EqualFold(response.Header().Get("Access-Control-Expose-Headers"), "Retry-After") || response.Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatalf("missing JSON/backoff/CORS headers: %v", response.Header()) + } + rejections := 0 + for _, line := range bytes.Split(bytes.TrimSpace(logs.Bytes()), []byte("\n")) { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal(err) + } + if record["status"] == float64(http.StatusTooManyRequests) { + rejections++ + if record["level"] != "WARN" || record["client_ip"] != "198.51.100.25" || record["path"] != "/api/v1/*" { + t.Fatalf("wrong rejection log: %v", record) + } + } + } + if rejections != 1 { + t.Fatalf("got %d rejection logs, want 1", rejections) + } +} + +func TestAPIRateLimitClientIdentity(t *testing.T) { + for _, tc := range []struct { + name, firstPeer, secondPeer, firstHeader, secondHeader string + trusted, shared bool + }{ + {"untrusted header rotation", "198.51.100.1:1", "198.51.100.1:2", "192.0.2.1", "192.0.2.2", false, true}, + {"independent peers", "198.51.100.1:1", "198.51.100.2:1", "192.0.2.1", "192.0.2.1", false, false}, + {"trusted independent clients", "127.0.0.1:1", "127.0.0.1:2", "198.51.100.1", "198.51.100.2", true, false}, + {"trusted shared client", "127.0.0.1:1", "127.0.0.1:2", "198.51.100.1", "198.51.100.1", true, true}, + {"IPv6 shared prefix", "[2001:db8:1::1]:1", "[2001:db8:1::2]:2", "", "", false, true}, + {"IPv6 independent prefixes", "[2001:db8:1::1]:1", "[2001:db8:2::1]:2", "", "", false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + proxy := config.ServerConfig{} + if tc.trusted { + proxy.TrustedProxies = []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")} + } + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, Server: proxy, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1}}) + if response := rateRequest(handler, "/api/v1/brokers", tc.firstPeer, tc.firstHeader); response.Code != http.StatusOK { + t.Fatalf("first client: %d", response.Code) + } + want := http.StatusOK + if tc.shared { + want = http.StatusTooManyRequests + } + if response := rateRequest(handler, "/api/v1/brokers", tc.secondPeer, tc.secondHeader); response.Code != want { + t.Fatalf("second client: %d, want %d", response.Code, want) + } + }) + } +} + +func TestAPIRateLimitWindowsAndExclusions(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 4, Burst: 2}}) + request := func() *httptest.ResponseRecorder { + return rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", "") + } + preflight := httptest.NewRequest(http.MethodOptions, "/api/v1/brokers", nil) + preflight.RemoteAddr = "198.51.100.1:1" + preflight.Header.Set("Origin", "https://example.test") + preflight.Header.Set("Access-Control-Request-Method", "GET") + response := httptest.NewRecorder() + handler.ServeHTTP(response, preflight) + if response.Code != http.StatusOK || request().Code != http.StatusOK || request().Code != http.StatusOK { + t.Fatal("preflight consumed the API budget") + } + if response := request(); response.Code != http.StatusTooManyRequests || response.Header().Get("Retry-After") != "1" { + t.Fatalf("burst limit: %d %v", response.Code, response.Header()) + } + time.Sleep(2 * time.Second) + if request().Code != http.StatusOK { + t.Fatal("burst did not recover") + } + if response := request(); response.Code != http.StatusTooManyRequests || response.Header().Get("Retry-After") != "60" { + t.Fatalf("minute limit: %d %v", response.Code, response.Header()) + } + response = httptest.NewRecorder() + handler.ServeHTTP(response, preflight) + if response.Code != http.StatusOK { + t.Fatal("exhausted API budget blocked preflight") + } + if response := rateRequest(handler, "/swagger", "198.51.100.1:1", ""); response.Code != http.StatusMovedPermanently { + t.Fatalf("Swagger shares the API budget: %d", response.Code) + } + time.Sleep(2 * time.Minute) + if request().Code != http.StatusOK { + t.Fatal("minute budget did not recover") + } + }) + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, RateLimit: config.ResolvedRateLimitConfig{Enabled: false, RequestsPerMinute: 1, Burst: 1}}) + for range 5 { + if response := rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", ""); response.Code != http.StatusOK || response.Header().Get("Retry-After") != "" { + t.Fatal("disabled limiter still applied") + } + } +} + +func TestAPIRateLimitConcurrentRequests(t *testing.T) { + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 5, Burst: 5}}) + var accepted, rejected atomic.Int32 + var group sync.WaitGroup + for range 20 { + group.Go(func() { + switch rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", "").Code { + case http.StatusOK: + accepted.Add(1) + case http.StatusTooManyRequests: + rejected.Add(1) + } + }) + } + group.Wait() + if accepted.Load() != 5 || rejected.Load() != 15 { + t.Fatalf("concurrent budget: accepted=%d rejected=%d", accepted.Load(), rejected.Load()) + } +} diff --git a/internal/api/router/router.go b/internal/api/router/router.go index ce4ac666..d6bbbc97 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -3,8 +3,7 @@ // Package router wires all HTTP routes onto the Chi router and injects // dependencies (hub, reader, ingest workers) into the handler closures. -// All routes are mounted under /api/v1 with public and private groups -// stubbed for future auth middleware. +// REST routes are mounted under /api/v1; its admin subtree requires a bearer key. package router import ( @@ -12,12 +11,10 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - "github.com/go-chi/cors" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" mw "github.com/MeshCore-Beacon/beacon-server/internal/api/middleware" - "github.com/MeshCore-Beacon/beacon-server/internal/config" "github.com/MeshCore-Beacon/beacon-server/internal/hub" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/MeshCore-Beacon/beacon-server/internal/ws" @@ -39,40 +36,47 @@ import ( // /regions → regions subrouter // /stats → stats subrouter // -// The private group is stubbed and ready for the auth middleware drop-in -// described in Future Features → Admin authentication. -func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP int, corsCfg config.CORSConfig) http.Handler { +// Admin endpoints require a configured bearer key. +func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) http.Handler { r := chi.NewRouter() // ── CORS ───────────────────────────────────────────────────────────────── - allowedOrigins := corsCfg.AllowedOrigins + allowedOrigins := opts.CORS.AllowedOrigins if len(allowedOrigins) == 0 { allowedOrigins = []string{"*"} } - allowedMethods := corsCfg.AllowedMethods + allowedMethods := opts.CORS.AllowedMethods if len(allowedMethods) == 0 { allowedMethods = []string{"GET", "HEAD", "OPTIONS"} } - allowedHeaders := corsCfg.AllowedHeaders + allowedHeaders := opts.CORS.AllowedHeaders if len(allowedHeaders) == 0 { allowedHeaders = []string{"Accept", "Authorization", "Content-Type"} } - maxAge := corsCfg.MaxAge + maxAge := opts.CORS.MaxAge if maxAge == 0 { maxAge = 300 } - r.Use(cors.Handler(cors.Options{ - AllowedOrigins: allowedOrigins, - AllowedMethods: allowedMethods, - AllowedHeaders: allowedHeaders, - AllowCredentials: corsCfg.AllowCredentials, - MaxAge: maxAge, - })) + // Capture only values used by this router. Do not retain Config, credentials + // or caller-owned slices in the admin response. + adminConfig := api.AdminConfig{ + Auth: api.AdminAuthConfig{Configured: opts.Auth.APIKey != ""}, + CORS: api.AdminCORSConfig{ + AllowedOrigins: append([]string{}, allowedOrigins...), + AllowedMethods: append([]string{}, allowedMethods...), + AllowedHeaders: append([]string{}, allowedHeaders...), + AllowCredentials: opts.CORS.AllowCredentials, + MaxAge: maxAge, + }, + Ingest: api.AdminIngestConfig{BrokerCount: len(workers)}, + } + runtimeConfig := mw.NewRuntimeConfig(adminConfig, []string{"Retry-After"}) + r.Use(runtimeConfig.CORS) // ── Global middleware ──────────────────────────────────────────────────── r.Use(middleware.RequestID) - r.Use(middleware.RealIP) - r.Use(middleware.Logger) + r.Use(mw.TrustedProxyIP(opts.Server.TrustedProxies)) + r.Use(mw.RequestLogger) r.Use(middleware.Recoverer) r.Use(middleware.CleanPath) r.Use(middleware.StripSlashes) @@ -87,11 +91,12 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP )) // ── WebSocket ──────────────────────────────────────────────────────────── - r.Get("/ws", ws.Handler(h, reader, maxConnsPerIP)) + r.Get("/ws", ws.Handler(h, reader, opts.MaxConnsPerIP, opts.MaxConnectsPerMinute, opts.WSAllowedOrigins)) // ── Public REST API (v1) ───────────────────────────────────────────────── r.Route("/api/v1", func(r chi.Router) { - // Public group — no authentication required (all of v1 is public). + r.Use(mw.RateLimit(opts.RateLimit)) + // Public group — no authentication required. r.Group(func(r chi.Router) { r.Mount("/packets", handlers.PacketsRouter(reader)) r.Mount("/nodes", handlers.NodesRouter(reader)) @@ -102,18 +107,13 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP r.Mount("/iatas", handlers.IATAsRouter(reader)) r.Mount("/regions", handlers.RegionsRouter(reader)) r.Mount("/routes", handlers.RoutesRouter(reader)) - r.Mount("/scopes", handlers.ScopesRouter(reader)) - r.Mount("/stats", handlers.StatsRouter(reader)) + r.Mount("/scopes", handlers.ScopesRouter(reader, opts.Scopes)) + r.Mount("/stats", handlers.StatsRouter(reader, handlers.StatsOptions{Scopes: opts.Scopes, SeriesWindow: opts.RollupRetention})) r.Mount("/traces", handlers.TracesRouter(reader)) }) - // Private group — auth middleware applied. - // Stubbed for the admin endpoints described in Future Features. - // Swap mw.NoopAuth for a real JWT/session middleware when ready. - r.Group(func(r chi.Router) { - r.Use(mw.NoopAuth) - // r.Mount("/admin", handlers.AdminRouter()) - }) + // Protect the entire subtree, including its root and unknown paths. + r.Mount("/admin", mw.BearerAuth(opts.Auth.APIKey, handlers.AdminRouter(runtimeConfig, opts.AdminRoutes))) }) return r diff --git a/internal/api/router/router_test.go b/internal/api/router/router_test.go new file mode 100644 index 00000000..bbb434dd --- /dev/null +++ b/internal/api/router/router_test.go @@ -0,0 +1,92 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "context" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" +) + +func TestWebSocketLimitIgnoresUntrustedHeaders(t *testing.T) { + checkWebSocketLimit(t, config.ServerConfig{}, "198.51.100.2", true) +} + +func TestWebSocketLimitUsesTrustedClientIP(t *testing.T) { + cfg := config.ServerConfig{TrustedProxies: []netip.Prefix{ + netip.MustParsePrefix("127.0.0.1/8"), netip.MustParsePrefix("::1/128"), + }} + t.Run("different clients", func(t *testing.T) { + checkWebSocketLimit(t, cfg, "198.51.100.2", false) + }) + t.Run("same client", func(t *testing.T) { + checkWebSocketLimit(t, cfg, "198.51.100.1", true) + }) +} + +func checkWebSocketLimit(t *testing.T, cfg config.ServerConfig, secondIP string, wantShed bool) { + t.Helper() + server := httptest.NewServer(New(hub.New(), nil, nil, Options{MaxConnsPerIP: 1, MaxConnectsPerMinute: 10, Server: cfg, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1}})) + defer server.Close() + // Exhaust this client's REST budget before checking its independent WS cap. + for _, want := range []int{http.StatusOK, http.StatusTooManyRequests} { + request, err := http.NewRequest(http.MethodGet, server.URL+"/api/v1/brokers", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("X-Real-IP", "198.51.100.1") + response, err := server.Client().Do(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != want { + t.Fatalf("REST budget setup: %d, want %d", response.StatusCode, want) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + url := "ws" + strings.TrimPrefix(server.URL, "http") + "/ws" + dial := func(ip string) (*websocket.Conn, *http.Response, error) { + return websocket.Dial(ctx, url, &websocket.DialOptions{HTTPHeader: http.Header{ + "X-Real-Ip": {ip}, "True-Client-Ip": {ip}, "X-Forwarded-For": {ip}, + }}) + } + first, _, err := dial("198.51.100.1") + if err != nil { + t.Fatal(err) + } + defer first.CloseNow() + if _, _, err := first.Read(ctx); err != nil { + t.Fatalf("read hello: %v", err) + } + second, response, err := dial(secondIP) + if err != nil || response == nil || response.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("second handshake: response=%v err=%v", response, err) + } + defer second.CloseNow() + _, _, err = second.Read(ctx) + if wantShed && websocket.CloseStatus(err) != websocket.StatusTryAgainLater { + t.Fatalf("expected close 1013, got %v", err) + } + if !wantShed && err != nil { + t.Fatalf("independent client did not receive hello: %v", err) + } + if err := first.Write(ctx, websocket.MessageText, []byte(`{"v":1,"type":"ping","id":"still-active"}`)); err != nil { + t.Fatal(err) + } + _, pong, err := first.Read(ctx) + if err != nil || !strings.Contains(string(pong), `"type":"pong"`) { + t.Fatalf("established client stopped responding: %s %v", pong, err) + } +} diff --git a/internal/api/router/runtime_config_test.go b/internal/api/router/runtime_config_test.go new file mode 100644 index 00000000..6ac6073e --- /dev/null +++ b/internal/api/router/runtime_config_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestRuntimeConfigUpdate(t *testing.T) { + newRouter := func() http.Handler { + return New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, CORS: config.CORSConfig{AllowedOrigins: []string{"https://before.test"}}, Auth: config.AuthConfig{APIKey: "test-key"}}) + } + handler := newRouter() + request := func(method, path, body, token, media, origin string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, path, strings.NewReader(body)) + r.Header.Set("Content-Type", media) + if token != "" { + r.Header.Set("Authorization", "Bearer "+token) + } + if origin != "" { + r.Header.Set("Origin", origin) + } + if method == "OPTIONS" { + r.Header.Set("Access-Control-Request-Method", "GET") + r.Header.Set("Access-Control-Request-Headers", "Authorization") + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + return w + } + body := `{"cors":{"allowed_origins":["https://after.test"]}}` + for _, token := range []string{"", "wrong"} { + if request("PUT", "/api/v1/admin/config", body, token, "application/json", "").Code != 401 { + t.Fatal("unauthorized update") + } + } + w := request("PUT", "/api/v1/admin/config", body, "test-key", "application/json", "") + var result api.UpdateAdminConfigResponse + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &result) != nil || result.Persisted || result.RequiresRestart || w.Header().Get("Cache-Control") != "no-store" { + t.Fatal("update contract") + } + if result.Config.CORS.AllowedOrigins[0] != "https://after.test" { + t.Fatal("wrong returned policy") + } + read := request("GET", "/api/v1/admin/config", "", "test-key", "", "") + var actual api.AdminConfig + if json.Unmarshal(read.Body.Bytes(), &actual) != nil || actual.CORS.AllowedOrigins[0] != "https://after.test" { + t.Fatal("GET still reports old config") + } + for _, tc := range []struct { + body, media string + status int + }{ + {`{}`, "application/json", 400}, {`null`, "application/json", 400}, {`{"cors":null}`, "application/json", 400}, + {`{"cors":{"allowed_origins":null}}`, "application/json", 400}, {`{"cors":{"allowed_origins":[]}}`, "application/json", 400}, + {`{"auth":{"api_key":"secret-sentinel"}}`, "application/json", 400}, {`{"ingest":{"worker_count":4}}`, "application/json", 400}, + {`{"cors":{"allowed_origins":["*"],"max_age":0}}`, "application/json", 400}, + {`{"cors":{"allowed_origins":["https://user:secret-sentinel@site.test"]}}`, "application/json", 400}, + {body + ` {}`, "application/json", 400}, {body, "text/plain", 415}, {body + strings.Repeat(" ", 16384), "application/json", 413}, + } { + bad := request("PUT", "/api/v1/admin/config", tc.body, "test-key", tc.media, "") + if bad.Code != tc.status || strings.Contains(bad.Body.String(), "secret-sentinel") { + t.Fatalf("invalid update status=%d want=%d", bad.Code, tc.status) + } + if request("GET", "/api/v1/admin/config", "", "test-key", "", "").Body.String() != read.Body.String() { + t.Fatal("invalid update changed state") + } + } + if request("OPTIONS", "/api/v1/brokers", "", "", "", "https://before.test").Header().Get("Access-Control-Allow-Origin") != "" { + t.Fatal("old origin allowed") + } + if request("OPTIONS", "/api/v1/brokers", "", "", "", "https://after.test").Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("new origin blocked") + } + if exposed := request("GET", "/api/v1/brokers", "", "", "", "https://after.test").Header().Get("Access-Control-Expose-Headers"); !strings.EqualFold(exposed, "Retry-After") { + t.Fatalf("runtime update lost rate-limit backoff header: %q", exposed) + } + if request("GET", "/api/v1/brokers", "", "", "", "").Code != 200 { + t.Fatal("public reads blocked") + } + handler = newRouter() + if request("OPTIONS", "/api/v1/brokers", "", "", "", "https://before.test").Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("new router did not reload startup policy") + } +} diff --git a/internal/api/router/ws_attempt_test.go b/internal/api/router/ws_attempt_test.go new file mode 100644 index 00000000..853ecde6 --- /dev/null +++ b/internal/api/router/ws_attempt_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "net/netip" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestWebSocketAttemptClientIdentity(t *testing.T) { + for _, tc := range []struct { + name, firstPeer, secondPeer, firstHeader, secondHeader string + trusted, shared bool + }{ + {"untrusted rotation", "198.51.100.1:1", "198.51.100.1:2", "192.0.2.1", "192.0.2.2", false, true}, + {"independent peers", "198.51.100.1:1", "198.51.100.2:1", "192.0.2.1", "192.0.2.1", false, false}, + {"trusted independent clients", "127.0.0.1:1", "127.0.0.1:2", "198.51.100.1", "198.51.100.2", true, false}, + {"trusted shared client", "127.0.0.1:1", "127.0.0.1:2", "198.51.100.1", "198.51.100.1", true, true}, + {"IPv6 shared prefix", "[2001:db8:1::1]:1", "[2001:db8:1::2]:2", "", "", false, true}, + {"IPv6 independent prefixes", "[2001:db8:1::1]:1", "[2001:db8:2::1]:2", "", "", false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := config.ServerConfig{} + if tc.trusted { + cfg.TrustedProxies = []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")} + } + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1, Server: cfg}) + attempt := func(peer, header string) int { + request := httptest.NewRequest(http.MethodGet, "/ws", nil) + request.RemoteAddr = peer + request.Header.Set("X-Real-IP", header) + request.Header.Set("X-Forwarded-For", header) + request.Header.Set("True-Client-IP", header) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response.Code + } + first := attempt(tc.firstPeer, tc.firstHeader) + if first < 400 || first == http.StatusTooManyRequests { + t.Fatalf("first failed handshake: %d", first) + } + second := attempt(tc.secondPeer, tc.secondHeader) + if (second == http.StatusTooManyRequests) != tc.shared || second < 400 { + t.Fatalf("second handshake status %d; shared budget=%v", second, tc.shared) + } + }) + } +} diff --git a/internal/api/routes.go b/internal/api/routes.go index 3bc7eced..3f1bee1e 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -16,6 +16,7 @@ type RouteHop struct { // have been confirmed as high confidence. type KnownRoute struct { ID int64 `json:"id"` + PathKey string `json:"pathKey,omitempty"` // stable identity within this IATA; use for route evidence links IATA string `json:"iata"` HopCount int32 `json:"hopCount"` Hops []RouteHop `json:"hops"` diff --git a/internal/api/scopes.go b/internal/api/scopes.go index c4b23c1b..58a1fc28 100644 --- a/internal/api/scopes.go +++ b/internal/api/scopes.go @@ -3,14 +3,6 @@ package api -// ScopeSummary is the minimal scope representation used in filtered list responses. -type ScopeSummary struct { - Name string `json:"name"` - ObserverCount int64 `json:"observerCount"` - NodeCount int64 `json:"nodeCount"` - IATACount int64 `json:"iataCount"` -} - // ScopeDetail is the full scope representation including packet count and IATA list. type ScopeDetail struct { Name string `json:"name"` diff --git a/internal/api/signal.go b/internal/api/signal.go new file mode 100644 index 00000000..1514dce4 --- /dev/null +++ b/internal/api/signal.go @@ -0,0 +1,42 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +// SignalStats describes a materialized reception snapshot in [Since, Until). +// Since and Until are the effective UTC-hour boundaries, rounded down from input. +// A reception is a stored observation, not a unique packet or a lost-packet estimate. +type SignalStats struct { + Since int64 `json:"since"` + Until int64 `json:"until"` + Receptions int64 `json:"receptions"` + SNR SignalMetric `json:"snr"` + RSSI SignalMetric `json:"rssi"` + Hourly []SignalHour `json:"hourly"` +} + +// SignalMetric counts available finite samples; Average is null without samples. +type SignalMetric struct { + Samples int64 `json:"samples"` + Average *float64 `json:"average"` + Histogram []SignalBin `json:"histogram"` +} + +// SignalBin includes Lower and excludes Upper. Null denotes an unbounded end. +// SNR uses 5 dB bins from -30 to 30; RSSI uses 10 dBm bins from -140 to 0, +// both with underflow and overflow bins. These are display bins, not quality ratings. +type SignalBin struct { + Lower *float64 `json:"lower"` + Upper *float64 `json:"upper"` + Count int64 `json:"count"` +} + +// SignalHour is a complete UTC bucket in the effective window. Missing hours are omitted. +type SignalHour struct { + Hour int64 `json:"hour"` + Receptions int64 `json:"receptions"` + SNRSamples int64 `json:"snrSamples"` + SNRAverage *float64 `json:"snrAverage"` + RSSISamples int64 `json:"rssiSamples"` + RSSIAverage *float64 `json:"rssiAverage"` +} diff --git a/internal/api/stats.go b/internal/api/stats.go index 714c95d4..7e1e7596 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -5,6 +5,20 @@ package api import "github.com/google/uuid" +// ObserverComparison counts distinct flood packet hashes in [Since, Until). +// The three groups partition TotalPackets, the union heard by either observer. +// Counts describe retained reported receptions, not radio packet loss. +type ObserverComparison struct { + ObserverA uuid.UUID `json:"observerA"` + ObserverB uuid.UUID `json:"observerB"` + Since int64 `json:"since"` // inclusive, epoch milliseconds + Until int64 `json:"until"` // exclusive, epoch milliseconds + TotalPackets int64 `json:"totalPackets"` + OnlyA int64 `json:"onlyA"` + OnlyB int64 `json:"onlyB"` + Both int64 `json:"both"` +} + // RadioPreset represents a unique radio configuration observed in a given IATA, // aggregated from both observer status messages and node adverts. type RadioPreset struct { @@ -15,21 +29,58 @@ type RadioPreset struct { } // StatsOverview is the top-level network summary for the overview endpoint. +// Covers the last 24 hours that can have been rolled; hours still missing contribute nothing. type StatsOverview struct { - TotalPackets int64 `json:"totalPackets"` + TotalPackets int64 `json:"totalPackets"` // distinct per hour (see StatsSeriesValues) TotalObservations int64 `json:"totalObservations"` ActiveObservers int64 `json:"activeObservers"` ActiveIATAs int64 `json:"activeIatas"` - WindowHours int `json:"windowHours"` // always 24 for now + WindowHours int `json:"windowHours"` // complete rolled hours the totals cover, at most 24 + Since int64 `json:"since"` // epoch ms, start of the window + Until int64 `json:"until"` // epoch ms, exclusive end of the window +} + +// StatsSeries is hourly network activity read from the analytics rollups. Only complete +// hours carry values; the summary combines them (see docs/historical-stats.md). +type StatsSeries struct { + Since int64 `json:"since"` // effective start, epoch ms on a UTC hour + Until int64 `json:"until"` // effective exclusive end, epoch ms on a UTC hour + Revision int64 `json:"revision"` // changes whenever rolled hours, their status or coverage change + EarliestComplete *int64 `json:"earliestComplete"` // first complete hour held, epoch ms; null if none + CompleteHours int `json:"completeHours"` // hours contributing to summary + Hours []StatsSeriesHour `json:"hours"` + Summary StatsSeriesValues `json:"summary"` +} + +// StatsSeriesHour is one UTC hour of a StatsSeries. +type StatsSeriesHour struct { + Hour int64 `json:"hour"` // epoch ms, start of the hour + Status string `json:"status" enums:"complete,partial,missing"` // partial: raw rows deleted before it was rolled + Values *StatsSeriesValues `json:"values"` // null unless complete +} + +// StatsSeriesValues are the card metrics for an hour or a whole series. Packet counts are +// distinct per hour, so a packet heard across an hour boundary counts in both hours. +type StatsSeriesValues struct { + Observations int64 `json:"observations"` + UniquePackets int64 `json:"uniquePackets"` + ActiveObservers int64 `json:"activeObservers"` // distinct across the hour or window + ActiveIATAs int64 `json:"activeIatas"` // distinct across the hour or window + ScopedPackets int64 `json:"scopedPackets"` + ActiveScopes int64 `json:"activeScopes"` // distinct across the hour or window + MaxPathEntries int32 `json:"maxPathEntries"` + SNRSum float64 `json:"snrSum"` // average = snrSum / snrSamples + SNRSamples int64 `json:"snrSamples"` + RSSISum float64 `json:"rssiSum"` // average = rssiSum / rssiSamples + RSSISamples int64 `json:"rssiSamples"` } // ObservationPoint is a single time-bucketed observation count for charting. +// Distinct counts don't sum across IATAs, so use /stats/series for those. type ObservationPoint struct { Hour int64 `json:"hour"` // epoch ms, start of the 1-hour bucket IATA string `json:"iata"` ObservationCount int64 `json:"observationCount"` - UniquePackets int64 `json:"uniquePackets"` - ActiveObservers int64 `json:"activeObservers"` } // PayloadBreakdownItem is a single payload type with its observation count. @@ -41,21 +92,31 @@ type PayloadBreakdownItem struct { // ScopeStats represents aggregate statistics for a single transport scope. type ScopeStats struct { - Name string `json:"name"` // normalized scope name e.g. "#bc" - PacketCount int64 `json:"packetCount"` // distinct packets matched to this scope - ObserverCount int64 `json:"observerCount"` // distinct observers that forwarded packets in this scope - NodeCount int64 `json:"nodeCount"` // distinct nodes with this as their default scope + Name string `json:"name"` // normalized scope name e.g. "#bc" + PacketCount int64 `json:"packetCount"` // distinct packets matched to this scope + ObserverCount int64 `json:"observerCount"` // distinct observers that forwarded packets in this scope + NodeCount int64 `json:"nodeCount"` // distinct nodes with this as their default scope + Hourly []ScopeHour `json:"hourly"` // packetCount by hour, oldest first; hours with none omitted +} + +// ScopeHour is one UTC hour of a scope's activity; a scope's hours' packets sum to its packetCount. +type ScopeHour struct { + Hour int64 `json:"hour"` // epoch ms, start of the UTC hour + Packets int64 `json:"packets"` + Observers int64 `json:"observers"` // distinct observers that heard the scope's packets that hour + Nodes int64 `json:"nodes"` // distinct nodes whose adverts were heard in the scope that hour } -// TopNode is a node ranked by observation count from the mv_top_nodes_by_iata materialized view. +// TopNode is a node ranked by how often its adverts were heard in the window. type TopNode struct { - NodeID uuid.UUID `json:"nodeId"` - NodeName *string `json:"nodeName,omitempty"` - NodeType int16 `json:"nodeType"` - NodeTypeName string `json:"nodeTypeName"` - IATA string `json:"iata"` - ObservationCount int64 `json:"observationCount"` - LastHeard int64 `json:"lastHeard"` // epoch ms + NodeID *uuid.UUID `json:"nodeId"` // null once the node row has been deleted + PublicKey string `json:"publicKey"` // hex + NodeName *string `json:"nodeName,omitempty"` + NodeType int16 `json:"nodeType"` + NodeTypeName string `json:"nodeTypeName"` + IATA string `json:"iata"` + ObservationCount int64 `json:"observationCount"` + LastHeard int64 `json:"lastHeard"` // epoch ms } // TopObserver is an observer ranked by observation count. @@ -70,13 +131,20 @@ type TopObserver struct { // TopAdvertiser is a node ranked by distinct ADVERT packet count within the requested // window. Count is per-advert, not per-hearing -- see GetStatsTopAdvertisers. type TopAdvertiser struct { - NodeID uuid.UUID `json:"nodeId"` - NodeName *string `json:"nodeName,omitempty"` - NodeType int16 `json:"nodeType"` - NodeTypeName string `json:"nodeTypeName"` - IATA string `json:"iata"` - AdvertCount int64 `json:"advertCount"` - LastHeard int64 `json:"lastHeard"` // epoch ms + NodeID *uuid.UUID `json:"nodeId"` // null once the node row has been deleted + PublicKey string `json:"publicKey"` // hex + NodeName *string `json:"nodeName,omitempty"` + NodeType int16 `json:"nodeType"` + NodeTypeName string `json:"nodeTypeName"` + IATA string `json:"iata"` + AdvertCount int64 `json:"advertCount"` + // FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed: + // flood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path; + // direct = route type 2 (direct) or 3 (transport_direct), routed along a known path. + // FloodAdvertCount + DirectAdvertCount == AdvertCount. + FloodAdvertCount int64 `json:"floodAdvertCount"` + DirectAdvertCount int64 `json:"directAdvertCount"` + LastHeard int64 `json:"lastHeard"` // epoch ms } // TopTalker is a companion name ranked by decrypted channel message count within the @@ -94,3 +162,18 @@ type NodeTypeCount struct { NodeTypeName string `json:"nodeTypeName"` Count int64 `json:"count"` } + +// ClockDriftEntry is a repeater or room server whose most recent advert-derived clock drift +// exceeds the configured threshold (nodes.clock_drift_threshold, default 5m) -- see +// GetStatsClockDrift. Ordered worst-drift-first. ClockDriftSeconds/ClockCheckedAt mirror the +// same-named fields on Node; unlike Node this list only ever contains out-of-sync nodes, so +// there's no ClockOutOfSync bool here -- being in the list already means true. +type ClockDriftEntry struct { + NodeID uuid.UUID `json:"nodeId"` + NodeName *string `json:"nodeName,omitempty"` + NodeType int16 `json:"nodeType"` + NodeTypeName string `json:"nodeTypeName"` + ClockDriftSeconds int `json:"clockDriftSeconds"` // signed; +ve = device ahead of server + ClockCheckedAt int64 `json:"clockCheckedAt"` // epoch ms + IATAs []NodeIATA `json:"iatas,omitempty"` // IATAs this node has been heard in +} diff --git a/internal/api/telemetry.go b/internal/api/telemetry.go new file mode 100644 index 00000000..94fce69c --- /dev/null +++ b/internal/api/telemetry.go @@ -0,0 +1,164 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "math" + "time" +) + +// Fastest a counter can honestly climb, per second of wall time: radio airtime can't +// outrun the clock, and real receive errors peak well under 1/s. +const ( + airtimeRate = 1.0 + errorRate = 10.0 +) + +// counter turns a since-boot running total into increments, rejecting readings +// that can't be real: dips (partial stats) and jumps faster than rate allows +// (garbage values). Three rejections in a row mean the baseline itself was bad. +// Growth across a gap longer than maxGap can't be placed in a bucket, so it's dropped. +type counter struct { + rate float64 + maxGap int64 + last float64 + lastT int64 + seeded bool + rejects int +} + +func (c *counter) add(v *float64, t int64, rebooted bool) float64 { + if v == nil { + return 0 + } + if !c.seeded || t-c.lastT > c.maxGap { + c.last, c.lastT, c.seeded, c.rejects = *v, t, true, 0 + return 0 + } + d := *v - c.last + if rebooted && d < 0 { + d = *v // restarted with the device; some firmware keeps counting instead + } + // Rows are truncated to the telemetry resolution, so allow twice the gap. + if d < 0 || d > c.rate*2*float64(t-c.lastT)/1000 { + if c.rejects++; c.rejects < 3 { + return 0 + } + d = 0 + } + c.last, c.lastT, c.rejects = *v, t, 0 + return d +} + +type mean struct { + sum float64 + n int +} + +func (m *mean) add(v *float64) { + if v != nil { + m.sum += *v + m.n++ + } +} + +type telemetryBucket struct { + start time.Time + tx, rx, errs float64 + hasTx, hasRx, hasErr bool + battery, noise, qlen mean + uptime *int64 +} + +// BucketTelemetry folds raw telemetry rows (ascending, with any rows before since +// serving only as counter baselines) into fixed UTC buckets. Counters are summed +// increments; battery, noise floor and queue are averaged; uptime is the max. +func BucketTelemetry(points []ObserverTelemetryPoint, since time.Time, size time.Duration) []ObserverTelemetryPoint { + maxGap := size.Milliseconds() + tx := counter{rate: airtimeRate, maxGap: maxGap} + rx := counter{rate: airtimeRate, maxGap: maxGap} + errs := counter{rate: errorRate, maxGap: maxGap} + var last *ObserverTelemetryPoint + var buckets []*telemetryBucket + + for i := range points { + p := &points[i] + rebooted := false + if last != nil && p.UptimeSeconds != nil && last.UptimeSeconds != nil && *p.UptimeSeconds < *last.UptimeSeconds { + if *p.UptimeSeconds > 2*(p.T-last.T)/1000 { + continue // an old snapshot replayed, not a reboot + } + rebooted = true + } + last = p + + dTx := tx.add(f64(p.AirtimeTxSecs), p.T, rebooted) + dRx := rx.add(f64(p.AirtimeRxSecs), p.T, rebooted) + dErr := errs.add(i32f(p.ReceiveErrors), p.T, rebooted) + t := time.UnixMilli(p.T).UTC() + if t.Before(since) { + continue + } + start := t.Truncate(size) + if len(buckets) == 0 || !buckets[len(buckets)-1].start.Equal(start) { + buckets = append(buckets, &telemetryBucket{start: start}) + } + b := buckets[len(buckets)-1] + b.tx += dTx + b.rx += dRx + b.errs += dErr + b.hasTx = b.hasTx || p.AirtimeTxSecs != nil + b.hasRx = b.hasRx || p.AirtimeRxSecs != nil + b.hasErr = b.hasErr || p.ReceiveErrors != nil + b.battery.add(i32f(p.BatteryMV)) + b.noise.add(f64(p.NoiseFloorDB)) + b.qlen.add(i32f(p.QueueLength)) + if p.UptimeSeconds != nil && (b.uptime == nil || *p.UptimeSeconds > *b.uptime) { + b.uptime = p.UptimeSeconds + } + } + + out := make([]ObserverTelemetryPoint, 0, len(buckets)) + for _, b := range buckets { + pt := ObserverTelemetryPoint{T: b.start.UnixMilli(), UptimeSeconds: b.uptime} + if b.hasTx { + pt.AirtimeTxSecs = ptr(float32(b.tx)) + } + if b.hasRx { + pt.AirtimeRxSecs = ptr(float32(b.rx)) + } + if b.hasErr { + pt.ReceiveErrors = ptr(int32(b.errs)) + } + if b.battery.n > 0 { + pt.BatteryMV = ptr(int32(math.Round(b.battery.sum / float64(b.battery.n)))) + } + if b.noise.n > 0 { + pt.NoiseFloorDB = ptr(float32(b.noise.sum / float64(b.noise.n))) + } + if b.qlen.n > 0 { + pt.QueueLength = ptr(int32(math.Round(b.qlen.sum / float64(b.qlen.n)))) + } + out = append(out, pt) + } + return out +} + +func f64(v *float32) *float64 { + if v == nil { + return nil + } + f := float64(*v) + return &f +} + +func i32f(v *int32) *float64 { + if v == nil { + return nil + } + f := float64(*v) + return &f +} + +func ptr[T any](v T) *T { return &v } diff --git a/internal/api/telemetry_test.go b/internal/api/telemetry_test.go new file mode 100644 index 00000000..37e08335 --- /dev/null +++ b/internal/api/telemetry_test.go @@ -0,0 +1,199 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "testing" + "time" +) + +var telemetryBase = time.Date(2026, 9, 30, 0, 0, 0, 0, time.UTC) + +type row struct { + hour int + tx, rx float32 + errs int32 + noise float32 + uptime int64 +} + +func telemetryPoints(rows ...row) []ObserverTelemetryPoint { + pts := make([]ObserverTelemetryPoint, 0, len(rows)) + for _, r := range rows { + tx, rx, errs, noise, uptime := r.tx, r.rx, r.errs, r.noise, r.uptime + pts = append(pts, ObserverTelemetryPoint{ + T: telemetryBase.Add(time.Duration(r.hour) * time.Hour).UnixMilli(), + AirtimeTxSecs: &tx, + AirtimeRxSecs: &rx, + ReceiveErrors: &errs, + NoiseFloorDB: &noise, + UptimeSeconds: &uptime, + }) + } + return pts +} + +func bucketAt(t *testing.T, buckets []ObserverTelemetryPoint, hour int) ObserverTelemetryPoint { + t.Helper() + want := telemetryBase.Add(time.Duration(hour) * time.Hour).UnixMilli() + for _, b := range buckets { + if b.T == want { + return b + } + } + t.Fatalf("no bucket at hour %d in %+v", hour, buckets) + return ObserverTelemetryPoint{} +} + +func checkCounters(t *testing.T, b ObserverTelemetryPoint, tx, rx float32, errs int32) { + t.Helper() + if *b.AirtimeTxSecs != tx || *b.AirtimeRxSecs != rx || *b.ReceiveErrors != errs { + t.Errorf("counters = tx %v rx %v errs %v, want tx %v rx %v errs %v", + *b.AirtimeTxSecs, *b.AirtimeRxSecs, *b.ReceiveErrors, tx, rx, errs) + } +} + +func TestBucketTelemetry_SumsIncrements(t *testing.T) { + pts := telemetryPoints( + row{0, 100, 1000, 50, -90, 10000}, + row{1, 110, 1060, 55, -92, 13600}, + row{2, 125, 1100, 60, -94, 17200}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + if len(b) != 1 { + t.Fatalf("got %d buckets, want 1", len(b)) + } + checkCounters(t, b[0], 25, 100, 10) + if *b[0].NoiseFloorDB != -92 || *b[0].UptimeSeconds != 17200 { + t.Errorf("noise %v uptime %v, want -92 17200", *b[0].NoiseFloorDB, *b[0].UptimeSeconds) + } +} + +// Observer e61a62a7 on 2026-09-30: an uptime-only status stored zeros mid-run. +func TestBucketTelemetry_ZeroedRowIsNotARestart(t *testing.T) { + pts := telemetryPoints( + row{5, 7935, 47484, 93689, -96, 1093083}, + row{6, 0, 0, 0, 0, 1096701}, + row{7, 7951, 47586, 93884, -97, 1100319}, + ) + b := BucketTelemetry(pts, telemetryBase.Add(5*time.Hour), 6*time.Hour) + checkCounters(t, bucketAt(t, b, 6), 16, 102, 195) +} + +func TestBucketTelemetry_SingleCounterDip(t *testing.T) { + pts := telemetryPoints( + row{0, 3108, 19536, 38083, -108, 473828}, + row{1, 3160, 19837, 0, -92, 477447}, + row{2, 3208, 20145, 39211, -94, 481065}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 100, 609, 1128) +} + +func TestBucketTelemetry_RebootCountsSinceBoot(t *testing.T) { + pts := telemetryPoints( + row{0, 50, 500, 10, -90, 10000}, + row{1, 60, 560, 12, -90, 13600}, + row{2, 5, 40, 3, -90, 1200}, + row{3, 9, 70, 4, -90, 4800}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 10+5+4, 60+40+30, 2+3+1) +} + +// Observer 50350675 replays an old snapshot between live rows. +func TestBucketTelemetry_StaleReplaySkipped(t *testing.T) { + pts := telemetryPoints( + row{0, 121, 4788, 0, -120, 225259}, + row{1, 123, 4849, 0, -120, 228860}, + row{2, 88, 3879, 0, -50, 184130}, + row{3, 126, 4972, 0, -120, 236061}, + row{4, 88, 3879, 0, -50, 184130}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 5, 184, 0) + if *b[0].NoiseFloorDB != -120 || *b[0].UptimeSeconds != 236061 { + t.Errorf("stale row leaked into averages: noise %v uptime %v", *b[0].NoiseFloorDB, *b[0].UptimeSeconds) + } +} + +func TestBucketTelemetry_LookbackSeedsBaseline(t *testing.T) { + pts := telemetryPoints( + row{5, 100, 1000, 50, -90, 10000}, + row{6, 110, 1030, 52, -90, 13600}, + row{12, 130, 1090, 58, -90, 35200}, + ) + b := BucketTelemetry(pts, telemetryBase.Add(6*time.Hour), 6*time.Hour) + if len(b) != 2 { + t.Fatalf("got %d buckets, want 2 (lookback row must not get its own)", len(b)) + } + checkCounters(t, bucketAt(t, b, 6), 10, 30, 2) + checkCounters(t, bucketAt(t, b, 12), 20, 60, 6) +} + +// Some firmware keeps counting airtime across a reboot (ed5a20b6). +func TestBucketTelemetry_RebootWithPersistentCounters(t *testing.T) { + pts := telemetryPoints( + row{0, 900, 178000, 10, -90, 500000}, + row{1, 910, 178400, 12, -90, 503600}, + row{2, 915, 178485, 1, -90, 2250}, + row{3, 920, 178589, 3, -90, 5851}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 20, 589, 2+1+2) +} + +// Observer 408159b8 reported one-off garbage between sane readings. +func TestBucketTelemetry_GarbageSpikeRejected(t *testing.T) { + pts := telemetryPoints( + row{0, 96, 164, 144, -89, 145911}, + row{1, 101, 172, 146, -89, 149536}, + row{2, 111, 201329470, 2080374784, -90, 156847}, + row{3, 114, 197, 160, -89, 160464}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 18, 33, 16) +} + +func TestBucketTelemetry_GarbageBaselineRecovers(t *testing.T) { + pts := telemetryPoints( + row{0, 100, 2080374784, 50, -90, 10000}, + row{1, 110, 200, 55, -90, 13600}, + row{2, 120, 210, 60, -90, 17200}, + row{3, 130, 220, 65, -90, 20800}, + row{4, 140, 235, 70, -90, 24400}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 40, 15, 20) +} + +func TestBucketTelemetry_GapLongerThanBucketDropped(t *testing.T) { + pts := telemetryPoints( + row{0, 100, 22018, 50, -90, 500000}, + row{1, 110, 22100, 55, -90, 503600}, + row{20, 400, 39295, 900, -90, 572000}, + row{21, 410, 39384, 905, -90, 575600}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, bucketAt(t, b, 18), 10, 89, 5) +} + +func TestBucketTelemetry_AveragesAndNils(t *testing.T) { + b1, b2 := int32(3700), int32(3705) + q := int32(2) + pts := []ObserverTelemetryPoint{ + {T: telemetryBase.UnixMilli(), BatteryMV: &b1, QueueLength: &q}, + {T: telemetryBase.Add(time.Hour).UnixMilli(), BatteryMV: &b2}, + } + b := BucketTelemetry(pts, telemetryBase, 24*time.Hour) + if len(b) != 1 { + t.Fatalf("got %d buckets, want 1", len(b)) + } + if *b[0].BatteryMV != 3703 || *b[0].QueueLength != 2 { + t.Errorf("battery %v queue %v, want 3703 2", *b[0].BatteryMV, *b[0].QueueLength) + } + if b[0].AirtimeTxSecs != nil || b[0].NoiseFloorDB != nil || b[0].UptimeSeconds != nil { + t.Errorf("fields with no data should stay nil: %+v", b[0]) + } +} diff --git a/internal/background/background.go b/internal/background/background.go index 5e8c1896..e537328b 100644 --- a/internal/background/background.go +++ b/internal/background/background.go @@ -6,7 +6,7 @@ package background import ( "context" - "log" + "log/slog" "time" ) @@ -36,11 +36,12 @@ func (s *Scheduler) Start(ctx context.Context) { for { select { case <-ticker.C: - log.Printf("background[%s]: running", t.Name) + slog.Debug("task running", "component", "background", "task", t.Name) if err := t.Run(ctx); err != nil { - log.Printf("background[%s]: %v", t.Name, err) + slog.Error("task failed", "component", "background", "task", t.Name, "error", err) + } else { + slog.Debug("task complete", "component", "background", "task", t.Name) } - log.Printf("background[%s]: complete", t.Name) case <-ctx.Done(): return } diff --git a/internal/background/background_test.go b/internal/background/background_test.go index 18fbe851..f4978776 100644 --- a/internal/background/background_test.go +++ b/internal/background/background_test.go @@ -7,98 +7,108 @@ import ( "context" "sync/atomic" "testing" + "testing/synctest" "time" ) func TestScheduler_RunsTask(t *testing.T) { - var calls atomic.Int32 - - tasks := []Task{ - { - Name: "test-task", - Interval: 10 * time.Millisecond, - Run: func(ctx context.Context) error { - calls.Add(1) - return nil + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + + tasks := []Task{ + { + Name: "test-task", + Interval: 10 * time.Millisecond, + Run: func(ctx context.Context) error { + calls.Add(1) + return nil + }, }, - }, - } + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - s := New(tasks) - go s.Start(ctx) + s := New(tasks) + go s.Start(ctx) - time.Sleep(50 * time.Millisecond) - if calls.Load() == 0 { - t.Error("expected task to have run at least once") - } + time.Sleep(50 * time.Millisecond) + synctest.Wait() + if calls.Load() == 0 { + t.Error("expected task to have run at least once") + } + }) } func TestScheduler_StopsOnContextCancel(t *testing.T) { - var calls atomic.Int32 - - tasks := []Task{ - { - Name: "test-task", - Interval: 10 * time.Millisecond, - Run: func(ctx context.Context) error { - calls.Add(1) - return nil + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + + tasks := []Task{ + { + Name: "test-task", + Interval: 10 * time.Millisecond, + Run: func(ctx context.Context) error { + calls.Add(1) + return nil + }, }, - }, - } + } - ctx, cancel := context.WithCancel(context.Background()) - s := New(tasks) - go s.Start(ctx) + ctx, cancel := context.WithCancel(context.Background()) + s := New(tasks) + go s.Start(ctx) - time.Sleep(30 * time.Millisecond) - cancel() - time.Sleep(10 * time.Millisecond) + time.Sleep(30 * time.Millisecond) + synctest.Wait() + cancel() + synctest.Wait() - snapshot := calls.Load() - time.Sleep(30 * time.Millisecond) + snapshot := calls.Load() + time.Sleep(30 * time.Millisecond) - if calls.Load() != snapshot { - t.Errorf("expected task to stop after cancel, got %d more calls", calls.Load()-snapshot) - } + if calls.Load() != snapshot { + t.Errorf("expected task to stop after cancel, got %d more calls", calls.Load()-snapshot) + } + }) } func TestScheduler_MultipleTasksRunIndependently(t *testing.T) { - var calls1, calls2 atomic.Int32 - - tasks := []Task{ - { - Name: "task1", - Interval: 10 * time.Millisecond, - Run: func(ctx context.Context) error { - calls1.Add(1) - return nil + synctest.Test(t, func(t *testing.T) { + var calls1, calls2 atomic.Int32 + + tasks := []Task{ + { + Name: "task1", + Interval: 10 * time.Millisecond, + Run: func(ctx context.Context) error { + calls1.Add(1) + return nil + }, }, - }, - { - Name: "task2", - Interval: 10 * time.Millisecond, - Run: func(ctx context.Context) error { - calls2.Add(1) - return nil + { + Name: "task2", + Interval: 10 * time.Millisecond, + Run: func(ctx context.Context) error { + calls2.Add(1) + return nil + }, }, - }, - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - s := New(tasks) - go s.Start(ctx) - - time.Sleep(50 * time.Millisecond) - if calls1.Load() == 0 { - t.Error("expected task1 to have run") - } - if calls2.Load() == 0 { - t.Error("expected task2 to have run") - } + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s := New(tasks) + go s.Start(ctx) + + time.Sleep(50 * time.Millisecond) + synctest.Wait() + if calls1.Load() == 0 { + t.Error("expected task1 to have run") + } + if calls2.Load() == 0 { + t.Error("expected task2 to have run") + } + }) } diff --git a/internal/background/reconfirm_test.go b/internal/background/reconfirm_test.go new file mode 100644 index 00000000..991a7c4c --- /dev/null +++ b/internal/background/reconfirm_test.go @@ -0,0 +1,106 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "testing" + "testing/synctest" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" +) + +type reconfirmStore struct { + run func(context.Context, int32, time.Time) (int64, error) + ambCalls int + neighbors bool +} + +func (*reconfirmStore) DeleteOldRoutes(context.Context, time.Time, int64, time.Time) error { + return nil +} + +func (s *reconfirmStore) AmbiguousPrefixes(context.Context) (db.AmbiguousPrefixes, error) { + s.ambCalls++ + return db.AmbiguousPrefixes{}, nil +} + +func (s *reconfirmStore) ReconfirmRoutes(ctx context.Context, n int32, before time.Time, _ db.AmbiguousPrefixes) (int64, error) { + return s.run(ctx, n, before) +} + +func (s *reconfirmStore) ReconfirmNeighbors(context.Context) error { + s.neighbors = true + return nil +} + +func TestReconfirmCoverage(t *testing.T) { + for _, tc := range []struct { + name string + available, wantLeft int64 + }{ + {"empty", 0, 0}, + {"partial last batch", 1501, 0}, + {"full run budget", 750010, 10}, + } { + t.Run(tc.name, func(t *testing.T) { + left := tc.available + var cutoff time.Time + s := &reconfirmStore{run: func(ctx context.Context, n int32, before time.Time) (int64, error) { + if n <= 0 || n > 1000 { + t.Fatalf("transaction requested %d routes", n) + } + if cutoff.IsZero() { + cutoff = before + } else if !cutoff.Equal(before) { + t.Fatal("cutoff advanced, allowing completed routes to be revisited") + } + checked := min(int64(n), left) + left -= checked + return checked, ctx.Err() + }} + if err := ReconfirmTask(s, time.Hour, time.Minute, 3, time.Hour).Run(context.Background()); err != nil { + t.Fatal(err) + } + if left != tc.wantLeft || !s.neighbors { + t.Fatalf("remaining %d, neighbor cleanup %t", left, s.neighbors) + } + if s.ambCalls != 1 { + t.Fatalf("ambiguity computed %d times, want once per run", s.ambCalls) + } + }) + } +} + +func TestReconfirmStopsOnFailure(t *testing.T) { + want := errors.New("database unavailable") + calls := 0 + s := &reconfirmStore{run: func(context.Context, int32, time.Time) (int64, error) { + calls++ + if calls == 1 { + return 1000, nil + } + return 0, want + }} + err := ReconfirmTask(s, time.Hour, time.Minute, 3, time.Hour).Run(context.Background()) + if !errors.Is(err, want) || calls != 2 || s.neighbors { + t.Fatalf("failure was not propagated: error %v, calls %d, neighbors %t", err, calls, s.neighbors) + } +} + +func TestReconfirmBatchDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s := &reconfirmStore{run: func(ctx context.Context, _ int32, _ time.Time) (int64, error) { + <-ctx.Done() + return 0, ctx.Err() + }} + start := time.Now() + err := ReconfirmTask(s, time.Hour, time.Minute, 3, time.Hour).Run(context.Background()) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 5*time.Second || s.neighbors { + t.Fatalf("batch exceeded its deadline: error %v, duration %s, neighbors %t", err, time.Since(start), s.neighbors) + } + }) +} diff --git a/internal/background/rollup.go b/internal/background/rollup.go new file mode 100644 index 00000000..a23fe30c --- /dev/null +++ b/internal/background/rollup.go @@ -0,0 +1,161 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" +) + +// Per-pass budgets keep one pass short; the boot catch-up loops passes instead. +const ( + rollupInterval = 5 * time.Minute + rollupMissingBudget = 48 + rollupDirtyBatch = 24 + rollupDirtyBudget = 168 // a key import can dirty a week of hours at once +) + +// RollupSession is the lock-holding connection a pass works through (see db.RollupSession). +type RollupSession interface { + RegisterHours(ctx context.Context, since time.Time) error + MarkPartialHours(ctx context.Context) error + MissingHours(ctx context.Context, limit int32) ([]time.Time, error) + DirtyHours(ctx context.Context, limit int32) ([]time.Time, error) + RollHour(ctx context.Context, hour time.Time) (db.RollOutcome, bool, error) + Close(ctx context.Context) +} + +// Rollup rolls closed hours into the analytics rollup tables. Only the process holding +// the advisory lock does any work; the others skip the pass. +type Rollup struct { + begin func(context.Context) (RollupSession, bool, error) + packetRetention time.Duration + rollupRetention time.Duration + now func() time.Time +} + +// NewRollup rolls hours still inside both retentions using store's rollup lock. +func NewRollup(store *db.Store, packetRetention, rollupRetention time.Duration) *Rollup { + return &Rollup{ + begin: func(ctx context.Context) (RollupSession, bool, error) { + s, ok, err := store.BeginRollup(ctx) + if s == nil { + return nil, ok, err + } + return s, ok, err + }, + packetRetention: packetRetention, + rollupRetention: rollupRetention, + now: time.Now, + } +} + +// Task runs a pass every five minutes. +func (r *Rollup) Task() Task { + return Task{ + Name: "analytics_rollup", + Interval: rollupInterval, + Run: func(ctx context.Context) error { + _, _, err := r.Pass(ctx) + return err + }, + } +} + +// CatchUp runs passes until the backlog fits in one, so a restart fills gaps without +// waiting for the ticker. +func (r *Rollup) CatchUp(ctx context.Context) error { + for { + _, more, err := r.Pass(ctx) + if err != nil || !more { + return err + } + } +} + +// registerSince is the first hour worth rolling: raw rows must still exist, and cleanup +// (DeleteOldRollups, hour < now - rollupRetention) must not delete it again next run. +func (r *Rollup) registerSince() time.Time { + now := r.now() + since := now.Add(-r.packetRetention) + if rc := now.Add(-r.rollupRetention); r.rollupRetention > 0 && rc.After(since) { + since = rc + } + if h := since.Truncate(time.Hour); h.Before(since) { + return h.Add(time.Hour) + } + return since +} + +// Pass registers and rolls missing hours oldest first, then re-rolls dirty ones. A failed +// hour doesn't stop the pass; its error is returned with the others. rolled counts hours +// attempted; more reports a budget ran out with work likely left. +func (r *Rollup) Pass(ctx context.Context) (rolled int, more bool, err error) { + s, ok, err := r.begin(ctx) + if err != nil || !ok { + return 0, false, err + } + defer func() { + closeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + s.Close(closeCtx) + cancel() + }() + + if err := s.RegisterHours(ctx, r.registerSince()); err != nil { + return 0, false, fmt.Errorf("register hours: %w", err) + } + if err := s.MarkPartialHours(ctx); err != nil { + return 0, false, fmt.Errorf("mark partial hours: %w", err) + } + var errs []error + roll := func(hours []time.Time) { + for _, h := range hours { + if ctx.Err() != nil { + return + } + if _, _, err := s.RollHour(ctx, h); err != nil { + errs = append(errs, fmt.Errorf("roll %s: %w", h.UTC().Format(time.RFC3339), err)) + } + } + } + missing, err := s.MissingHours(ctx, rollupMissingBudget) + if err != nil { + return 0, false, fmt.Errorf("missing hours: %w", err) + } + roll(missing) + rolled, more = len(missing), len(missing) == rollupMissingBudget + // Hours that fail stay queued; never retry one within a pass. + tried := map[time.Time]bool{} + for budget := rollupDirtyBudget; budget > 0 && ctx.Err() == nil; { + dirty, err := s.DirtyHours(ctx, min(rollupDirtyBatch, int32(budget))) + if err != nil { + errs = append(errs, fmt.Errorf("dirty hours: %w", err)) + break + } + var fresh []time.Time + for _, h := range dirty { + if !tried[h] { + tried[h] = true + fresh = append(fresh, h) + } + } + if len(fresh) == 0 { + break + } + roll(fresh) + rolled += len(fresh) + budget -= len(fresh) + if budget == 0 { + more = true + } + } + if err := ctx.Err(); err != nil { + errs = append(errs, err) + } + return rolled, more, errors.Join(errs...) +} diff --git a/internal/background/rollup_test.go b/internal/background/rollup_test.go new file mode 100644 index 00000000..5b6d6ad6 --- /dev/null +++ b/internal/background/rollup_test.go @@ -0,0 +1,195 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" +) + +type rollupStub struct { + calls []string + since time.Time + missing [][]time.Time // one batch per pass + dirty []time.Time + failAt map[time.Time]error + done map[time.Time]bool // rolled dirty hours leave the queue + closed int +} + +func (s *rollupStub) RegisterHours(_ context.Context, since time.Time) error { + s.calls = append(s.calls, "register") + s.since = since + return nil +} +func (s *rollupStub) MarkPartialHours(context.Context) error { + s.calls = append(s.calls, "partial") + return nil +} +func (s *rollupStub) MissingHours(_ context.Context, limit int32) ([]time.Time, error) { + s.calls = append(s.calls, fmt.Sprintf("missing(%d)", limit)) + if len(s.missing) == 0 { + return nil, nil + } + batch := s.missing[0] + s.missing = s.missing[1:] + return batch, nil +} +func (s *rollupStub) DirtyHours(_ context.Context, limit int32) ([]time.Time, error) { + s.calls = append(s.calls, fmt.Sprintf("dirty(%d)", limit)) + var out []time.Time + for _, h := range s.dirty { + if !s.done[h] && int32(len(out)) < limit { + out = append(out, h) + } + } + return out, nil +} +func (s *rollupStub) RollHour(_ context.Context, h time.Time) (db.RollOutcome, bool, error) { + s.calls = append(s.calls, "roll "+h.Format("15")) + if err := s.failAt[h]; err != nil { + return db.RollSkipped, false, err + } + if s.done == nil { + s.done = map[time.Time]bool{} + } + s.done[h] = true + return db.RollComplete, true, nil +} +func (s *rollupStub) Close(context.Context) { s.closed++ } + +func newTestRollup(s *rollupStub, locked bool, beginErr error) *Rollup { + now := time.Date(2026, 1, 8, 12, 0, 0, 0, time.UTC) + return &Rollup{ + begin: func(context.Context) (RollupSession, bool, error) { + if beginErr != nil || !locked { + return nil, false, beginErr + } + return s, true, nil + }, + packetRetention: 7 * 24 * time.Hour, + rollupRetention: 90 * 24 * time.Hour, + now: func() time.Time { return now }, + } +} + +func hours(hs ...int) []time.Time { + out := make([]time.Time, len(hs)) + for i, h := range hs { + out[i] = time.Date(2026, 1, 8, h, 0, 0, 0, time.UTC) + } + return out +} + +func TestRollupPass(t *testing.T) { + s := &rollupStub{missing: [][]time.Time{hours(1, 2, 3)}, dirty: hours(0, 4)} + s.failAt = map[time.Time]error{hours(2)[0]: errors.New("boom"), hours(4)[0]: errors.New("stuck")} + rolled, more, err := newTestRollup(s, true, nil).Pass(context.Background()) + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Errorf("err = %v, want the failed hour's error", err) + } + if rolled != 5 || more { + t.Errorf("rolled = %d more = %v, want 5, false", rolled, more) + } + // A failing dirty hour stays queued but isn't retried within the pass. + want := []string{"register", "partial", "missing(48)", "roll 01", "roll 02", "roll 03", "dirty(24)", "roll 00", "roll 04", "dirty(24)"} + if !reflect.DeepEqual(s.calls, want) { + t.Errorf("calls = %v, want %v", s.calls, want) + } + if !s.since.Equal(time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)) { + t.Errorf("registered since %s, want now - packet retention", s.since) + } + if s.closed != 1 { + t.Errorf("session closed %d times", s.closed) + } +} + +func TestRollupRegisterSince(t *testing.T) { + now := time.Date(2026, 1, 8, 12, 20, 0, 0, time.UTC) + for _, tc := range []struct { + name string + packet, rollups time.Duration + want time.Time + }{ + {"packet retention shorter", 7 * 24 * time.Hour, 90 * 24 * time.Hour, time.Date(2026, 1, 1, 13, 0, 0, 0, time.UTC)}, + // Hours cleanup would delete next run must not be registered again. + {"rollup retention shorter", 30 * 24 * time.Hour, 24 * time.Hour, time.Date(2026, 1, 7, 13, 0, 0, 0, time.UTC)}, + } { + t.Run(tc.name, func(t *testing.T) { + r := &Rollup{packetRetention: tc.packet, rollupRetention: tc.rollups, now: func() time.Time { return now }} + got := r.registerSince() + if !got.Equal(tc.want) { + t.Errorf("registerSince = %s, want %s", got, tc.want) + } + if got.Before(now.Add(-tc.rollups)) { + t.Error("registers an hour DeleteOldRollups would remove") + } + }) + } +} + +func TestRollupPassLockHeldElsewhere(t *testing.T) { + s := &rollupStub{} + if rolled, _, err := newTestRollup(s, false, nil).Pass(context.Background()); err != nil || rolled != 0 || len(s.calls) != 0 { + t.Errorf("rolled %d err %v calls %v; want a silent skip", rolled, err, s.calls) + } + failure := errors.New("no connection") + if _, _, err := newTestRollup(s, true, failure).Pass(context.Background()); !errors.Is(err, failure) { + t.Errorf("err = %v, want %v", err, failure) + } +} + +func TestRollupCatchUp(t *testing.T) { + full := make([]int, rollupMissingBudget) + for i := range full { + full[i] = i % 24 + } + s := &rollupStub{missing: [][]time.Time{hours(full...), hours(full...), hours(5)}} + if err := newTestRollup(s, true, nil).CatchUp(context.Background()); err != nil { + t.Fatal(err) + } + passes := 0 + for _, c := range s.calls { + if c == "register" { + passes++ + } + } + if passes != 3 || s.closed != 3 { + t.Errorf("passes %d, closes %d; want 3 (two full batches, then a short one)", passes, s.closed) + } +} + +func TestRollupDrainsDirtyBacklog(t *testing.T) { + var backlog []time.Time + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i := range 200 { + backlog = append(backlog, start.Add(time.Duration(i)*time.Hour)) + } + s := &rollupStub{dirty: backlog} + r := newTestRollup(s, true, nil) + rolled, more, err := r.Pass(context.Background()) + if err != nil || rolled != rollupDirtyBudget || !more { + t.Fatalf("first pass rolled %d more %v err %v; want %d, true", rolled, more, err, rollupDirtyBudget) + } + if err := r.CatchUp(context.Background()); err != nil { + t.Fatal(err) + } + if len(s.done) != len(backlog) { + t.Errorf("drained %d of %d dirty hours", len(s.done), len(backlog)) + } +} + +func TestRollupTaskInterval(t *testing.T) { + task := newTestRollup(&rollupStub{}, true, nil).Task() + if task.Name != "analytics_rollup" || task.Interval != 5*time.Minute { + t.Errorf("task %q every %s", task.Name, task.Interval) + } +} diff --git a/internal/background/tasks.go b/internal/background/tasks.go index 6a5cf68f..3fabc56f 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -6,83 +6,187 @@ package background import ( "context" "fmt" - "log" + "log/slog" "time" "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/google/uuid" ) -// ViewRefreshTask returns a Task that refreshes all materialized views. -func ViewRefreshTask(store *db.Store, interval time.Duration) Task { +type viewRefresher interface { + RefreshRadioPresets(context.Context) error +} + +// ViewRefreshTask rebuilds the radio preset view; historical stats come from hourly rollups. +func ViewRefreshTask(store viewRefresher, interval time.Duration) Task { return Task{ Name: "view_refresh", Interval: interval, Run: func(ctx context.Context) error { - if err := store.RefreshHourlyStats(ctx); err != nil { - log.Printf("background[view_refresh]: hourly stats: %v", err) - } - if err := store.RefreshTopNodes(ctx); err != nil { - log.Printf("background[view_refresh]: top nodes: %v", err) - } - if err := store.RefreshTopObservers(ctx); err != nil { - log.Printf("background[view_refresh]: top observers: %v", err) - } - if err := store.RefreshPayloadBreakdown(ctx); err != nil { - log.Printf("background[view_refresh]: payload breakdown: %v", err) - } - if err := store.RefreshTopTalkers(ctx); err != nil { - log.Printf("background[view_refresh]: top talkers: %v", err) - } - if err := store.RefreshTopAdvertisers(ctx); err != nil { - log.Printf("background[view_refresh]: top advertisers: %v", err) - } if err := store.RefreshRadioPresets(ctx); err != nil { - log.Printf("background[view_refresh]: radio presets: %v", err) + return fmt.Errorf("radio presets: %w", err) } return nil }, } } -// CleanupTask returns a Task that prunes old telemetry, packet, and node rows. -func CleanupTask(store *db.Store, telemetryRetention, packetRetention, nodeDeleteAfter, interval time.Duration) Task { +type cleanupStore interface { + DeleteOldTelemetry(ctx context.Context, cutoff time.Time) error + OldestMissingRollupHour(ctx context.Context) (time.Time, bool, error) + DeleteOldPackets(ctx context.Context, cutoff time.Time) error + DeleteOldChannelIATAs(ctx context.Context, cutoff time.Time) error + DeleteOldTraceIATAs(ctx context.Context, cutoff time.Time) error + DeleteOldTraceTags(ctx context.Context, cutoff time.Time) error + DeleteOldRollups(ctx context.Context, cutoff time.Time) error + DeleteOldNodes(ctx context.Context, cutoff time.Time) error +} + +// CleanupConfig holds the retention windows CleanupTask enforces. +type CleanupConfig struct { + TelemetryRetention time.Duration + PacketRetention time.Duration + RollupRetention time.Duration + NodeDeleteAfter time.Duration + Interval time.Duration +} + +// Raw rows wait for unrolled hours, but not forever: past the cap those hours become partial. +const ( + rawHoldbackMargin = 35 * time.Minute // ingest clamps heard_at to ±30 min of last_heard_at + rawHoldbackWarn = 6 * time.Hour + rawHoldbackCap = 24 * time.Hour + traceTagLag = 30 * time.Minute +) + +// CleanupTask returns a Task that prunes old telemetry, packets, rollups and nodes. +func CleanupTask(store cleanupStore, cfg CleanupConfig) Task { + return cleanupTask(store, cfg, time.Now) +} + +func cleanupTask(store cleanupStore, cfg CleanupConfig, now func() time.Time) Task { return Task{ Name: "cleanup", - Interval: interval, + Interval: cfg.Interval, Run: func(ctx context.Context) error { - if err := store.DeleteOldTelemetry(ctx, time.Now().Add(-telemetryRetention)); err != nil { + t := now() + if err := store.DeleteOldTelemetry(ctx, t.Add(-cfg.TelemetryRetention)); err != nil { return err } - // One cutoff for all three so the IATA tables stay in step - // with the packets they mirror. - cutoff := time.Now().Add(-packetRetention) - if err := store.DeleteOldPackets(ctx, cutoff); err != nil { + cutoff, err := rawCutoff(ctx, store, t.Add(-cfg.PacketRetention)) + if err != nil { return err } - if err := store.DeleteOldChannelIATAs(ctx, cutoff); err != nil { + // One cutoff so the IATA and trace tables stay in step with the packets they mirror. + for _, del := range []func(context.Context, time.Time) error{ + store.DeleteOldPackets, store.DeleteOldChannelIATAs, store.DeleteOldTraceIATAs, + } { + if err := del(ctx, cutoff); err != nil { + return err + } + } + // Tag times are hearing times, up to the clamp behind their packets' last_heard_at; + // trailing by it keeps a re-heard packet's tag from being deleted out from under it. + if err := store.DeleteOldTraceTags(ctx, cutoff.Add(-traceTagLag)); err != nil { return err } - if err := store.DeleteOldTraceIATAs(ctx, cutoff); err != nil { + if err := store.DeleteOldRollups(ctx, t.Add(-cfg.RollupRetention)); err != nil { return err } - if err := store.DeleteOldNodes(ctx, time.Now().Add(-nodeDeleteAfter)); err != nil { + return store.DeleteOldNodes(ctx, t.Add(-cfg.NodeDeleteAfter)) + }, + } +} + +// rawCutoff holds packet deletion behind the oldest unrolled hour, up to rawHoldbackCap. +func rawCutoff(ctx context.Context, store cleanupStore, retentionCutoff time.Time) (time.Time, error) { + oldest, ok, err := store.OldestMissingRollupHour(ctx) + if err != nil || !ok { + return retentionCutoff, err + } + held := oldest.Add(-rawHoldbackMargin) + if !held.Before(retentionCutoff) { + return retentionCutoff, nil + } + holdback := retentionCutoff.Sub(held) + if holdback > rawHoldbackCap { + held, holdback = retentionCutoff.Add(-rawHoldbackCap), rawHoldbackCap + } + if holdback > rawHoldbackWarn { + slog.Warn("packet cleanup held back by unrolled analytics hours", "component", "background", + "holdback", holdback.Round(time.Minute), "oldest_missing_hour", oldest.UTC()) + } + return held, nil +} + +// Limit lock lifetime while retaining the per-run work budget. +const ( + reconfirmRunLimit = 750_000 + reconfirmBatchSize = 1_000 + reconfirmBatchTimeout = 5 * time.Second +) + +type routeMaintainer interface { + DeleteOldRoutes(context.Context, time.Time, int64, time.Time) error + AmbiguousPrefixes(context.Context) (db.AmbiguousPrefixes, error) + ReconfirmRoutes(context.Context, int32, time.Time, db.AmbiguousPrefixes) (int64, error) + ReconfirmNeighbors(context.Context) error +} + +type observerCleaner interface { + DeleteOldObservers(context.Context, time.Time) ([]uuid.UUID, error) +} + +// ObserverCleanupTask ages out unreferenced observers through the presence +// coalescer, then invalidates any cached details for the deleted IDs. +func ObserverCleanupTask(store observerCleaner, deleteAfter, interval time.Duration, onDelete func(context.Context, uuid.UUID)) Task { + return Task{ + Name: "observer_cleanup", Interval: interval, + Run: func(ctx context.Context) error { + if deleteAfter <= 0 { + return nil + } + ids, err := store.DeleteOldObservers(ctx, time.Now().Add(-deleteAfter)) + if err != nil { return err } + if onDelete != nil { + for _, id := range ids { + onDelete(ctx, id) + } + } return nil }, } } -// ReconfirmTask returns a Task that prunes stale and ambiguous resolved paths -// and neighbors. Runs after routes to ensure neighbors are cleaned against -// already-reconfirmed path data. -func ReconfirmTask(store *db.Store, interval time.Duration) Task { +// ReconfirmTask returns a Task that prunes aged routes first, then reconfirms +// stale and ambiguous resolved paths and neighbors in order. +func ReconfirmTask(store routeMaintainer, routeRetention, routeGrace time.Duration, routeMinObservations int64, interval time.Duration) Task { return Task{ Name: "reconfirm", Interval: interval, Run: func(ctx context.Context) error { - if err := store.ReconfirmRoutes(ctx); err != nil { - return fmt.Errorf("routes: %w", err) + now := time.Now() + if err := store.DeleteOldRoutes(ctx, now.Add(-routeRetention), routeMinObservations, now.Add(-routeGrace)); err != nil { + return fmt.Errorf("route retention: %w", err) + } + amb, err := store.AmbiguousPrefixes(ctx) + if err != nil { + return fmt.Errorf("ambiguous prefixes: %w", err) + } + for remaining := int64(reconfirmRunLimit); remaining > 0; { + limit := min(int64(reconfirmBatchSize), remaining) + batchCtx, cancel := context.WithTimeout(ctx, reconfirmBatchTimeout) + n, err := store.ReconfirmRoutes(batchCtx, int32(limit), now, amb) + cancel() + if err != nil { + return fmt.Errorf("routes: %w", err) + } + remaining -= n + if n < limit { + break + } } if err := store.ReconfirmNeighbors(ctx); err != nil { return fmt.Errorf("neighbors: %w", err) diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go new file mode 100644 index 00000000..7c668de4 --- /dev/null +++ b/internal/background/tasks_test.go @@ -0,0 +1,248 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log" + "log/slog" + "reflect" + "strings" + "testing" + "testing/synctest" + "time" + + "github.com/google/uuid" +) + +type refreshStub struct { + calls []string + errs map[string]error +} + +type observerCleanupStub struct { + ids []uuid.UUID + err error + calls int + cutoff time.Time +} + +func (s *observerCleanupStub) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { + s.calls++ + s.cutoff = cutoff + if err := ctx.Err(); err != nil { + return nil, err + } + return s.ids, s.err +} + +func TestObserverCleanupTask(t *testing.T) { + failure := errors.New("cleanup unavailable") + for _, tc := range []struct { + name string + retention time.Duration + err error + cancelled bool + }{ + {"disabled", 0, nil, false}, + {"negative disables", -time.Hour, nil, false}, + {"enabled", 24 * time.Hour, nil, false}, + {"failure", 24 * time.Hour, failure, false}, + {"cancelled", 24 * time.Hour, context.Canceled, true}, + } { + t.Run(tc.name, func(t *testing.T) { + store := &observerCleanupStub{ids: []uuid.UUID{uuid.New(), uuid.New()}, err: tc.err} + var invalidated []uuid.UUID + task := ObserverCleanupTask(store, tc.retention, time.Minute, func(_ context.Context, id uuid.UUID) { + invalidated = append(invalidated, id) + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if tc.cancelled { + cancel() + } + before := time.Now().Add(-tc.retention) + err := task.Run(ctx) + if !errors.Is(err, tc.err) || task.Name != "observer_cleanup" || task.Interval != time.Minute { + t.Fatalf("unexpected task outcome: %v, %+v", err, task) + } + if tc.retention <= 0 { + if store.calls != 0 || len(invalidated) != 0 { + t.Fatal("disabled retention performed work") + } + return + } + if store.calls != 1 || store.cutoff.Before(before) || store.cutoff.After(time.Now().Add(-tc.retention)) { + t.Fatal("incorrect cleanup cutoff or number of calls") + } + if tc.err != nil { + if len(invalidated) != 0 { + t.Fatal("failed cleanup invalidated cache entries") + } + } else if len(invalidated) != 2 || invalidated[0] != store.ids[0] || invalidated[1] != store.ids[1] { + t.Fatal("deleted observers were not invalidated") + } + }) + } +} + +func (s *refreshStub) refresh(ctx context.Context, name string) error { + s.calls = append(s.calls, name) + if err := ctx.Err(); err != nil { + return err + } + return s.errs[name] +} +func (s *refreshStub) RefreshRadioPresets(ctx context.Context) error { + return s.refresh(ctx, "radio presets") +} + +func TestViewRefreshTask(t *testing.T) { + failure := errors.New("refresh failed") + for _, tc := range []struct { + name string + errs map[string]error + }{ + {"success", nil}, + {"failure", map[string]error{"radio presets": failure}}, + } { + t.Run(tc.name, func(t *testing.T) { + store := &refreshStub{errs: tc.errs} + err := ViewRefreshTask(store, time.Minute).Run(context.Background()) + if len(store.calls) != 1 { + t.Fatalf("refreshed %d views, want 1", len(store.calls)) + } + if len(tc.errs) == 0 && err != nil { + t.Fatal(err) + } + if len(tc.errs) > 0 && (!errors.Is(err, failure) || !strings.Contains(err.Error(), "radio presets")) { + t.Errorf("err = %v", err) + } + }) + } +} + +func TestViewRefreshTaskCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := ViewRefreshTask(&refreshStub{}, time.Minute).Run(ctx) + if !errors.Is(err, context.Canceled) { + t.Fatalf("got %v, want cancellation", err) + } +} + +func TestSchedulerFailureIsNotComplete(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var out bytes.Buffer + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, &slog.HandlerOptions{Level: slog.LevelDebug}))) + defer func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + New([]Task{{Name: "fails", Interval: time.Millisecond, Run: func(context.Context) error { cancel(); return errors.New("refresh unavailable") }}}).Start(ctx) + time.Sleep(2 * time.Millisecond) // advance the virtual clock to the first task tick + synctest.Wait() + failed := false + for _, line := range bytes.Split(bytes.TrimSpace(out.Bytes()), []byte("\n")) { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal(err) + } + if record["msg"] == "task complete" { + t.Fatal("failed task reported complete") + } + if record["msg"] == "task failed" { + failed = true + if record["level"] != "ERROR" || record["error"] != "refresh unavailable" || record["task"] != "fails" { + t.Fatalf("incorrect failure: %v", record) + } + } + } + if !failed { + t.Fatal("scheduler did not report task failure") + } + }) +} + +type cleanupStub struct { + calls []string + cutoffs map[string]time.Time + oldest time.Time // zero = no missing hour +} + +func (s *cleanupStub) record(name string, cutoff time.Time) error { + s.calls = append(s.calls, name) + if s.cutoffs == nil { + s.cutoffs = map[string]time.Time{} + } + s.cutoffs[name] = cutoff + return nil +} +func (s *cleanupStub) DeleteOldTelemetry(_ context.Context, c time.Time) error { + return s.record("telemetry", c) +} +func (s *cleanupStub) OldestMissingRollupHour(context.Context) (time.Time, bool, error) { + s.calls = append(s.calls, "oldest") + return s.oldest, !s.oldest.IsZero(), nil +} +func (s *cleanupStub) DeleteOldPackets(_ context.Context, c time.Time) error { + return s.record("packets", c) +} +func (s *cleanupStub) DeleteOldChannelIATAs(_ context.Context, c time.Time) error { + return s.record("channel iatas", c) +} +func (s *cleanupStub) DeleteOldTraceIATAs(_ context.Context, c time.Time) error { + return s.record("trace iatas", c) +} +func (s *cleanupStub) DeleteOldTraceTags(_ context.Context, c time.Time) error { + return s.record("trace tags", c) +} +func (s *cleanupStub) DeleteOldRollups(_ context.Context, c time.Time) error { + return s.record("rollups", c) +} +func (s *cleanupStub) DeleteOldNodes(_ context.Context, c time.Time) error { + return s.record("nodes", c) +} + +func TestCleanupTask(t *testing.T) { + now := time.Date(2026, 1, 10, 12, 0, 0, 0, time.UTC) + cfg := CleanupConfig{TelemetryRetention: 31 * 24 * time.Hour, PacketRetention: 7 * 24 * time.Hour, + RollupRetention: 90 * 24 * time.Hour, NodeDeleteAfter: 30 * 24 * time.Hour, Interval: time.Hour} + retention := now.Add(-cfg.PacketRetention) + for _, tc := range []struct { + name string + oldest time.Time + want time.Time + }{ + {"no unrolled hours", time.Time{}, retention}, + {"unrolled hour after the cutoff", retention.Add(time.Hour), retention}, + {"held behind an unrolled hour", retention.Add(-2 * time.Hour), retention.Add(-2*time.Hour - 35*time.Minute)}, + {"holdback capped", retention.Add(-72 * time.Hour), retention.Add(-24 * time.Hour)}, + } { + t.Run(tc.name, func(t *testing.T) { + s := &cleanupStub{oldest: tc.oldest} + if err := cleanupTask(s, cfg, func() time.Time { return now }).Run(context.Background()); err != nil { + t.Fatal(err) + } + want := []string{"telemetry", "oldest", "packets", "channel iatas", "trace iatas", "trace tags", "rollups", "nodes"} + if !reflect.DeepEqual(s.calls, want) { + t.Errorf("calls %v, want %v", s.calls, want) + } + for _, name := range []string{"packets", "channel iatas", "trace iatas"} { + if !s.cutoffs[name].Equal(tc.want) { + t.Errorf("%s cutoff %s, want %s", name, s.cutoffs[name], tc.want) + } + } + if !s.cutoffs["trace tags"].Equal(tc.want.Add(-30 * time.Minute)) { + t.Errorf("trace tags cutoff %s, want 30 min behind packets", s.cutoffs["trace tags"]) + } + if !s.cutoffs["rollups"].Equal(now.Add(-cfg.RollupRetention)) || !s.cutoffs["telemetry"].Equal(now.Add(-cfg.TelemetryRetention)) { + t.Errorf("rollup/telemetry cutoffs %v", s.cutoffs) + } + }) + } +} diff --git a/internal/backup/client.go b/internal/backup/client.go new file mode 100644 index 00000000..55e6f975 --- /dev/null +++ b/internal/backup/client.go @@ -0,0 +1,40 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "context" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" +) + +var dumpVersion = regexp.MustCompile(`^pg_dump \(PostgreSQL\) ([0-9]+)(?:\.[0-9]+)+(?:[ \r\n]|$)`) + +// CheckClient checks the client inside this runtime before enabling downloads. +// pg_dump cannot export a server newer than its own major version. +func CheckClient(ctx context.Context, serverVersion int) error { + output, err := exec.CommandContext(ctx, "pg_dump", "--version").Output() + if err != nil { + return errors.New("cannot run pg_dump --version; install the PostgreSQL client in this runtime") + } + return checkClientVersion(output, serverVersion) +} + +func checkClientVersion(output []byte, serverVersion int) error { + match := dumpVersion.FindSubmatch(output) + if match == nil || serverVersion < 100000 { + return errors.New("cannot verify pg_dump and PostgreSQL server major versions") + } + client, err := strconv.Atoi(string(match[1])) + if err != nil { + return errors.New("cannot verify pg_dump major version") + } + if client < serverVersion/10000 { + return fmt.Errorf("pg_dump major %d is older than PostgreSQL server major %d; install a compatible client and restart Beacon", client, serverVersion/10000) + } + return nil +} diff --git a/internal/backup/client_test.go b/internal/backup/client_test.go new file mode 100644 index 00000000..605921dd --- /dev/null +++ b/internal/backup/client_test.go @@ -0,0 +1,31 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "strings" + "testing" +) + +func TestCheckClientVersion(t *testing.T) { + for _, tc := range []struct { + output string + server int + valid bool + }{ + {"pg_dump (PostgreSQL) 16.10\n", 160010, true}, + {"pg_dump (PostgreSQL) 17.6 (Debian 17.6)\n", 160010, true}, + {"pg_dump (PostgreSQL) 16.10\n", 170006, false}, + {"pg_dump (PostgreSQL) 16.10\n", 180000, false}, + {"pg_dump (PostgreSQL) 18.0\n", 180000, true}, + {"PRIVATE_CANARY", 160010, false}, + {"pg_dump (PostgreSQL) 16.bad", 160010, false}, + {"pg_dump (PostgreSQL) 16.10\n", 0, false}, + } { + err := checkClientVersion([]byte(tc.output), tc.server) + if (err == nil) != tc.valid || (err != nil && strings.Contains(err.Error(), "PRIVATE_CANARY")) { + t.Fatalf("server %d: %v", tc.server, err) + } + } +} diff --git a/internal/backup/connection.go b/internal/backup/connection.go new file mode 100644 index 00000000..59fc2fbc --- /dev/null +++ b/internal/backup/connection.go @@ -0,0 +1,97 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "errors" + "net/url" + "os" + "sort" + "strconv" + "strings" + "unicode" +) + +// ConnectionService captures a single-host PostgreSQL URL for pg_dump. The +// result contains secrets and must only be written to private staging, never +// logged or passed as a process argument. Unsupported settings fail closed. +func ConnectionService(dsn string) (string, error) { + return connectionService(dsn, os.Environ()) +} + +func connectionService(dsn string, environ []string) (string, error) { + invalid := errors.New("backup requires a single-host PostgreSQL URL with explicit user and database and supported libpq settings") + u, err := url.Parse(dsn) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.User == nil || u.User.Username() == "" || + u.Hostname() == "" || strings.Contains(u.Hostname(), ",") || strings.TrimLeft(u.Path, "/") == "" || u.Fragment != "" { + return "", invalid + } + settings := map[string]string{"host": u.Hostname(), "port": "5432", "user": u.User.Username(), + "dbname": strings.TrimLeft(u.Path, "/"), "sslmode": "prefer", "gssencmode": "disable"} + // Match the pgx startup environment for the supported native client settings. + // Target fields come exclusively from the URL. Service indirection and newer + // protocol options need separate compatibility work, not a guessed translation. + envKeys := map[string]string{"PGPASSWORD": "password", "PGPASSFILE": "passfile", "PGAPPNAME": "application_name", + "PGCONNECT_TIMEOUT": "connect_timeout", "PGSSLMODE": "sslmode", "PGSSLKEY": "sslkey", "PGSSLCERT": "sslcert", + "PGSSLROOTCERT": "sslrootcert", "PGSSLPASSWORD": "sslpassword", "PGSSLSNI": "sslsni", + "PGTARGETSESSIONATTRS": "target_session_attrs", "PGOPTIONS": "options"} + for _, entry := range environ { + key, value, _ := strings.Cut(entry, "=") + if value == "" { + continue + } + switch key { + case "PGSERVICE", "PGSERVICEFILE", "PGSSLNEGOTIATION", "PGMINPROTOCOLVERSION", "PGMAXPROTOCOLVERSION", "PGTZ": + return "", invalid + } + if name := envKeys[key]; name != "" { + settings[name] = value + } + if key == "PGPORT" && u.Port() == "" { + settings["port"] = value + } + } + if port := u.Port(); port != "" { + settings["port"] = port + } + if port, err := strconv.Atoi(settings["port"]); err != nil || port < 1 || port > 65535 { + return "", invalid + } + if password, ok := u.User.Password(); ok { + settings["password"] = password + } + query, err := url.ParseQuery(u.RawQuery) + if err != nil { + return "", invalid + } + for key, values := range query { + if len(values) != 1 || values[0] == "" { + return "", invalid + } + switch key { + case "pool_max_conns", "pool_min_conns", "pool_min_idle_conns", "pool_max_conn_lifetime", "pool_max_conn_idle_time", "pool_health_check_period", "pool_max_conn_lifetime_jitter": + continue // pgx-only pool controls; main has already validated them. + case "sslmode", "sslcert", "sslkey", "sslrootcert", "sslpassword", "sslsni", "passfile", "connect_timeout", "application_name", "target_session_attrs", "options": + settings[key] = values[0] + default: + return "", invalid + } + } + var result strings.Builder + result.WriteString("[beacon_backup]\n") + keys := make([]string, 0, len(settings)) + for key, value := range settings { + // libpq service files have line-based, unquoted values. Reject anything + // their parser would trim or interpret as another entry. + if strings.TrimSpace(value) != value || strings.IndexFunc(value, unicode.IsControl) >= 0 || len(value) > 1000 { + return "", invalid + } + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + result.WriteString(key + "=" + settings[key] + "\n") + } + return result.String(), nil +} diff --git a/internal/backup/connection_test.go b/internal/backup/connection_test.go new file mode 100644 index 00000000..e3c0b118 --- /dev/null +++ b/internal/backup/connection_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "strings" + "testing" +) + +func TestConnectionService(t *testing.T) { + service, err := connectionService("postgresql://user:p%23ass%3Dword@[::1]:5544/db%20name?sslmode=verify-full&sslrootcert=%2Fprivate%2Fca.pem&passfile=%2Fprivate%2Fpass&pool_max_conns=2", []string{ + "PGHOST=wrong", "PGDATABASE=wrong", "PGUSER=wrong", "PGPASSWORD=wrong", "PGSSLMODE=disable", "PGPASSFILE=/wrong", "PGSSLKEY=/private/key.pem"}) + if err != nil { + t.Fatal(err) + } + for _, line := range []string{"[beacon_backup]", "host=::1", "port=5544", "user=user", "password=p#ass=word", "dbname=db name", + "sslmode=verify-full", "sslrootcert=/private/ca.pem", "passfile=/private/pass", "sslkey=/private/key.pem"} { + if !strings.Contains(service, line+"\n") { + t.Fatalf("missing expected setting %q", line) + } + } + if strings.Contains(service, "wrong") || strings.Contains(service, "pool_") { + t.Fatal("unrelated target or pool settings leaked into libpq") + } + service, err = connectionService("postgres://user@host/db", []string{"PGPASSWORD=from-environment", "PGPORT=5555", "PGSSLMODE=require"}) + if err != nil || !strings.Contains(service, "password=from-environment\n") || !strings.Contains(service, "port=5555\n") || !strings.Contains(service, "sslmode=require\n") { + t.Fatal("supported environment defaults were not captured") + } +} + +func TestConnectionServiceRejectsAmbiguousSettings(t *testing.T) { + for _, dsn := range []string{ + "host=host user=user dbname=db", "postgres://host/db", "postgres://user@host/", "postgres://user@a,b/db", + "postgres://user@host/db?service=other", "postgres://user@host/db?host=other", "postgres://user@host/db?sslmode=require&sslmode=disable", + "postgres://user@host/db?unknown=x", "postgres://user:secret%0Ahost%3Dother@host/db", "postgres://user:%20secret@host/db", + "postgres://user@host:99999/db", "postgres://user@host/db#fragment", "postgres://user@host/db?sslnegotiation=direct", + } { + if _, err := connectionService(dsn, nil); err == nil || strings.Contains(err.Error(), "secret") { + t.Fatal("unsupported connection must fail without echoing input") + } + } + for _, env := range []string{"PGSERVICE=other", "PGSERVICEFILE=/private", "PGSSLNEGOTIATION=direct", "PGMINPROTOCOLVERSION=3.0", "PGTZ=UTC", "PGPORT=5432,5433"} { + if _, err := connectionService("postgres://user@host/db", []string{env}); err == nil { + t.Fatal("unsupported ambient connection settings accepted") + } + } +} diff --git a/internal/backup/export.go b/internal/backup/export.go new file mode 100644 index 00000000..e7b1a56d --- /dev/null +++ b/internal/backup/export.go @@ -0,0 +1,261 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package backup exports a database and saved configuration into a private bundle. +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +const ( + DefaultMaxBytes = int64(1 << 30) + DefaultTimeout = 10 * time.Minute + maxConfigBytes = 1 << 20 +) + +var ErrTooLarge = errors.New("backup size limit exceeded") + +// Options selects local files and finite resource limits. Connection settings +// come from libpq's PG* environment; PGDATABASE must explicitly name the database. +type Options struct { + ConfigPath string + OutputPath string + MaxBytes int64 // Maximum uncompressed database dump size. + Timeout time.Duration + Version string + // ConnectionService overrides PG* using a private service file. Construct it + // with ConnectionService at startup; empty retains the standalone PG* mode. + ConnectionService string +} + +// Manifest describes format 1; hashes cover the uncompressed archive members. +type Manifest struct { + FormatVersion int `json:"format_version"` + CreatedAt time.Time `json:"created_at"` + ToolVersion string `json:"tool_version"` + DatabaseFormat string `json:"database_format"` + Files []File `json:"files"` + Excluded []string `json:"excluded"` +} + +// File identifies one payload by its fixed archive name, size and SHA-256. +type File struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} + +// Export creates a complete .tar.gz without replacing an existing destination. +// pg_dump must be installed on PATH. The destination filesystem must support +// hard links; a link publishes the finished private file without an overwrite race. +// The bundle contains secrets if they are present in the database or saved YAML. +func Export(ctx context.Context, opts Options) error { + return export(ctx, opts, dumpCommand) +} + +func dumpCommand(ctx context.Context) *exec.Cmd { + return exec.CommandContext(ctx, "pg_dump", "--format=plain", "--no-owner", + "--no-acl", "--no-tablespaces", "--no-password", "--lock-wait-timeout=5000") +} + +func export(ctx context.Context, opts Options, command func(context.Context) *exec.Cmd) (exportErr error) { + if opts.ConfigPath == "" || opts.OutputPath == "" || opts.MaxBytes <= 0 || opts.MaxBytes > 1<<40 || opts.Timeout <= 0 { + return errors.New("config, output, positive timeout and max-bytes (at most 1 TiB) are required") + } + if opts.ConnectionService == "" && os.Getenv("PGDATABASE") == "" { + return errors.New("PGDATABASE must explicitly name the database; POSTGRES_DSN is not used") + } + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) + defer cancel() + if err := ctx.Err(); err != nil { + return err + } + if _, err := os.Lstat(opts.OutputPath); !errors.Is(err, os.ErrNotExist) { + return errors.New("output already exists or cannot be inspected") + } + config, err := readConfig(opts.ConfigPath) + if err != nil { + return err + } + parent, err := filepath.Abs(filepath.Dir(opts.OutputPath)) + if err != nil { + return errors.New("invalid output directory") + } + temp, err := os.MkdirTemp(parent, ".beacon-backup-") + if err != nil { + return errors.New("cannot create private backup staging directory") + } + defer os.RemoveAll(temp) + dump, err := os.OpenFile(filepath.Join(temp, "database.sql"), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return errors.New("cannot create private database dump") + } + defer closeBackupFile(dump, &exportErr) + hash := sha256.New() + output := &limitedWriter{ctx: ctx, cancel: cancel, dst: io.MultiWriter(dump, hash), remaining: opts.MaxBytes} + cmd := command(ctx) + if opts.ConnectionService != "" { + servicePath := filepath.Join(temp, "pg_service.conf") + if err := os.WriteFile(servicePath, []byte(opts.ConnectionService), 0600); err != nil { + return errors.New("cannot prepare private backup connection") + } + // Do not let unrelated PG* values redirect the client, and never mutate + // the Beacon process environment while another request is running. + cmd.Env = nil + for _, entry := range os.Environ() { + if !strings.HasPrefix(strings.ToUpper(entry), "PG") { + cmd.Env = append(cmd.Env, entry) + } + } + cmd.Env = append(cmd.Env, "PGSERVICEFILE="+servicePath, "PGSERVICE=beacon_backup") + } + cmd.Stdout, cmd.Stderr = output, io.Discard + cmd.WaitDelay = time.Second + err = cmd.Run() + if output.err != nil { + return output.err + } + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil { + // Client diagnostics may contain credentials, hostnames or database contents. + return errors.New("pg_dump failed; check the installed client and private connection settings") + } + size := opts.MaxBytes - output.remaining + if size == 0 { + return errors.New("pg_dump produced an empty dump") + } + if _, err = dump.Seek(0, io.SeekStart); err != nil { + return errors.New("cannot read completed database dump") + } + configHash := sha256.Sum256(config) + manifest := Manifest{ + FormatVersion: 1, CreatedAt: time.Now().UTC(), ToolVersion: opts.Version, + DatabaseFormat: "postgresql-plain-sql", + Files: []File{ + {Name: "database.sql", Size: size, SHA256: hex.EncodeToString(hash.Sum(nil))}, + {Name: "config.yaml", Size: int64(len(config)), SHA256: hex.EncodeToString(configHash[:])}, + }, + Excluded: []string{"deployment_environment", "external_config_files", "runtime_only_changes", "roles_ownership_acl_tablespaces", "redis_cache"}, + } + metadata, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return errors.New("cannot encode backup manifest") + } + archivePath := filepath.Join(temp, "bundle.tar.gz") + archive, err := os.OpenFile(archivePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return errors.New("cannot create private backup archive") + } + defer closeBackupFile(archive, &exportErr) + // Input is bounded; reserve room for YAML, tar headers and compression overhead. + archiveOutput := &limitedWriter{ctx: ctx, cancel: cancel, dst: archive, remaining: opts.MaxBytes + opts.MaxBytes/100 + 2*maxConfigBytes} + gz := gzip.NewWriter(archiveOutput) + tw := tar.NewWriter(gz) + for _, member := range []struct { + name string + size int64 + data io.Reader + }{ + {"manifest.json", int64(len(metadata)), bytes.NewReader(metadata)}, + {"database.sql", size, dump}, + {"config.yaml", int64(len(config)), bytes.NewReader(config)}, + } { + if err = tw.WriteHeader(&tar.Header{Name: member.name, Size: member.size, Mode: 0600, ModTime: manifest.CreatedAt}); err == nil { + _, err = io.Copy(tw, member.data) + } + if err != nil { + return errors.New("cannot write complete backup archive") + } + } + if err = dump.Close(); err != nil { + return errors.New("cannot close completed database dump") + } + if err = tw.Close(); err != nil { + return errors.New("cannot complete backup tar") + } + if err = gz.Close(); err != nil { + return errors.New("cannot complete backup compression") + } + if err = archive.Sync(); err != nil { + return errors.New("cannot sync completed backup") + } + if err = archive.Close(); err != nil { + return errors.New("cannot close completed backup") + } + if err = ctx.Err(); err != nil { + return err + } + if err = os.Link(archivePath, opts.OutputPath); err != nil { + return errors.New("cannot publish backup without overwriting; check destination and hard-link support") + } + return nil +} + +// Cleanup also checks close errors. Both writable files are explicitly closed +// before publication; a deferred second close may therefore report ErrClosed. +func closeBackupFile(file *os.File, exportErr *error) { + if err := file.Close(); err != nil && !errors.Is(err, os.ErrClosed) && *exportErr == nil { + *exportErr = errors.New("cannot close private backup file") + } +} + +func readConfig(path string) ([]byte, error) { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() > maxConfigBytes { + return nil, errors.New("config must be a readable regular file of at most 1 MiB") + } + f, err := os.Open(path) + if err != nil { + return nil, errors.New("cannot read saved config") + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, maxConfigBytes+1)) + if err != nil || len(data) > maxConfigBytes { + return nil, errors.New("cannot read saved config within the 1 MiB limit") + } + return data, nil +} + +type limitedWriter struct { + ctx context.Context + cancel context.CancelFunc + dst io.Writer + remaining int64 + err error +} + +func (w *limitedWriter) Write(p []byte) (n int, err error) { + if w.err == nil { + w.err = w.ctx.Err() + } + if w.err == nil && int64(len(p)) > w.remaining { + w.err = ErrTooLarge + } + if w.err == nil { + n, w.err = w.dst.Write(p) + w.remaining -= int64(n) + if w.err == nil && n != len(p) { + w.err = io.ErrShortWrite + } + } + if w.err != nil { + w.cancel() + } + return n, w.err +} diff --git a/internal/backup/export_integration_test.go b/internal/backup/export_integration_test.go new file mode 100644 index 00000000..42830573 --- /dev/null +++ b/internal/backup/export_integration_test.go @@ -0,0 +1,229 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// TestExportPostgres runs only on an explicitly selected private test server with +// pg_dump/psql installed. It creates and drops two randomly named test databases. +// BEACON_BACKUP_TEST_BINARY points to the exact compiled command being verified. +func TestExportPostgres(t *testing.T) { + if os.Getenv("BEACON_BACKUP_TEST_POSTGRES") != "1" { + t.Skip("set BEACON_BACKUP_TEST_POSTGRES=1 and BEACON_BACKUP_TEST_BINARY on a private PostgreSQL test server") + } + binary := os.Getenv("BEACON_BACKUP_TEST_BINARY") + if !filepath.IsAbs(binary) { + t.Fatal("the exact backup command must be an absolute path") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, "") + if err != nil { + t.Fatal("cannot connect to private test PostgreSQL") + } + defer admin.Close(context.Background()) + names := []string{"beacon_backup_test_" + strings.ToLower(rand.Text()), "beacon_backup_test_" + strings.ToLower(rand.Text())} + for _, name := range names { + ident := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil { + t.Fatal(err) + } + defer func() { + cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + if _, err := admin.Exec(cleanup, "DROP DATABASE "+ident); err != nil { + t.Errorf("cannot remove own test database: %v", err) + } + }() + } + poolFor := func(name string) *pgxpool.Pool { + t.Helper() + cfg, err := pgxpool.ParseConfig("") + if err != nil { + t.Fatal("invalid private test connection settings") + } + cfg.ConnConfig.Database = name + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal("cannot open own test database") + } + return pool + } + source, target := poolFor(names[0]), poolFor(names[1]) + defer source.Close() + defer target.Close() + if err := db.RunMigrations(ctx, source); err != nil { + t.Fatal(err) + } + _, err = source.Exec(ctx, ` +CREATE TABLE backup_fixture (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, message text, raw bytea, detail jsonb); +INSERT INTO backup_fixture (message, raw, detail) VALUES ('café 雪', decode('00ff10', 'hex'), '{"nested":[1,null,true]}'), ('line one +line two', NULL, '{}'); +CREATE TABLE backup_child (id bigint PRIMARY KEY REFERENCES backup_fixture(id)); +INSERT INTO backup_child VALUES (1); +CREATE VIEW backup_view AS SELECT id, message FROM backup_fixture; +CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM backup_child; +`) + if err != nil { + t.Fatal(err) + } + const check = `SELECT json_build_object( + 'rows', (SELECT json_agg(x ORDER BY id) FROM backup_fixture x), + 'children', (SELECT json_agg(x ORDER BY id) FROM backup_child x), + 'view', (SELECT json_agg(x ORDER BY id) FROM backup_view x), + 'materialized', (SELECT json_agg(x) FROM backup_materialized x), + 'migrations', (SELECT json_agg(x ORDER BY filename) FROM schema_migrations x), + 'columns', (SELECT md5(string_agg(table_name||column_name||data_type, ',' ORDER BY table_name,ordinal_position)) FROM information_schema.columns WHERE table_schema='public') +)::text` + var before, after string + if err := source.QueryRow(ctx, check).Scan(&before); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + config := []byte("# saved configuration, including a synthetic key\nchannel_keys:\n keys:\n '00': {key: '00000000000000000000000000000000', name: fixture}\n") + configPath, output := filepath.Join(dir, "config.yaml"), filepath.Join(dir, "backup.tar.gz") + if err := os.WriteFile(configPath, config, 0600); err != nil { + t.Fatal(err) + } + command := exec.CommandContext(ctx, binary, "-config", configPath, "-output", output, "-max-bytes", "16777216", "-timeout", "1m") + command.Env = append(os.Environ(), "PGDATABASE="+names[0]) + if err := command.Run(); err != nil { + t.Fatal("exact backup command failed") + } + original, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + verify := exec.CommandContext(ctx, binary, "-verify", output, "-max-bytes", "16777216", "-timeout", "30s") + verify.Env = []string{"PATH=" + dir} // verification needs no PostgreSQL client or connection + message, err := verify.CombinedOutput() + if err != nil || !bytes.Contains(message, []byte("structure and checksums verified")) || bytes.Contains(message, []byte("café")) { + t.Fatal("exact command could not verify its export offline") + } + corrupt := filepath.Join(dir, "corrupt.tar.gz") + if err := os.WriteFile(corrupt, original[:len(original)-1], 0600); err != nil { + t.Fatal(err) + } + if err := exec.CommandContext(ctx, binary, "-verify", corrupt).Run(); err == nil { + t.Fatal("exact command accepted a truncated export") + } + for _, failure := range []string{"existing", "size", "connection", "timeout"} { + t.Run(failure, func(t *testing.T) { + failedOutput := filepath.Join(dir, failure+".tar.gz") + limit, timeout, database := "16777216", "30s", names[0] + if failure == "existing" { + failedOutput = output + } else if failure == "size" { + limit = "64" + } else if failure == "connection" { + database = "beacon_backup_test_missing_" + strings.ToLower(rand.Text()) + } else { + lock, err := source.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer lock.Rollback(context.Background()) + if _, err := lock.Exec(ctx, "LOCK TABLE backup_fixture IN ACCESS EXCLUSIVE MODE"); err != nil { + t.Fatal(err) + } + timeout = "1s" + } + failed := exec.CommandContext(ctx, binary, "-config", configPath, "-output", failedOutput, "-max-bytes", limit, "-timeout", timeout) + failed.Env = append(os.Environ(), "PGDATABASE="+database) + message, err := failed.CombinedOutput() + if err == nil || bytes.Contains(message, []byte(database)) || bytes.Contains(message, []byte("café")) { + t.Fatal("expected a failure without database identity or payload diagnostics") + } + if failure == "existing" { + retained, err := os.ReadFile(output) + if err != nil || !bytes.Equal(retained, original) { + t.Fatal("existing output changed") + } + } else if _, err := os.Lstat(failedOutput); !os.IsNotExist(err) { + t.Fatal("failed export published an output") + } + staging, _ := filepath.Glob(filepath.Join(dir, ".beacon-backup-*")) + if len(staging) != 0 { + t.Fatal("failed export retained private staging") + } + }) + } + f, err := os.Open(output) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tr, members := tar.NewReader(gz), map[string][]byte{} + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + members[h.Name], err = io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + t.Fatal(err) + } + var manifest Manifest + if err := json.Unmarshal(members["manifest.json"], &manifest); err != nil { + t.Fatal(err) + } + if len(members) != 3 || manifest.FormatVersion != 1 || !bytes.Equal(members["config.yaml"], config) { + t.Fatal("unexpected bundle members or saved config") + } + for _, member := range manifest.Files { + hash := sha256.Sum256(members[member.Name]) + if member.SHA256 != hex.EncodeToString(hash[:]) || member.Size != int64(len(members[member.Name])) { + t.Fatal("bundle checksum mismatch") + } + } + restore := exec.CommandContext(ctx, "psql", "-X", "--set=ON_ERROR_STOP=on", "--single-transaction") + restore.Env = append(os.Environ(), "PGDATABASE="+names[1]) + restore.Stdin = bytes.NewReader(members["database.sql"]) + if err := restore.Run(); err != nil { + t.Fatal("restore into own empty database failed") + } + if err := target.QueryRow(ctx, check).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("schema, journal, data, relationships or materialized view changed after restore") + } + var nextID int + if err := target.QueryRow(ctx, "INSERT INTO backup_fixture (message) VALUES ('next') RETURNING id").Scan(&nextID); err != nil || nextID != 3 { + t.Fatal("identity sequence did not survive restore") + } + t.Log("exact command verified its export offline and restored all migrations, columns, Unicode/bytea/JSON data, relationships, views and identity sequence") +} diff --git a/internal/backup/export_test.go b/internal/backup/export_test.go new file mode 100644 index 00000000..a3481af7 --- /dev/null +++ b/internal/backup/export_test.go @@ -0,0 +1,232 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +const testSQL = "CREATE TABLE fixture (id bigint PRIMARY KEY, name text);\nINSERT INTO fixture VALUES (1, 'café');\n" + +// TestDumpProcess supplies an actual subprocess, including failure and cancellation. +func TestDumpProcess(t *testing.T) { + mode := os.Getenv("BEACON_BACKUP_TEST_PROCESS") + if mode == "" { + return + } + switch mode { + case "ok": + _, _ = io.WriteString(os.Stdout, testSQL) + case "fail": + _, _ = io.WriteString(os.Stdout, "incomplete dump") + _, _ = io.WriteString(os.Stderr, "PRIVATE_PASSWORD_CANARY") + os.Exit(17) + case "large": + _, _ = os.Stdout.Write(bytes.Repeat([]byte("x"), 65536)) + case "hang": + _, _ = io.WriteString(os.Stdout, "incomplete dump") + time.Sleep(time.Minute) + case "empty": + default: + os.Exit(2) + } + os.Exit(0) +} + +func helper(t *testing.T, mode string) func(context.Context) *exec.Cmd { + t.Helper() + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + return func(ctx context.Context) *exec.Cmd { + cmd := exec.CommandContext(ctx, exe, "-test.run=^TestDumpProcess$") + cmd.Env = append(os.Environ(), "BEACON_BACKUP_TEST_PROCESS="+mode) + return cmd + } +} + +func setup(t *testing.T) Options { + t.Helper() + t.Setenv("PGDATABASE", "fixture") + dir := t.TempDir() + opts := Options{ConfigPath: filepath.Join(dir, "saved.yaml"), OutputPath: filepath.Join(dir, "backup.tar.gz"), + MaxBytes: int64(len(testSQL)), Timeout: 10 * time.Second, Version: "test-revision"} + if err := os.WriteFile(opts.ConfigPath, []byte("# preserve comments and this synthetic key\nchannel_keys:\n keys:\n '00': {key: '00000000000000000000000000000000', name: fixture}\n"), 0600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + paths, err := filepath.Glob(filepath.Join(dir, ".beacon-backup-*")) + if err != nil || len(paths) != 0 { + t.Errorf("private staging was not cleaned: %v %v", paths, err) + } + }) + return opts +} + +func TestExportBundle(t *testing.T) { + opts := setup(t) + if err := export(context.Background(), opts, helper(t, "ok")); err != nil { + t.Fatal(err) + } + f, err := os.Open(opts.OutputPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + info, err := f.Stat() + if err != nil || (runtime.GOOS != "windows" && info.Mode().Perm() != 0600) { + t.Fatalf("backup must be private: %v %v", info, err) + } + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tr := tar.NewReader(gz) + files := map[string][]byte{} + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil || header.Mode != 0600 || header.Typeflag != tar.TypeReg { + t.Fatalf("invalid archive member: %v %v", header, err) + } + if _, duplicate := files[header.Name]; duplicate { + t.Fatal("duplicate archive name") + } + files[header.Name], err = io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + t.Fatal("invalid gzip footer:", err) + } + config, _ := os.ReadFile(opts.ConfigPath) + if len(files) != 3 || string(files["database.sql"]) != testSQL || !bytes.Equal(files["config.yaml"], config) { + t.Fatal("payloads differ from the supplied SQL and saved config") + } + var manifest Manifest + if err := json.Unmarshal(files["manifest.json"], &manifest); err != nil { + t.Fatal(err) + } + if manifest.FormatVersion != 1 || manifest.ToolVersion != opts.Version || manifest.CreatedAt.IsZero() || len(manifest.Files) != 2 || len(manifest.Excluded) != 5 { + t.Fatalf("incomplete manifest: %+v", manifest) + } + for _, member := range manifest.Files { + digest := sha256.Sum256(files[member.Name]) + if member.Size != int64(len(files[member.Name])) || member.SHA256 != hex.EncodeToString(digest[:]) { + t.Errorf("manifest mismatch for %s", member.Name) + } + } +} + +func TestExportFailureCleanup(t *testing.T) { + for _, mode := range []string{"fail", "large", "hang", "empty"} { + t.Run(mode, func(t *testing.T) { + opts := setup(t) + if mode == "hang" { + opts.Timeout = time.Second + } + err := export(context.Background(), opts, helper(t, mode)) + if err == nil || strings.Contains(err.Error(), "PRIVATE_PASSWORD_CANARY") { + t.Fatalf("expected sanitized failure, got %v", err) + } + if mode == "large" && !errors.Is(err, ErrTooLarge) { + t.Fatalf("wrong limit error: %v", err) + } + if mode == "hang" && !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("wrong timeout error: %v", err) + } + if _, err := os.Lstat(opts.OutputPath); !errors.Is(err, os.ErrNotExist) { + t.Fatal("failed export published an output") + } + }) + } +} + +func TestExportNeverReplacesDestination(t *testing.T) { + for _, race := range []bool{false, true} { + t.Run(map[bool]string{false: "existing", true: "created-during-export"}[race], func(t *testing.T) { + opts := setup(t) + putExisting := func() { + if err := os.WriteFile(opts.OutputPath, []byte("retain this backup"), 0600); err != nil { + t.Fatal(err) + } + } + if !race { + putExisting() + } + cmd := helper(t, "ok") + err := export(context.Background(), opts, func(ctx context.Context) *exec.Cmd { + if !race { + t.Fatal("existing output must be rejected before starting pg_dump") + } + putExisting() + return cmd(ctx) + }) + data, readErr := os.ReadFile(opts.OutputPath) + if err == nil || readErr != nil || string(data) != "retain this backup" { + t.Fatalf("existing backup changed: %v %v", err, readErr) + } + }) + } +} + +func TestExportValidation(t *testing.T) { + for _, mode := range []string{"database", "limit", "config", "canceled"} { + t.Run(mode, func(t *testing.T) { + opts := setup(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + switch mode { + case "database": + t.Setenv("PGDATABASE", "") + case "limit": + opts.MaxBytes = 0 + case "config": + if err := os.WriteFile(opts.ConfigPath, make([]byte, maxConfigBytes+1), 0600); err != nil { + t.Fatal(err) + } + case "canceled": + cancel() + } + if err := export(ctx, opts, func(context.Context) *exec.Cmd { t.Fatal("dump ran on invalid input"); return nil }); err == nil { + t.Fatal("invalid export succeeded") + } + }) + } +} + +func TestDumpArgumentsExcludeConnectionSecrets(t *testing.T) { + t.Setenv("PGPASSWORD", "PRIVATE_PASSWORD_CANARY") + t.Setenv("POSTGRES_DSN", "postgres://user:PRIVATE_PASSWORD_CANARY@localhost/db") + cmd := dumpCommand(context.Background()) + if strings.Contains(strings.Join(cmd.Args, " "), "PRIVATE_PASSWORD_CANARY") { + t.Fatal("credentials reached command arguments") + } + args := strings.Join(cmd.Args, " ") + for _, required := range []string{"--no-password", "--lock-wait-timeout=5000", "--no-owner", "--no-acl", "--no-tablespaces", "--format=plain"} { + if !strings.Contains(args, required) { + t.Errorf("missing dump boundary: %s", required) + } + } +} diff --git a/internal/backup/verify.go b/internal/backup/verify.go new file mode 100644 index 00000000..68847452 --- /dev/null +++ b/internal/backup/verify.go @@ -0,0 +1,212 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "strings" +) + +const maxManifestBytes = 64 << 10 + +var ErrInvalidArchive = errors.New("invalid backup archive") + +// Verify streams a format-1 bundle without extracting files or executing SQL. +// maxBytes bounds the SQL payload (up to 1 TiB); the saved YAML and manifest have +// separate fixed limits. The caller owns input and its deadline. Cancellation is +// checked between reads; the supplied Reader must not block indefinitely. +// The returned manifest is untrusted metadata, not proof of origin or restorability. +func Verify(ctx context.Context, input io.Reader, maxBytes int64) (*Manifest, error) { + if input == nil || maxBytes <= 0 || maxBytes > 1<<40 { + return nil, errors.New("input and positive max-bytes (at most 1 TiB) are required") + } + // Match the exporter's compressed-output allowance. ByteReader prevents gzip + // from swallowing a following stream before we can reject it. + compressed := &verifyReader{ctx: ctx, r: input, limit: maxBytes + maxBytes/100 + 2*maxConfigBytes} + buffered := bufio.NewReader(compressed) + gz, err := gzip.NewReader(buffered) + if err != nil { + return nil, verificationError(ctx, err, "gzip header") + } + defer gz.Close() + gz.Multistream(false) + decoded := &verifyReader{ctx: ctx, r: gz} + tr := tar.NewReader(decoded) + files := make(map[string]File, 3) + var metadata []byte + var padding int64 + for { + before := decoded.n + // Native headers are USTAR, or one size-only PAX record for SQL >8 GiB. + // Bound Next itself: it otherwise hides and allocates extension records. + decoded.limit = before + padding + 3*512 + h, err := tr.Next() + if errors.Is(err, ErrTooLarge) { + // The 3-block header budget is a structure check, not the size limit. + return nil, verificationError(ctx, nil, "tar header") + } + if err == io.EOF { + // tar.Reader also accepts EOF without both closing zero blocks. + if decoded.n-before != padding+2*512 { + return nil, verificationError(ctx, nil, "missing tar terminator") + } + break + } + if err != nil { + return nil, verificationError(ctx, err, "tar header") + } + headerBytes := decoded.n - before - padding + ordinary := h.Format == tar.FormatUSTAR && headerBytes == 512 && len(h.PAXRecords) == 0 + large := h.Format == tar.FormatPAX && headerBytes == 3*512 && len(h.PAXRecords) == 1 && h.PAXRecords["size"] == strconv.FormatInt(h.Size, 10) + if (!ordinary && !large) || (h.Typeflag != tar.TypeReg && h.Typeflag != tar.TypeRegA) || h.Linkname != "" || h.Mode != 0600 { + return nil, verificationError(ctx, nil, "unsupported member type or metadata") + } + if _, duplicate := files[h.Name]; duplicate || len(files) == 3 { + return nil, verificationError(ctx, nil, "duplicate or extra member") + } + var limit int64 + switch h.Name { + case "manifest.json": + limit = maxManifestBytes + case "database.sql": + limit = maxBytes + if h.Size == 0 { + return nil, verificationError(ctx, nil, "empty database dump") + } + case "config.yaml": + limit = maxConfigBytes + default: + return nil, verificationError(ctx, nil, "unexpected member name") + } + if h.Size < 0 || h.Size > limit { + return nil, ErrTooLarge + } + decoded.limit = decoded.n + h.Size + digest := sha256.New() + if h.Name == "manifest.json" { + metadata, err = io.ReadAll(tr) + } else { + _, err = io.Copy(digest, tr) + } + if err != nil { + return nil, verificationError(ctx, err, "member data") + } + files[h.Name] = File{Name: h.Name, Size: h.Size, SHA256: hex.EncodeToString(digest.Sum(nil))} + padding = (512 - h.Size%512) % 512 + } + // Read through the gzip checksum/trailer, but allow no data after tar EOF. + decoded.limit = decoded.n + var tail [1]byte + if _, err := decoded.Read(tail[:]); err != io.EOF { + if errors.Is(err, ErrTooLarge) { + err = nil // bytes after the tar terminator are corruption, not oversize + } + return nil, verificationError(ctx, err, "gzip completion or trailing tar data") + } + if _, err := buffered.ReadByte(); err != io.EOF { + return nil, verificationError(ctx, err, "trailing compressed data") + } + if len(files) != 3 || uniqueManifestKeys(json.NewDecoder(bytes.NewReader(metadata)), 0) != nil { + return nil, verificationError(ctx, nil, "missing members or invalid manifest") + } + var manifest Manifest + decoder := json.NewDecoder(bytes.NewReader(metadata)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&manifest); err != nil || decoder.Decode(new(any)) != io.EOF || manifest.FormatVersion != 1 || manifest.DatabaseFormat != "postgresql-plain-sql" || manifest.CreatedAt.IsZero() || len(manifest.Files) != 2 { + return nil, verificationError(ctx, nil, "unsupported or incomplete manifest") + } + for _, declared := range manifest.Files { + actual, exists := files[declared.Name] + digest, err := hex.DecodeString(declared.SHA256) + if !exists || declared.Name == "manifest.json" || declared.Size != actual.Size || err != nil || len(digest) != sha256.Size || !strings.EqualFold(declared.SHA256, actual.SHA256) { + return nil, verificationError(ctx, nil, "manifest size or checksum mismatch") + } + delete(files, declared.Name) // duplicate manifest file records fail as well. + } + if err := ctx.Err(); err != nil { + return nil, err + } + return &manifest, nil +} + +// Avoid returning filenames, manifest values, reader errors or payload text. +func verificationError(ctx context.Context, err error, stage string) error { + if ctx.Err() != nil { + return ctx.Err() + } + if errors.Is(err, ErrTooLarge) { + return ErrTooLarge + } + return fmt.Errorf("%w: %s", ErrInvalidArchive, stage) +} + +type verifyReader struct { + ctx context.Context + r io.Reader + n, limit int64 +} + +func (r *verifyReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + if r.n > r.limit { + return 0, ErrTooLarge + } + if int64(len(p)) > r.limit-r.n+1 { + p = p[:r.limit-r.n+1] + } + n, err := r.r.Read(p) + r.n += int64(n) + if r.n > r.limit { + return 0, ErrTooLarge + } + return n, err +} + +// encoding/json accepts duplicate keys and case aliases. Reject both before +// decoding the fixed schema, so another reader cannot interpret different values. +func uniqueManifestKeys(d *json.Decoder, depth int) error { + if depth > 8 { + return ErrInvalidArchive + } + token, err := d.Token() + if err != nil { + return err + } + if delim, ok := token.(json.Delim); ok { + keys := map[string]bool{} + for d.More() { + if delim == '{' { + key, err := d.Token() + if err != nil { + return err + } + name, ok := key.(string) + // Schema keys are ASCII. Unicode simple-fold aliases (such as + // long-s for s) also match encoding/json fields, even in lowercase. + if !ok || keys[name] || strings.IndexFunc(name, func(r rune) bool { return r > 127 || r >= 'A' && r <= 'Z' }) >= 0 { + return ErrInvalidArchive + } + keys[name] = true + } + if err := uniqueManifestKeys(d, depth+1); err != nil { + return err + } + } + _, err = d.Token() + } + return err +} diff --git a/internal/backup/verify_test.go b/internal/backup/verify_test.go new file mode 100644 index 00000000..06eb1239 --- /dev/null +++ b/internal/backup/verify_test.go @@ -0,0 +1,266 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "strings" + "testing" + "time" +) + +type verifyEntry struct { + name string + data []byte + kind byte + pax map[string]string +} + +func verifyFixture(t testing.TB) []verifyEntry { + t.Helper() + sql, config := []byte(testSQL), []byte("secret: PRIVATE_CANARY\n") + m := Manifest{FormatVersion: 1, CreatedAt: time.Now().UTC(), ToolVersion: "fixture", DatabaseFormat: "postgresql-plain-sql", Excluded: []string{"deployment_environment"}} + for _, entry := range []verifyEntry{{name: "database.sql", data: sql}, {name: "config.yaml", data: config}} { + hash := sha256.Sum256(entry.data) + m.Files = append(m.Files, File{Name: entry.name, Size: int64(len(entry.data)), SHA256: hex.EncodeToString(hash[:])}) + } + metadata, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return []verifyEntry{{name: "manifest.json", data: metadata}, {name: "database.sql", data: sql}, {name: "config.yaml", data: config}} +} + +func verifyTar(t testing.TB, entries []verifyEntry) []byte { + t.Helper() + var data bytes.Buffer + tw := tar.NewWriter(&data) + for _, e := range entries { + h := &tar.Header{Name: e.name, Size: int64(len(e.data)), Mode: 0600, Typeflag: e.kind} + if e.kind == tar.TypeSymlink || e.kind == tar.TypeLink { + h.Linkname = "../PRIVATE_CANARY" + } + if e.pax != nil { + h.Format, h.PAXRecords = tar.FormatPAX, e.pax + } + if err := tw.WriteHeader(h); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(e.data); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + return data.Bytes() +} + +func verifyGzip(t testing.TB, data []byte) []byte { + t.Helper() + var encoded bytes.Buffer + gz := gzip.NewWriter(&encoded) + if _, err := gz.Write(data); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return encoded.Bytes() +} + +func TestVerifyExportAndReorderedMembers(t *testing.T) { + opts := setup(t) + if err := export(context.Background(), opts, helper(t, "ok")); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(opts.OutputPath) + if err != nil { + t.Fatal(err) + } + m, err := Verify(context.Background(), bytes.NewReader(original), opts.MaxBytes) + if err != nil || m.ToolVersion != opts.Version || len(m.Files) != 2 { + t.Fatalf("native export: %v %v", m, err) + } + entries := verifyFixture(t) + entries[0], entries[2] = entries[2], entries[0] + if _, err := Verify(context.Background(), bytes.NewReader(verifyGzip(t, verifyTar(t, entries))), int64(len(testSQL))); err != nil { + t.Fatal("valid reordered members:", err) + } + if current, err := os.ReadFile(opts.OutputPath); err != nil || !bytes.Equal(original, current) { + t.Fatal("verification changed input") + } +} + +func TestVerifyRejectsInvalidMembersAndManifest(t *testing.T) { + for _, tc := range []struct { + name string + change func([]verifyEntry) []verifyEntry + }{ + {"missing", func(e []verifyEntry) []verifyEntry { return e[:2] }}, + {"duplicate", func(e []verifyEntry) []verifyEntry { return append(e, e[1]) }}, + {"traversal", func(e []verifyEntry) []verifyEntry { e[2].name = "../PRIVATE_CANARY"; return e }}, + {"absolute", func(e []verifyEntry) []verifyEntry { e[2].name = "/config.yaml"; return e }}, + {"windows path", func(e []verifyEntry) []verifyEntry { e[2].name = `C:\config.yaml`; return e }}, + {"directory prefix", func(e []verifyEntry) []verifyEntry { e[2].name = "./config.yaml"; return e }}, + {"symlink", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeSymlink, nil; return e }}, + {"hardlink", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeLink, nil; return e }}, + {"device", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeChar, nil; return e }}, + {"directory", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeDir, nil; return e }}, + {"payload tamper", func(e []verifyEntry) []verifyEntry { e[1].data[0] ^= 1; return e }}, + {"large sql", func(e []verifyEntry) []verifyEntry { e[1].data = bytes.Repeat([]byte("x"), len(testSQL)+1); return e }}, + {"large config", func(e []verifyEntry) []verifyEntry { e[2].data = bytes.Repeat([]byte("x"), maxConfigBytes+1); return e }}, + {"large manifest", func(e []verifyEntry) []verifyEntry { + e[0].data = bytes.Repeat([]byte("x"), maxManifestBytes+1) + return e + }}, + {"extended attributes", func(e []verifyEntry) []verifyEntry { + e[1].pax = map[string]string{"SCHILY.xattr.secret": "PRIVATE_CANARY"} + return e + }}, + {"hidden oversized metadata", func(e []verifyEntry) []verifyEntry { + e[1].pax = map[string]string{"comment": strings.Repeat("x", 8000)} + return e + }}, + {"unknown version", func(e []verifyEntry) []verifyEntry { + e[0].data = bytes.Replace(e[0].data, []byte(`"format_version":1`), []byte(`"format_version":2`), 1) + return e + }}, + {"duplicate key", func(e []verifyEntry) []verifyEntry { + e[0].data = append([]byte(`{"format_version":2,`), e[0].data[1:]...) + return e + }}, + {"case alias", func(e []verifyEntry) []verifyEntry { + e[0].data = bytes.Replace(e[0].data, []byte("format_version"), []byte("FORMAT_VERSION"), 1) + return e + }}, + {"unicode case alias", func(e []verifyEntry) []verifyEntry { + e[0].data = bytes.Replace(e[0].data, []byte("database_format"), []byte(`databa\u017fe_format`), 1) + return e + }}, + {"duplicate file record", func(e []verifyEntry) []verifyEntry { + var m Manifest + _ = json.Unmarshal(e[0].data, &m) + m.Files[1] = m.Files[0] + e[0].data, _ = json.Marshal(m) + return e + }}, + {"claimed size", func(e []verifyEntry) []verifyEntry { + var m Manifest + _ = json.Unmarshal(e[0].data, &m) + m.Files[0].Size++ + e[0].data, _ = json.Marshal(m) + return e + }}, + {"unknown field", func(e []verifyEntry) []verifyEntry { + e[0].data = append([]byte(`{"private":"PRIVATE_CANARY",`), e[0].data[1:]...) + return e + }}, + {"second json value", func(e []verifyEntry) []verifyEntry { e[0].data = append(e[0].data, []byte(`{}`)...); return e }}, + } { + t.Run(tc.name, func(t *testing.T) { + data := verifyGzip(t, verifyTar(t, tc.change(verifyFixture(t)))) + _, err := Verify(context.Background(), bytes.NewReader(data), int64(len(testSQL))) + if err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") { + t.Fatalf("expected a sanitized rejection, got %v", err) + } + // PAX header size errors should be corruption, not size limit. + if tc.name == "hidden oversized metadata" && errors.Is(err, ErrTooLarge) { + t.Fatalf("oversized header reported as size limit: %v", err) + } + }) + } +} + +func TestVerifyFraming(t *testing.T) { + raw := verifyTar(t, verifyFixture(t)) + valid := verifyGzip(t, raw) + badCRC := bytes.Clone(valid) + badCRC[len(badCRC)-8] ^= 1 + cases := map[string][]byte{ + "not gzip": []byte("PRIVATE_CANARY"), "truncated gzip": valid[:len(valid)-1], "bad checksum": badCRC, + "no tar terminator": verifyGzip(t, raw[:len(raw)-1024]), "one tar terminator": verifyGzip(t, raw[:len(raw)-512]), + "truncated payload": verifyGzip(t, raw[:1500]), "trailing tar bytes": verifyGzip(t, append(bytes.Clone(raw), 0)), + "trailing compressed bytes": append(bytes.Clone(valid), 0), "second gzip stream": append(bytes.Clone(valid), verifyGzip(t, nil)...), + } + for name, data := range cases { + t.Run(name, func(t *testing.T) { + _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes) + if err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") { + t.Fatalf("expected sanitized framing error, got %v", err) + } + if errors.Is(err, ErrTooLarge) { + t.Fatalf("framing corruption reported as size limit: %v", err) + } + }) + } +} + +func TestVerifySizeOnlyPAX(t *testing.T) { + // A native dump over 8 GiB requires a size-only PAX header. Generate the + // real header without allocating its body; verification must reach the + // truncated member, rather than reject the exporter's metadata format. + var raw bytes.Buffer + tw := tar.NewWriter(&raw) + if err := tw.WriteHeader(&tar.Header{Name: "database.sql", Mode: 0600, Size: 1 << 34}); err != nil { + t.Fatal(err) + } + h, err := tar.NewReader(bytes.NewReader(raw.Bytes())).Next() + if err != nil || h.Format != tar.FormatPAX || len(h.PAXRecords) != 1 || h.PAXRecords["size"] != "17179869184" { + t.Fatalf("fixture must contain native size-only PAX metadata: %v %v", h, err) + } + data := verifyGzip(t, raw.Bytes()) + if _, err := Verify(context.Background(), bytes.NewReader(data), 1<<35); !errors.Is(err, ErrInvalidArchive) || !strings.Contains(err.Error(), "member data") { + t.Fatalf("native PAX header was rejected before reading the payload: %v", err) + } + if _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes); !errors.Is(err, ErrTooLarge) { + t.Fatalf("PAX size did not respect the SQL limit: %v", err) + } +} + +func TestVerifyBoundsAndCancellation(t *testing.T) { + for _, limit := range []int64{0, -1, 1<<40 + 1} { + if _, err := Verify(context.Background(), bytes.NewReader(nil), limit); err == nil { + t.Fatal("invalid limit accepted") + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := Verify(ctx, bytes.NewReader(nil), 1); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + for _, size := range []int{2, 3} { + r := &verifyReader{ctx: context.Background(), r: bytes.NewReader(make([]byte, size)), limit: 2} + _, err := io.Copy(io.Discard, r) + if (size == 3 && !errors.Is(err, ErrTooLarge)) || (size == 2 && err != nil) { + t.Fatalf("read limit boundary: %d %v", size, err) + } + } +} + +func FuzzVerify(f *testing.F) { + f.Add([]byte("invalid gzip")) + f.Add(verifyGzip(f, verifyTar(f, verifyFixture(f)))) + f.Add(verifyTar(f, verifyFixture(f))) + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 2<<20 { + t.Skip() + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, _ = Verify(ctx, bytes.NewReader(data), 4096) + // Fuzz tar/manifest parsing too, without every mutation first failing + // the outer gzip checksum. + _, _ = Verify(ctx, bytes.NewReader(verifyGzip(t, data)), 4096) + }) +} diff --git a/internal/borders/borders.go b/internal/borders/borders.go new file mode 100644 index 00000000..ad2c560d --- /dev/null +++ b/internal/borders/borders.go @@ -0,0 +1,90 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package borders classifies advertised repeater positions against a site's +// immutable, configured operating area. It does not change heard-in IATAs. +package borders + +import ( + "fmt" + "math" + + "github.com/meshcore-go/meshcore-go" + "github.com/paulmach/orb" + "github.com/paulmach/orb/planar" +) + +type polygon struct { + rings orb.Polygon + bound orb.Bound +} + +// Local owns the configured polygon union and precomputed bounds. +// A nil Local disables classification. Methods do not mutate shared state. +type Local struct{ polygons []polygon } + +// New copies the polygons so the classifier can be shared across readers and +// ingest workers. Antimeridian regions must use GeoJSON's split representation. +func New(polygons []orb.Polygon) (*Local, error) { + if len(polygons) == 0 { + return nil, fmt.Errorf("no local border polygons configured") + } + local := &Local{} + for _, poly := range polygons { + if len(poly) == 0 { + return nil, fmt.Errorf("border polygon has no rings") + } + for _, ring := range poly { + if !ring.Closed() { + return nil, fmt.Errorf("border ring must be closed with at least four points") + } + for i, p := range ring { + if !validPosition(p.Lat(), p.Lon()) { + return nil, fmt.Errorf("border coordinates must be finite longitude/latitude in range") + } + if i > 0 && math.Abs(p.Lon()-ring[i-1].Lon()) > 180 { + return nil, fmt.Errorf("split antimeridian-crossing borders into a MultiPolygon") + } + } + } + if planar.Area(poly) <= 0 { + return nil, fmt.Errorf("border polygon must have positive area") + } + copy := poly.Clone() + local.polygons = append(local.polygons, polygon{copy, copy.Bound()}) + } + return local, nil +} + +func validPosition(lat, lng float64) bool { + return !math.IsNaN(lat) && !math.IsNaN(lng) && lat >= -90 && lat <= 90 && lng >= -180 && lng <= 180 +} + +// PossiblyForeign is nil when disabled, the role is not repeater, or position +// is unknown/invalid. +// All polygon boundaries, including hole edges, count as local; hole interiors +// remain outside. Location is self-reported, so this is an indication only. +func (local *Local) PossiblyForeign(role int16, lat, lng *float64) *bool { + if local == nil || role != int16(meshcore.AdvertTypeRepeater) || lat == nil || lng == nil || !validPosition(*lat, *lng) { + return nil + } + point := orb.Point{*lng, *lat} + foreign := true + for _, poly := range local.polygons { + if !poly.bound.Contains(point) || !planar.RingContains(poly.rings[0], point) { + continue + } + inside := true + for _, hole := range poly.rings[1:] { + if planar.RingContains(hole, point) && planar.DistanceFrom(orb.LineString(hole), point) != 0 { + inside = false + break + } + } + if inside { + foreign = false + break + } + } + return &foreign +} diff --git a/internal/borders/borders_test.go b/internal/borders/borders_test.go new file mode 100644 index 00000000..687f84de --- /dev/null +++ b/internal/borders/borders_test.go @@ -0,0 +1,141 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package borders + +import ( + "math" + "testing" + + "github.com/paulmach/orb" +) + +func TestPossiblyForeign(t *testing.T) { + outer := orb.Ring{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}} + hole := orb.Ring{{12, 12}, {14, 12}, {14, 14}, {12, 14}, {12, 12}} + other := orb.Ring{{30, 10}, {40, 10}, {40, 20}, {30, 20}, {30, 10}} + local, err := New([]orb.Polygon{{outer, hole}, {other}}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + role int16 + lat, lng float64 + want *bool + }{ + {"inside", 2, 15, 15, new(false)}, + {"outside", 2, 25, 25, new(true)}, + {"second polygon", 2, 15, 35, new(false)}, + {"hole", 2, 13, 13, new(true)}, + {"exterior edge", 2, 15, 10, new(false)}, + {"hole edge", 2, 13, 12, new(false)}, + {"vertex", 2, 10, 10, new(false)}, + {"companion", 1, 25, 25, nil}, + {"room", 3, 25, 25, nil}, + {"sensor", 4, 25, 25, nil}, + {"null island is a position", 2, 0, 0, new(true)}, + {"equator is valid", 2, 0, 25, new(true)}, + {"prime meridian is valid", 2, 25, 0, new(true)}, + {"invalid latitude", 2, 91, 25, nil}, + {"invalid longitude", 2, 25, -181, nil}, + {"NaN", 2, math.NaN(), 25, nil}, + {"infinite", 2, 25, math.Inf(1), nil}, + } { + t.Run(tc.name, func(t *testing.T) { + got := local.PossiblyForeign(tc.role, &tc.lat, &tc.lng) + if (got == nil) != (tc.want == nil) || got != nil && *got != *tc.want { + t.Fatalf("got %v, want %v", got, tc.want) + } + }) + } + if local.PossiblyForeign(2, nil, new(15.0)) != nil || local.PossiblyForeign(2, new(15.0), nil) != nil { + t.Fatal("missing coordinate classified") + } + var disabled *Local + if disabled.PossiblyForeign(2, new(25.0), new(25.0)) != nil { + t.Fatal("disabled classification") + } + // The shared startup classifier owns its geometry; callers cannot mutate it. + outer[0] = orb.Point{99, 99} + if *local.PossiblyForeign(2, new(15.0), new(15.0)) { + t.Fatal("input mutation changed classifier") + } +} + +func TestLocalBordersValidation(t *testing.T) { + for _, polys := range [][]orb.Polygon{ + nil, {{}}, {{{{0, 0}, {1, 0}, {1, 1}}}}, + {{{{0, 0}, {0, 0}, {0, 0}, {0, 0}}}}, + {{{{179, 10}, {-179, 10}, {-179, 20}, {179, 20}, {179, 10}}}}, + } { + if _, err := New(polys); err == nil { + t.Fatalf("accepted invalid/uncut geometry: %v", polys) + } + } + cut := []orb.Polygon{ + {{{179, 10}, {180, 10}, {180, 20}, {179, 20}, {179, 10}}}, + {{{-180, 10}, {-179, 10}, {-179, 20}, {-180, 20}, {-180, 10}}}, + } + local, err := New(cut) + if err != nil { + t.Fatal(err) + } + for _, lon := range []float64{179.5, -179.5, 180, -180} { + if *local.PossiblyForeign(2, new(15.0), &lon) { + t.Fatalf("dateline island excluded: %v", lon) + } + } + if !*local.PossiblyForeign(2, new(15.0), new(0.0)) { + t.Fatal("uncut global region inferred") + } +} + +func BenchmarkPossiblyForeign(b *testing.B) { + ring := make(orb.Ring, 10001) + for i := 0; i < 10000; i++ { + a := 2 * math.Pi * float64(i) / 10000 + ring[i] = orb.Point{20 + math.Cos(a), 50 + math.Sin(a)} + } + ring[10000] = ring[0] + local, err := New([]orb.Polygon{{ring}}) + if err != nil { + b.Fatal(err) + } + for _, tc := range []struct { + name string + lat, lng float64 + }{{"inside_10k_vertices", 50, 20}, {"outside_bounds_10k_vertices", 0, 1}} { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + local.PossiblyForeign(2, &tc.lat, &tc.lng) + } + }) + } +} + +func TestLiveSwap(t *testing.T) { + var disabled *Live + if disabled.PossiblyForeign(2, new(15.0), new(15.0)) != nil { + t.Fatal("nil live classified") + } + live := NewLive(nil) + if disabled.Ready() || live.Ready() { + t.Fatal("ready without boundaries") + } + if live.PossiblyForeign(2, new(15.0), new(15.0)) != nil { + t.Fatal("empty live classified") + } + local, err := New([]orb.Polygon{{{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}}}}) + if err != nil { + t.Fatal(err) + } + live.Store(local) + if !live.Ready() { + t.Fatal("not ready after store") + } + if got := live.PossiblyForeign(2, new(15.0), new(15.0)); got == nil || *got { + t.Fatal("stored classifier not used", got) + } +} diff --git a/internal/borders/live.go b/internal/borders/live.go new file mode 100644 index 00000000..ae8294ed --- /dev/null +++ b/internal/borders/live.go @@ -0,0 +1,28 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package borders + +import "sync/atomic" + +// Live lets boundary refreshes replace the classifier under concurrent readers. +// A nil Live disables classification; an empty one reports unknown until stored. +type Live struct{ current atomic.Pointer[Local] } + +func NewLive(local *Local) *Live { + live := &Live{} + live.current.Store(local) + return live +} + +func (l *Live) Store(local *Local) { l.current.Store(local) } + +// Ready reports whether any boundary is loaded. +func (l *Live) Ready() bool { return l != nil && l.current.Load() != nil } + +func (l *Live) PossiblyForeign(role int16, lat, lng *float64) *bool { + if l == nil { + return nil + } + return l.current.Load().PossiblyForeign(role, lat, lng) +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go index abee2760..90cf83f5 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -11,15 +11,18 @@ import ( "context" "encoding/json" "errors" + "log/slog" "time" "github.com/MeshCore-Beacon/beacon-server/internal/config" redis "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" ) // Client wraps a Redis client with helper methods used by CachedReader. type Client struct { rdb *redis.Client + sf singleflight.Group // one fetch per key across concurrent misses } // NewClient creates a new Redis client from the given address, password, and @@ -75,6 +78,24 @@ func (c *Client) del(ctx context.Context, keys ...string) { c.rdb.Del(ctx, keys...) } +// delPrefix SCANs the whole keyspace; call it rarely. +func (c *Client) delPrefix(ctx context.Context, prefix string) { + var cursor uint64 + for { + keys, next, err := c.rdb.Scan(ctx, cursor, prefix+"*", 100).Result() + if err != nil { + return + } + if len(keys) > 0 { + c.rdb.Del(ctx, keys...) + } + if next == 0 { + return + } + cursor = next + } +} + // getOrSet retrieves a cached value by key, deserialising it into T. // On a cache miss it calls fetch, stores the result under key with the given // TTL, and returns it. If Redis is unavailable or returns an unexpected error, @@ -82,38 +103,62 @@ func (c *Client) del(ctx context.Context, keys ...string) { // fails to unmarshal (corrupt or schema-changed), the entry is overwritten // with a fresh fetch. Errors from Set are ignored so a Redis hiccup never // fails a request. -func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Duration, fetch func() (T, error)) (T, error) { +func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Duration, fetch func(context.Context) (T, error)) (T, error) { raw, err := c.rdb.Get(ctx, key).Bytes() if err != nil && !errors.Is(err, redis.Nil) { // real Redis error, degrade gracefully - return fetch() + slog.DebugContext(ctx, "cache bypass", "component", "cache", "reason", "read_error") + return fetch(ctx) } var zero, out T - if errors.Is(err, redis.Nil) { - // cache miss — fetch, store, return - val, err := fetch() - if err != nil { - return zero, err - } - data, jsonErr := json.Marshal(val) - if jsonErr != nil { - return val, nil + if err == nil { + if err = json.Unmarshal(raw, &out); err == nil { + slog.DebugContext(ctx, "cache hit", "component", "cache") + return out, nil } - _ = c.rdb.Set(ctx, key, data, ttl) - return val, nil + slog.DebugContext(ctx, "cache invalid entry", "component", "cache") + } else { + slog.DebugContext(ctx, "cache miss", "component", "cache") } - if err = json.Unmarshal(raw, &out); err != nil { - // corrupt cache entry — overwrite it - val, err := fetch() + raw, err = fetchShared(ctx, c, key, ttl, func(ctx context.Context) (any, error) { return fetch(ctx) }) + if err != nil { + return zero, err + } + // Each caller decodes its own copy: handlers may decorate the value they get. + if err := json.Unmarshal(raw, &out); err != nil { + return zero, err + } + return out, nil +} + +// sharedFetchTimeout bounds a fetch that no single request owns. +const sharedFetchTimeout = 30 * time.Second + +// fetchShared runs one fetch per key across concurrent misses and returns its JSON. The fetch +// gets its own bounded context so one caller disconnecting can't fail the others, and each +// caller still stops waiting when its own context ends. +func fetchShared(ctx context.Context, c *Client, key string, ttl time.Duration, fetch func(context.Context) (any, error)) ([]byte, error) { + ch := c.sf.DoChan(key, func() (any, error) { + fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), sharedFetchTimeout) + defer cancel() + val, err := fetch(fctx) + if err != nil { + return nil, err + } + data, err := json.Marshal(val) if err != nil { - return zero, err + return nil, err } - data, jsonErr := json.Marshal(val) - if jsonErr != nil { - return val, nil + _ = c.rdb.Set(fctx, key, data, ttl) + return data, nil + }) + select { + case r := <-ch: + if r.Err != nil { + return nil, r.Err } - _ = c.rdb.Set(ctx, key, data, ttl) - return val, nil + return r.Val.([]byte), nil + case <-ctx.Done(): + return nil, ctx.Err() } - return out, nil } diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 42463c94..b5dcc7f5 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -31,12 +31,41 @@ type stubReader struct { calls int } +func (s *stubReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } +func (s *stubReader) GetStatsSeries(context.Context, time.Time, time.Time, []string) (*api.StatsSeries, error) { + return &api.StatsSeries{}, nil +} +func (s *stubReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { + return nil, s.err +} + +func (s *stubReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { + return nil, s.err +} + func (s *stubReader) ListIATAs(_ context.Context) ([]api.IATA, error) { s.calls++ return s.iatas, s.err } // implement remaining api.Reader methods as no-ops +func (s *stubReader) GetObserverComparison(_ context.Context, _, _ uuid.UUID, _, _ time.Time, _ []string) (*api.ObserverComparison, error) { + s.calls++ + return &api.ObserverComparison{TotalPackets: int64(s.calls)}, s.err +} + +func TestObserverComparisonPassThrough(t *testing.T) { + inner := &stubReader{} + reader := &CachedReader{inner: inner} + a, b, since, until := uuid.New(), uuid.New(), time.Now().Add(-time.Hour), time.Now() + for i := int64(1); i <= 2; i++ { + got, err := reader.GetObserverComparison(context.Background(), a, b, since, until, nil) + if err != nil || got.TotalPackets != i { + t.Fatalf("comparison unexpectedly cached: %+v, %v", got, err) + } + } +} + func (s *stubReader) GetIATA(_ context.Context, _ string) (*api.IATA, error) { return nil, nil } func (s *stubReader) GetIATABorder(_ context.Context, _ string) (json.RawMessage, error) { @@ -47,12 +76,10 @@ func (s *stubReader) GetRegion(_ context.Context, _ int32) (*api.Region, error) func (s *stubReader) GetRegionBySlug(_ context.Context, _ string) (*api.Region, error) { return nil, nil } -func (s *stubReader) GetScopeNames(_ context.Context) ([]string, error) { return nil, nil } -func (s *stubReader) GetScopeStats(_ context.Context) ([]api.ScopeStats, error) { return nil, nil } -func (s *stubReader) GetScopesByIATAs(_ context.Context, _ []string) ([]api.ScopeSummary, error) { +func (s *stubReader) GetScopeNames(_ context.Context) ([]string, error) { return nil, nil } +func (s *stubReader) GetScopeStats(_ context.Context, _ []string, _ time.Time) ([]api.ScopeStats, error) { return nil, nil } - func (s *stubReader) GetScopeByName(_ context.Context, _ string) (*api.ScopeDetail, error) { return nil, nil } @@ -69,7 +96,7 @@ func (s *stubReader) GetStatsPayloadBreakdown(_ context.Context, _ []string, _ t return nil, nil } -func (s *stubReader) GetStatsTopNodes(_ context.Context, _ []string, _ int32) ([]api.TopNode, error) { +func (s *stubReader) GetStatsTopNodes(_ context.Context, _ []string, _ time.Time, _ int32) ([]api.TopNode, error) { return nil, nil } @@ -81,6 +108,10 @@ func (s *stubReader) GetStatsTopAdvertisers(_ context.Context, _ []string, _ tim return nil, nil } +func (s *stubReader) GetStatsClockDrift(_ context.Context, _ []string, _ int32) ([]api.ClockDriftEntry, error) { + return nil, nil +} + func (s *stubReader) GetStatsTopTalkers(_ context.Context, _ []string, _ time.Time, _ int32) ([]api.TopTalker, error) { return nil, nil } @@ -117,6 +148,10 @@ func (s *stubReader) GetObserverTelemetryBucketed(_ context.Context, _ uuid.UUID return nil, nil } +func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { + return nil, nil +} + func (s *stubReader) GetPacket(_ context.Context, _ []byte) (*api.Packet, error) { return nil, nil } func (s *stubReader) GetChannel(_ context.Context, _ int32) (*api.Channel, error) { return nil, nil } @@ -133,8 +168,8 @@ func (s *stubReader) GetCrossIATANeighbors(_ context.Context, _ uuid.UUID, _ str return nil, nil } -func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ int64) (api.Page[api.ChannelSummary], error) { - return api.Page[api.ChannelSummary]{}, nil +func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + return api.ChannelPage{}, nil } func (s *stubReader) ListChannelMessages(_ context.Context, _ *int32, _ time.Time, _ int32, _ []string, _ string, _ int64) (api.Page[api.ChannelMessage], error) { @@ -173,7 +208,7 @@ func (s *stubReader) ListPacketsAfterID(_ context.Context, _ int64, _, _ int16, return nil, nil } -func (s *stubReader) ListKnownRoutes(_ context.Context, _ string, _ int32, _ time.Time, _ int32) ([]api.KnownRoute, error) { +func (s *stubReader) ListKnownRoutes(_ context.Context, _ string, _ int32, _ time.Time, _ int64, _ int32) ([]api.KnownRoute, error) { return nil, nil } @@ -185,7 +220,7 @@ func (s *stubReader) SearchCrossIATARoutes(_ context.Context, _, _, _, _ string) return nil, nil } -func (s *stubReader) ListTraceTags(_ context.Context, _ []string, _, _ string, _, _ time.Time, _ time.Time, _ int32) ([]api.TraceTagSummary, error) { +func (s *stubReader) ListTraceTags(_ context.Context, _ []string, _, _ string, _, _ time.Time, _ time.Time, _ string, _ int32) ([]api.TraceTagSummary, error) { return nil, nil } @@ -227,7 +262,7 @@ func TestGetOrSet_CacheMiss_FetchesAndStores(t *testing.T) { c, _ := newTestClient(t) stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -246,11 +281,11 @@ func TestGetOrSet_CacheHit_DoesNotFetch(t *testing.T) { stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} // prime the cache - _, _ = getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + _, _ = getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) // second call should hit cache - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -269,7 +304,7 @@ func TestGetOrSet_RedisError_DegradeGracefully(t *testing.T) { mr.Close() // kill Redis stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -285,7 +320,7 @@ func TestGetOrSet_CorruptEntry_Overwrites(t *testing.T) { mr.Set("test:key", "not-valid-json") stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -303,7 +338,7 @@ func TestGetOrSet_FetchError_Propagates(t *testing.T) { c, _ := newTestClient(t) stub := &stubReader{err: errors.New("db error")} - _, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + _, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err == nil { @@ -326,14 +361,14 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { } // call with unsorted IATAs - getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func() (*api.StatsOverview, error) { + getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func(ctx context.Context) (*api.StatsOverview, error) { calls++ return &api.StatsOverview{TotalPackets: 42}, nil }) // call CachedReader with reversed order — should hit same key _ = cr - result, err := getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func() (*api.StatsOverview, error) { + result, err := getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func(ctx context.Context) (*api.StatsOverview, error) { calls++ return &api.StatsOverview{TotalPackets: 42}, nil }) @@ -348,6 +383,30 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { } } +func TestInvalidateScopeNamesClearsDerivedKeys(t *testing.T) { + c, mr := newTestClient(t) + cr := &CachedReader{inner: &stubReader{}, c: c, ttl: CacheTTLs{}} + for _, k := range []string{keyScopeNames, keyScopeByNamePrefix + "#yow"} { + mr.Set(k, "x") + } + mr.Set("beacon:iatas", "keep") + cr.InvalidateScopeNames(context.Background()) + if got := mr.Keys(); len(got) != 1 || got[0] != "beacon:iatas" { + t.Fatalf("keys left: %v", got) + } +} + +func TestInvalidateIATABorder(t *testing.T) { + c, mr := newTestClient(t) + cr := &CachedReader{inner: &stubReader{}, c: c, ttl: CacheTTLs{}} + mr.Set(keyIATABorderPrefix+"YOW", "x") + mr.Set(keyIATABorderPrefix+"YYZ", "keep") + cr.InvalidateIATABorder(context.Background(), "YOW") + if got := mr.Keys(); len(got) != 1 || got[0] != keyIATABorderPrefix+"YYZ" { + t.Fatalf("keys left: %v", got) + } +} + func TestCachedReader_InvalidateNode(t *testing.T) { c, mr := newTestClient(t) nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000001") @@ -383,3 +442,7 @@ func TestCachedReader_InvalidateObserver(t *testing.T) { t.Error("expected observer scopes key to be deleted") } } + +func (s *stubReader) GetRouteEvidence(_ context.Context, _, _ string, _ api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + return nil, nil +} diff --git a/internal/cache/logging_test.go b/internal/cache/logging_test.go new file mode 100644 index 00000000..94a45c68 --- /dev/null +++ b/internal/cache/logging_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "bytes" + "context" + "encoding/json" + "log" + "log/slog" + "strings" + "testing" + "time" +) + +func TestCacheDiagnostics(t *testing.T) { + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) + var out bytes.Buffer + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, &slog.HandlerOptions{Level: slog.LevelDebug}))) + c, mr := newTestClient(t) + key, value := "private-cache-key", "private-cache-value" + calls := 0 + fetch := func(context.Context) (string, error) { calls++; return value, nil } + for i := 0; i < 2; i++ { + got, err := getOrSet(context.Background(), c, key, time.Minute, fetch) + if err != nil || got != value { + t.Fatalf("cache result changed: %v", err) + } + } + if calls != 1 { + t.Fatalf("fetches=%d, want 1", calls) + } + mr.Set(key, "invalid JSON") + if _, err := getOrSet(context.Background(), c, key, time.Minute, fetch); err != nil { + t.Fatal(err) + } + lines := bytes.Split(bytes.TrimSpace(out.Bytes()), []byte("\n")) + want := []string{"cache miss", "cache hit", "cache invalid entry"} + if len(lines) != len(want) { + t.Fatalf("got %d records, want %d", len(lines), len(want)) + } + for i, line := range lines { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal(err) + } + if record["component"] != "cache" || record["level"] != "DEBUG" || record["msg"] != want[i] { + t.Fatalf("unexpected record: %v", record) + } + } + if strings.Contains(out.String(), key) || strings.Contains(out.String(), value) { + t.Fatal("cache key or payload leaked") + } + out.Reset() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, nil))) + if _, err := getOrSet(context.Background(), c, key, time.Minute, fetch); err != nil { + t.Fatal(err) + } + if out.Len() != 0 { + t.Fatal("debug diagnostics appeared at info level") + } +} diff --git a/internal/cache/observer_window_test.go b/internal/cache/observer_window_test.go new file mode 100644 index 00000000..512e5800 --- /dev/null +++ b/internal/cache/observer_window_test.go @@ -0,0 +1,47 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" +) + +type activityWindowReader struct { + stubReader + ends []time.Time +} + +func (r *activityWindowReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, until time.Time) (*api.ObserverActivity, error) { + r.ends = append(r.ends, until) + return &api.ObserverActivity{WindowEnd: until.UnixMilli()}, nil +} + +func TestObserverActivityCacheUsesEffectiveWindow(t *testing.T) { + client, redis := newTestClient(t) + inner := &activityWindowReader{} + reader := &CachedReader{inner: inner, c: client} + id := uuid.New() + anchor := time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC) + for _, offset := range []time.Duration{time.Second, time.Minute, 59 * time.Minute} { + if _, err := reader.GetObserverActivity(context.Background(), id, 24*time.Hour, time.Hour, anchor.Add(offset)); err != nil { + t.Fatal(err) + } + } + if len(inner.ends) != 1 || !inner.ends[0].Equal(anchor) || len(redis.Keys()) != 1 { + t.Fatalf("equivalent fixed windows missed the cache: %v, %v", inner.ends, redis.Keys()) + } + for _, until := range []time.Time{anchor.Add(time.Hour), {}, {}} { + if _, err := reader.GetObserverActivity(context.Background(), id, 24*time.Hour, time.Hour, until); err != nil { + t.Fatal(err) + } + } + if len(inner.ends) != 3 || !inner.ends[2].IsZero() || len(redis.Keys()) != 3 { + t.Fatalf("distinct/live windows aliased: %v", inner.ends) + } +} diff --git a/internal/cache/paths_test.go b/internal/cache/paths_test.go new file mode 100644 index 00000000..2c55bf5c --- /dev/null +++ b/internal/cache/paths_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type pathStatsReader struct { + api.Reader + calls int64 +} + +func (r *pathStatsReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } + +func (r *pathStatsReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { + r.calls++ + return &api.PathStats{Receptions: r.calls}, nil +} + +func TestPathCacheSeparatesWindowAndRegion(t *testing.T) { + c, mr := newTestClient(t) + inner := &pathStatsReader{} + r := NewCachedReader(inner, c, CacheTTLs{Stats: time.Minute}) + for _, tc := range []struct { + iatas []string + since, until, want int64 + }{ + {nil, 0, 10, 1}, {[]string{}, 0, 10, 1}, {[]string{""}, 0, 10, 2}, {[]string{"YVR"}, 0, 10, 3}, + {[]string{"YVR", "YYJ"}, 0, 10, 4}, {[]string{"YYJ", "YVR", "YYJ"}, 0, 10, 4}, + {nil, 1, 10, 5}, {nil, 0, 11, 6}, {nil, 0, 10, 1}, + } { + before := slices.Clone(tc.iatas) + got, err := r.GetPathStats(context.Background(), time.UnixMilli(tc.since), time.UnixMilli(tc.until), tc.iatas) + if err != nil || got.Receptions != tc.want || !slices.Equal(before, tc.iatas) { + t.Fatalf("%+v: %+v %v", tc, got, err) + } + } + mr.FastForward(time.Minute) + got, err := r.GetPathStats(context.Background(), time.UnixMilli(0), time.UnixMilli(10), nil) + if err != nil || got.Receptions != 7 { + t.Fatalf("expiry: %+v %v", got, err) + } +} diff --git a/internal/cache/reader.go b/internal/cache/reader.go index afd45888..e491152d 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -24,14 +24,17 @@ const ( keyRegionSlugPrefix = "beacon:region:slug:" keyScopeNames = "beacon:scope:names" keyScopeStats = "beacon:scope:stats" - keyScopesByIATAsPrefix = "beacon:scopes:iatas:" keyScopeByNamePrefix = "beacon:scope:name:" keyStatsOverviewPrefix = "beacon:stats:overview:" keyStatsObservationsPrefix = "beacon:stats:observations:" + keySignalStatsPrefix = "beacon:stats:signal:" + keyStatsSeriesPrefix = "beacon:stats:series:" + keyPathStatsPrefix = "beacon:stats:paths:" keyStatsBreakdownPrefix = "beacon:stats:breakdown:" keyStatsTopNodesPrefix = "beacon:stats:top-nodes:" keyStatsTopObsPrefix = "beacon:stats:top-observers:" keyStatsTopAdvPrefix = "beacon:stats:top-advertisers:" + keyStatsClockDriftPrefix = "beacon:stats:clock-drift:" keyStatsTopTalkersPrefix = "beacon:stats:top-talkers:" keyStatsNodeTypes = "beacon:stats:node-types:" keyRadioPresetsPrefix = "beacon:radio-presets:" @@ -40,8 +43,12 @@ const ( keyNodesByIDsPrefix = "beacon:nodes:ids:" keyObserverPrefix = "beacon:observer:" keyObserverScopesPrefix = "beacon:observer:scopes:" + keyObserverActivityPrefix = "beacon:observer:activity:" ) +// observerActivityTTL is deliberately short: activity is a live chart, not reference data. +const observerActivityTTL = 60 * time.Second + // CachedReader wraps an api.Reader with a Redis caching layer. // It implements api.Reader and is a drop-in replacement for db.Store // at the wiring point in main.go. @@ -49,6 +56,8 @@ type CachedReader struct { inner api.Reader c *Client ttl CacheTTLs + rev revisionMemo + now func() time.Time } // CacheTTLs holds the resolved per-category TTLs for the cache layer. @@ -69,9 +78,34 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { inner: inner, c: c, ttl: ttl, + now: time.Now, } } +// InvalidateScopeNames makes newly committed catalogue names visible to every scope read. +func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { + cr.c.del(ctx, keyScopeNames) + cr.c.delPrefix(ctx, keyScopeByNamePrefix) + cr.c.delPrefix(ctx, keyScopeStats) +} + +// InvalidateIATABorder makes an imported or removed MeshMapper boundary visible. +func (cr *CachedReader) InvalidateIATABorder(ctx context.Context, iata string) { + cr.c.del(ctx, keyIATABorderPrefix+iata) +} + +// InvalidateIATAs makes imported MeshMapper names and locations visible to every IATA read. +func (cr *CachedReader) InvalidateIATAs(ctx context.Context) { + cr.c.del(ctx, keyIATAs) + cr.c.delPrefix(ctx, keyIATAPrefix) +} + +// InvalidateRegions makes imported MeshMapper regions visible to every region read. +func (cr *CachedReader) InvalidateRegions(ctx context.Context) { + cr.c.del(ctx, keyRegions) + cr.c.delPrefix(ctx, keyRegionPrefix) +} + // InvalidateNode removes the cached entries for a node by UUID. // Should be called from the ingest path after a node upsert. func (cr *CachedReader) InvalidateNode(ctx context.Context, nodeID uuid.UUID) { @@ -88,132 +122,57 @@ func (cr *CachedReader) InvalidateObserver(ctx context.Context, observerID uuid. // ListIATAs implements [api.Reader]. func (cr *CachedReader) ListIATAs(ctx context.Context) ([]api.IATA, error) { - return getOrSet(ctx, cr.c, keyIATAs, cr.ttl.Reference, func() ([]api.IATA, error) { + return getOrSet(ctx, cr.c, keyIATAs, cr.ttl.Reference, func(ctx context.Context) ([]api.IATA, error) { return cr.inner.ListIATAs(ctx) }) } // GetIATA implements [api.Reader]. func (cr *CachedReader) GetIATA(ctx context.Context, iata string) (*api.IATA, error) { - return getOrSet(ctx, cr.c, keyIATAPrefix+iata, cr.ttl.Reference, func() (*api.IATA, error) { + return getOrSet(ctx, cr.c, keyIATAPrefix+iata, cr.ttl.Reference, func(ctx context.Context) (*api.IATA, error) { return cr.inner.GetIATA(ctx, iata) }) } // GetIATABorder implements [api.Reader]. func (cr *CachedReader) GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) { - return getOrSet(ctx, cr.c, keyIATABorderPrefix+iata, cr.ttl.Reference, func() (json.RawMessage, error) { + return getOrSet(ctx, cr.c, keyIATABorderPrefix+iata, cr.ttl.Reference, func(ctx context.Context) (json.RawMessage, error) { return cr.inner.GetIATABorder(ctx, iata) }) } // ListRegions implements [api.Reader]. func (cr *CachedReader) ListRegions(ctx context.Context) ([]api.RegionSummary, error) { - return getOrSet(ctx, cr.c, keyRegions, cr.ttl.Reference, func() ([]api.RegionSummary, error) { + return getOrSet(ctx, cr.c, keyRegions, cr.ttl.Reference, func(ctx context.Context) ([]api.RegionSummary, error) { return cr.inner.ListRegions(ctx) }) } // GetRegion implements [api.Reader]. func (cr *CachedReader) GetRegion(ctx context.Context, regionID int32) (*api.Region, error) { - return getOrSet(ctx, cr.c, fmt.Sprintf("%s%d", keyRegionPrefix, regionID), cr.ttl.Reference, func() (*api.Region, error) { + return getOrSet(ctx, cr.c, fmt.Sprintf("%s%d", keyRegionPrefix, regionID), cr.ttl.Reference, func(ctx context.Context) (*api.Region, error) { return cr.inner.GetRegion(ctx, regionID) }) } // GetRegionBySlug implements [api.Reader]. func (cr *CachedReader) GetRegionBySlug(ctx context.Context, slug string) (*api.Region, error) { - return getOrSet(ctx, cr.c, keyRegionSlugPrefix+slug, cr.ttl.Reference, func() (*api.Region, error) { + return getOrSet(ctx, cr.c, keyRegionSlugPrefix+slug, cr.ttl.Reference, func(ctx context.Context) (*api.Region, error) { return cr.inner.GetRegionBySlug(ctx, slug) }) } // GetScopeNames implements [api.Reader]. func (cr *CachedReader) GetScopeNames(ctx context.Context) ([]string, error) { - return getOrSet(ctx, cr.c, keyScopeNames, cr.ttl.Reference, func() ([]string, error) { + return getOrSet(ctx, cr.c, keyScopeNames, cr.ttl.Reference, func(ctx context.Context) ([]string, error) { return cr.inner.GetScopeNames(ctx) }) } -// GetScopeStats implements [api.Reader]. -func (cr *CachedReader) GetScopeStats(ctx context.Context) ([]api.ScopeStats, error) { - return getOrSet(ctx, cr.c, keyScopeStats, cr.ttl.Reference, func() ([]api.ScopeStats, error) { - return cr.inner.GetScopeStats(ctx) - }) -} - -// GetScopesByIATAs implements [api.Reader]. -func (cr *CachedReader) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) { - sorted := make([]string, len(iatas)) - copy(sorted, iatas) - sort.Strings(sorted) - key := keyScopesByIATAsPrefix + strings.Join(sorted, ",") - return getOrSet(ctx, cr.c, key, cr.ttl.Reference, func() ([]api.ScopeSummary, error) { - return cr.inner.GetScopesByIATAs(ctx, iatas) - }) -} - -// GetScopeByName implements [api.Reader]. -func (cr *CachedReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { - return getOrSet(ctx, cr.c, keyScopeByNamePrefix+name, cr.ttl.Reference, func() (*api.ScopeDetail, error) { - return cr.inner.GetScopeByName(ctx, name) - }) -} - -// GetStatsOverview implements [api.Reader]. -func (cr *CachedReader) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s", keyStatsOverviewPrefix, segment) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.StatsOverview, error) { - return cr.inner.GetStatsOverview(ctx, iatas) - }) -} - -// GetStatsObservations implements [api.Reader]. -func (cr *CachedReader) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d", keyStatsObservationsPrefix, segment, since.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.ObservationPoint, error) { - return cr.inner.GetStatsObservations(ctx, iatas, since) - }) -} - -// GetStatsPayloadBreakdown implements [api.Reader]. -func (cr *CachedReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d", keyStatsBreakdownPrefix, segment, since.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.PayloadBreakdownItem, error) { - return cr.inner.GetStatsPayloadBreakdown(ctx, iatas, since) - }) -} - -// GetStatsTopNodes implements [api.Reader]. -func (cr *CachedReader) GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d", keyStatsTopNodesPrefix, segment, limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopNode, error) { - return cr.inner.GetStatsTopNodes(ctx, iatas, limit) - }) +// GetObserverComparison passes arbitrary observer/time combinations through; +// these explicit queries should not fill the shared statistics cache. +func (cr *CachedReader) GetObserverComparison(ctx context.Context, a, b uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { + return cr.inner.GetObserverComparison(ctx, a, b, since, until, iatas) } // GetStatsNodeTypes implements [api.Reader]. @@ -225,50 +184,22 @@ func (cr *CachedReader) GetStatsNodeTypes(ctx context.Context, iatas []string) ( segment = strings.Join(sorted, ",") } key := fmt.Sprintf("%s%s", keyStatsNodeTypes, segment) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.NodeTypeCount, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func(ctx context.Context) ([]api.NodeTypeCount, error) { return cr.inner.GetStatsNodeTypes(ctx, iatas) }) } -// GetStatsTopObservers implements [api.Reader]. -func (cr *CachedReader) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d:%d", keyStatsTopObsPrefix, segment, since.UnixMilli(), limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopObserver, error) { - return cr.inner.GetStatsTopObservers(ctx, iatas, since, limit) - }) -} - -// GetStatsTopAdvertisers implements [api.Reader]. -func (cr *CachedReader) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { +// GetStatsClockDrift implements [api.Reader]. +func (cr *CachedReader) GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { segment := "all" if len(iatas) > 0 { sorted := append([]string(nil), iatas...) sort.Strings(sorted) segment = strings.Join(sorted, ",") } - key := fmt.Sprintf("%s%s:%d:%d", keyStatsTopAdvPrefix, segment, since.UnixMilli(), limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopAdvertiser, error) { - return cr.inner.GetStatsTopAdvertisers(ctx, iatas, since, limit) - }) -} - -// GetStatsTopTalkers implements [api.Reader]. -func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d:%d", keyStatsTopTalkersPrefix, segment, since.UnixMilli(), limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopTalker, error) { - return cr.inner.GetStatsTopTalkers(ctx, iatas, since, limit) + key := fmt.Sprintf("%s%s:%d", keyStatsClockDriftPrefix, segment, limit) + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func(ctx context.Context) ([]api.ClockDriftEntry, error) { + return cr.inner.GetStatsClockDrift(ctx, iatas, limit) }) } @@ -281,21 +212,21 @@ func (cr *CachedReader) GetRadioPresets(ctx context.Context, preset string, iata segment = strings.Join(sorted, ",") } key := fmt.Sprintf("%s%s:%s", keyRadioPresetsPrefix, preset, segment) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.RadioPreset, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func(ctx context.Context) ([]api.RadioPreset, error) { return cr.inner.GetRadioPresets(ctx, preset, iatas) }) } // GetNode implements [api.Reader]. func (cr *CachedReader) GetNode(ctx context.Context, nodeID uuid.UUID) (*api.Node, error) { - return getOrSet(ctx, cr.c, keyNodePrefix+nodeID.String(), cr.ttl.Nodes, func() (*api.Node, error) { + return getOrSet(ctx, cr.c, keyNodePrefix+nodeID.String(), cr.ttl.Nodes, func(ctx context.Context) (*api.Node, error) { return cr.inner.GetNode(ctx, nodeID) }) } // GetNodeNeighbors implements [api.Reader]. func (cr *CachedReader) GetNodeNeighbors(ctx context.Context, nodeID uuid.UUID) ([]api.NodeNeighbor, error) { - return getOrSet(ctx, cr.c, keyNodeNeighborsPrefix+nodeID.String(), cr.ttl.Nodes, func() ([]api.NodeNeighbor, error) { + return getOrSet(ctx, cr.c, keyNodeNeighborsPrefix+nodeID.String(), cr.ttl.Nodes, func(ctx context.Context) ([]api.NodeNeighbor, error) { return cr.inner.GetNodeNeighbors(ctx, nodeID) }) } @@ -308,21 +239,21 @@ func (cr *CachedReader) GetNodesByIDs(ctx context.Context, ids []uuid.UUID) (map } sort.Strings(strs) key := keyNodesByIDsPrefix + strings.Join(strs, ",") - return getOrSet(ctx, cr.c, key, cr.ttl.Nodes, func() (map[uuid.UUID]*api.ResolvedNode, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Nodes, func(ctx context.Context) (map[uuid.UUID]*api.ResolvedNode, error) { return cr.inner.GetNodesByIDs(ctx, ids) }) } // GetObserver implements [api.Reader]. func (cr *CachedReader) GetObserver(ctx context.Context, observerID uuid.UUID) (*api.Observer, error) { - return getOrSet(ctx, cr.c, keyObserverPrefix+observerID.String(), cr.ttl.Observers, func() (*api.Observer, error) { + return getOrSet(ctx, cr.c, keyObserverPrefix+observerID.String(), cr.ttl.Observers, func(ctx context.Context) (*api.Observer, error) { return cr.inner.GetObserver(ctx, observerID) }) } // GetObserverScopes implements [api.Reader]. func (cr *CachedReader) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) { - return getOrSet(ctx, cr.c, keyObserverScopesPrefix+observerID.String(), cr.ttl.Observers, func() ([]string, error) { + return getOrSet(ctx, cr.c, keyObserverScopesPrefix+observerID.String(), cr.ttl.Observers, func(ctx context.Context) ([]string, error) { return cr.inner.GetObserverScopes(ctx, observerID) }) } @@ -337,6 +268,17 @@ func (cr *CachedReader) GetObserverTelemetryBucketed(ctx context.Context, observ return cr.inner.GetObserverTelemetryBucketed(ctx, observerID, since, until, bucketHours) } +// GetObserverActivity implements [api.Reader]. +func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { + if !until.IsZero() { + until = until.UTC().Truncate(interval) + } + key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano) + return getOrSet(ctx, cr.c, key, observerActivityTTL, func(ctx context.Context) (*api.ObserverActivity, error) { + return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until) + }) +} + // GetPacket implements [api.Reader]. func (cr *CachedReader) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, error) { return cr.inner.GetPacket(ctx, packetHash) @@ -363,8 +305,8 @@ func (cr *CachedReader) GetCrossIATANeighbors(ctx context.Context, nodeID uuid.U } // ListChannels implements [api.Reader]. -func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) { - return cr.inner.ListChannels(ctx, limit, hash, iatas, cursor) +func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { + return cr.inner.ListChannels(ctx, limit, hash, iatas, keyKnown, cursor, pageCursor) } // ListChannelMessages implements [api.Reader]. @@ -413,8 +355,8 @@ func (cr *CachedReader) ListPacketsAfterID(ctx context.Context, afterObservation } // ListKnownRoutes implements [api.Reader]. -func (cr *CachedReader) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) { - return cr.inner.ListKnownRoutes(ctx, iata, hopCount, cursor, limit) +func (cr *CachedReader) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]api.KnownRoute, error) { + return cr.inner.ListKnownRoutes(ctx, iata, hopCount, cursor, cursorID, limit) } // SearchKnownRoutes implements [api.Reader]. @@ -422,12 +364,17 @@ func (cr *CachedReader) SearchKnownRoutes(ctx context.Context, iata, fromHash, t return cr.inner.SearchKnownRoutes(ctx, iata, fromHash, toHash) } +// Precise cursor/window combinations are intentionally passed through, like route lists. +func (cr *CachedReader) GetRouteEvidence(ctx context.Context, iata, key string, query api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + return cr.inner.GetRouteEvidence(ctx, iata, key, query) +} + // SearchCrossIATARoutes implements [api.Reader]. func (cr *CachedReader) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, toHash, toIATA string) ([]api.CrossIATARoute, error) { return cr.inner.SearchCrossIATARoutes(ctx, fromHash, fromIATA, toHash, toIATA) } // ListTraceTags implements [api.Reader]. -func (cr *CachedReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) { - return cr.inner.ListTraceTags(ctx, iatas, scope, traceType, since, until, cursor, limit) +func (cr *CachedReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { + return cr.inner.ListTraceTags(ctx, iatas, scope, traceType, since, until, cursor, cursorTag, limit) } diff --git a/internal/cache/rollup_stats.go b/internal/cache/rollup_stats.go new file mode 100644 index 00000000..0b271e63 --- /dev/null +++ b/internal/cache/rollup_stats.go @@ -0,0 +1,185 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strconv" + "sync" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// revisionTTL bounds how stale a rollup revision read can be before a new roll shows up. +const revisionTTL = 10 * time.Second + +type revisionSource interface { + AnalyticsRevision(ctx context.Context) (int64, error) +} + +// revisionMemo spares Postgres a revision read on every cached stats request. +type revisionMemo struct { + mu sync.Mutex + rev int64 + at time.Time + now func() time.Time +} + +// get returns the analytics revision; ok is false when the inner reader can't supply one. +func (m *revisionMemo) get(ctx context.Context, inner api.Reader) (rev int64, ok bool) { + src, isSource := inner.(revisionSource) + if !isSource { + return 0, false + } + now := time.Now + if m.now != nil { + now = m.now + } + m.mu.Lock() + defer m.mu.Unlock() + if !m.at.IsZero() && now().Sub(m.at) < revisionTTL { + return m.rev, true + } + rev, err := src.AnalyticsRevision(ctx) + if err != nil { + return 0, false + } + m.rev, m.at = rev, now() + return rev, true +} + +// rollupKey builds a stats key that changes with the rollup revision. [] and [""] stay +// distinct (all IATAs vs an empty region). +func (cr *CachedReader) rollupKey(ctx context.Context, prefix string, iatas []string, parts ...any) (string, bool) { + rev, ok := cr.rev.get(ctx, cr.inner) + if !ok { + return "", false + } + sorted := append([]string{}, iatas...) + slices.Sort(sorted) + segment, _ := json.Marshal(slices.Compact(sorted)) + key := fmt.Sprintf("%sr%d:%s", prefix, rev, segment) + for _, p := range parts { + key += fmt.Sprintf(":%v", p) + } + return key, true +} + +// cachedRollup caches a rollup-backed read under a revision key, or reads through without one. +func cachedRollup[T any](ctx context.Context, cr *CachedReader, prefix string, iatas []string, fetch func(context.Context) (T, error), parts ...any) (T, error) { + key, ok := cr.rollupKey(ctx, prefix, iatas, parts...) + if !ok { + return fetch(ctx) + } + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, fetch) +} + +// hourKey matches the store's hour snapping so polls within an hour share a key. +func hourKey(t time.Time) int64 { + return t.UTC().Truncate(time.Hour).UnixMilli() +} + +// sinceKey keys a zero since on the current hour, since the store's default window slides with it. +func (cr *CachedReader) sinceKey(since time.Time) string { + if since.IsZero() { + return fmt.Sprintf("d%d", hourKey(cr.clock())) + } + return strconv.FormatInt(hourKey(since), 10) +} + +func (cr *CachedReader) clock() time.Time { + if cr.now == nil { + return time.Now() + } + return cr.now() +} + +// GetStatsSeries implements [api.Reader]. +func (cr *CachedReader) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { + return cachedRollup(ctx, cr, keyStatsSeriesPrefix, iatas, func(ctx context.Context) (*api.StatsSeries, error) { + return cr.inner.GetStatsSeries(ctx, since, until, iatas) + }, since.UnixMilli(), until.UnixMilli()) +} + +// GetSignalStats implements [api.Reader]. +func (cr *CachedReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + return cachedRollup(ctx, cr, keySignalStatsPrefix, iatas, func(ctx context.Context) (*api.SignalStats, error) { + return cr.inner.GetSignalStats(ctx, since, until, iatas) + }, since.UnixMilli(), until.UnixMilli()) +} + +// GetPathStats implements [api.Reader]. +func (cr *CachedReader) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + return cachedRollup(ctx, cr, keyPathStatsPrefix, iatas, func(ctx context.Context) (*api.PathStats, error) { + return cr.inner.GetPathStats(ctx, since, until, iatas) + }, since.UnixMilli(), until.UnixMilli()) +} + +// GetStatsOverview implements [api.Reader]. Its window follows the clock, so the current hour is part of the key. +func (cr *CachedReader) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { + return cachedRollup(ctx, cr, keyStatsOverviewPrefix, iatas, func(ctx context.Context) (*api.StatsOverview, error) { + return cr.inner.GetStatsOverview(ctx, iatas) + }, hourKey(cr.clock())) +} + +// GetStatsObservations implements [api.Reader]. +func (cr *CachedReader) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { + return cachedRollup(ctx, cr, keyStatsObservationsPrefix, iatas, func(ctx context.Context) ([]api.ObservationPoint, error) { + return cr.inner.GetStatsObservations(ctx, iatas, since) + }, cr.sinceKey(since)) +} + +// GetStatsPayloadBreakdown implements [api.Reader]. +func (cr *CachedReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { + return cachedRollup(ctx, cr, keyStatsBreakdownPrefix, iatas, func(ctx context.Context) ([]api.PayloadBreakdownItem, error) { + return cr.inner.GetStatsPayloadBreakdown(ctx, iatas, since) + }, cr.sinceKey(since)) +} + +// GetStatsTopNodes implements [api.Reader]. +func (cr *CachedReader) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { + return cachedRollup(ctx, cr, keyStatsTopNodesPrefix, iatas, func(ctx context.Context) ([]api.TopNode, error) { + return cr.inner.GetStatsTopNodes(ctx, iatas, since, limit) + }, cr.sinceKey(since), limit) +} + +// GetStatsTopObservers implements [api.Reader]. +func (cr *CachedReader) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { + return cachedRollup(ctx, cr, keyStatsTopObsPrefix, iatas, func(ctx context.Context) ([]api.TopObserver, error) { + return cr.inner.GetStatsTopObservers(ctx, iatas, since, limit) + }, cr.sinceKey(since), limit) +} + +// GetStatsTopAdvertisers implements [api.Reader]. +func (cr *CachedReader) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { + return cachedRollup(ctx, cr, keyStatsTopAdvPrefix, iatas, func(ctx context.Context) ([]api.TopAdvertiser, error) { + return cr.inner.GetStatsTopAdvertisers(ctx, iatas, since, limit) + }, cr.sinceKey(since), limit) +} + +// GetStatsTopTalkers implements [api.Reader]. +func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { + return cachedRollup(ctx, cr, keyStatsTopTalkersPrefix, iatas, func(ctx context.Context) ([]api.TopTalker, error) { + return cr.inner.GetStatsTopTalkers(ctx, iatas, since, limit) + }, cr.sinceKey(since), limit) +} + +// GetScopeStats implements [api.Reader]. +func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { + // The suffix changes with the hourly shape so older cached entries aren't served. + return cachedRollup(ctx, cr, keyScopeStats+":h2:", iatas, func(ctx context.Context) ([]api.ScopeStats, error) { + return cr.inner.GetScopeStats(ctx, iatas, since) + }, cr.sinceKey(since)) +} + +// GetScopeByName implements [api.Reader]. +func (cr *CachedReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { + return cachedRollup(ctx, cr, keyScopeByNamePrefix, nil, func(ctx context.Context) (*api.ScopeDetail, error) { + return cr.inner.GetScopeByName(ctx, name) + }, name) +} diff --git a/internal/cache/scope_stats_test.go b/internal/cache/scope_stats_test.go new file mode 100644 index 00000000..6e07d386 --- /dev/null +++ b/internal/cache/scope_stats_test.go @@ -0,0 +1,89 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type scopeStatsReader struct { + api.Reader + calls int64 +} + +func (r *scopeStatsReader) GetScopeStats(context.Context, []string, time.Time) ([]api.ScopeStats, error) { + r.calls++ + return []api.ScopeStats{{Name: "#test", PacketCount: r.calls}}, nil +} + +func (r *scopeStatsReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } + +func TestScopeStatsCacheSeparatesIATAs(t *testing.T) { + c, _ := newTestClient(t) + inner := &scopeStatsReader{} + reader := NewCachedReader(inner, c, CacheTTLs{Reference: time.Minute}) + for _, tc := range []struct { + iatas []string + want int64 + }{ + {nil, 1}, + {[]string{"YVR"}, 2}, + {[]string{"YYJ"}, 3}, + {[]string{"YYJ", "YVR"}, 4}, + {[]string{"YVR", "YYJ", "YVR"}, 4}, + {[]string{"YVR"}, 2}, + {[]string{}, 1}, + {[]string{"ZZZ"}, 5}, + {[]string{""}, 6}, + {nil, 1}, + } { + before := slices.Clone(tc.iatas) + rows, err := reader.GetScopeStats(context.Background(), tc.iatas, time.Time{}) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].PacketCount != tc.want { + t.Fatalf("IATAs %v: got %+v, want cached count %d", tc.iatas, rows, tc.want) + } + if !slices.Equal(before, tc.iatas) { + t.Fatalf("caller IATAs mutated: %v -> %v", before, tc.iatas) + } + } + if inner.calls != 6 { + t.Fatalf("underlying calls = %d, want 6", inner.calls) + } +} + +func TestDefaultWindowKeyFollowsTheHour(t *testing.T) { + c, _ := newTestClient(t) + inner := &scopeStatsReader{} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}).(*CachedReader) + clock := time.Date(2026, 10, 2, 10, 30, 0, 0, time.UTC) + cr.now = func() time.Time { return clock } + get := func(since time.Time) int64 { + t.Helper() + rows, err := cr.GetScopeStats(context.Background(), nil, since) + if err != nil { + t.Fatal(err) + } + return rows[0].PacketCount + } + if get(time.Time{}) != 1 || get(time.Time{}) != 1 { + t.Fatal("default window not cached within the hour") + } + clock = clock.Add(35 * time.Minute) + if get(time.Time{}) != 2 { + t.Fatal("default window served from the previous hour") + } + // An explicit since at the current hour is a different window. + if get(clock.Truncate(time.Hour)) != 3 { + t.Fatal("explicit since collided with the default window") + } + +} diff --git a/internal/cache/series_test.go b/internal/cache/series_test.go new file mode 100644 index 00000000..24ed0826 --- /dev/null +++ b/internal/cache/series_test.go @@ -0,0 +1,204 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type seriesReader struct { + api.Reader + calls atomic.Int64 + revision atomic.Int64 + revReads atomic.Int64 + gate chan struct{} // when set, fetches block until closed +} + +func (r *seriesReader) GetStatsSeries(context.Context, time.Time, time.Time, []string) (*api.StatsSeries, error) { + if r.gate != nil { + <-r.gate + } + return &api.StatsSeries{Revision: r.calls.Add(1)}, nil +} + +func (r *seriesReader) AnalyticsRevision(context.Context) (int64, error) { + r.revReads.Add(1) + return r.revision.Load(), nil +} + +// seriesOnly hides AnalyticsRevision. +type seriesOnly struct{ *seriesReader } + +func (seriesOnly) AnalyticsRevision() {} + +func TestSeriesCacheKeyedByRevision(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}).(*CachedReader) + clock := time.Unix(0, 0) + cr.rev.now = func() time.Time { return clock } + get := func(iatas []string) int64 { + t.Helper() + got, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), iatas) + if err != nil { + t.Fatal(err) + } + return got.Revision + } + if get(nil) != 1 || get(nil) != 1 || get([]string{""}) != 2 || get([]string{"YVR", "YYJ"}) != 3 || get([]string{"YYJ", "YVR"}) != 3 { + t.Fatalf("unexpected cache behaviour: %d fetches", inner.calls.Load()) + } + // A new roll is invisible until the memo expires, then gets a fresh key. + inner.revision.Store(1) + if get(nil) != 1 { + t.Fatal("revision re-read inside the memo window") + } + clock = clock.Add(revisionTTL) + if get(nil) != 4 { + t.Fatal("new revision did not miss the cache") + } + if n := inner.revReads.Load(); n != 2 { + t.Errorf("revision read %d times, want 2", n) + } +} + +func TestSeriesCacheWithoutRevisionIsUncached(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{} + cr := NewCachedReader(seriesOnly{inner}, c, CacheTTLs{Stats: time.Hour}) + for range 2 { + if _, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil); err != nil { + t.Fatal(err) + } + } + if inner.calls.Load() != 2 { + t.Errorf("fetches = %d, want 2", inner.calls.Load()) + } +} + +func TestSeriesCacheSingleflight(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{gate: make(chan struct{})} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + if _, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil); err != nil { + t.Error(err) + } + }) + } + time.Sleep(50 * time.Millisecond) // let every caller reach the miss + close(inner.gate) + wg.Wait() + if n := inner.calls.Load(); n != 1 { + t.Errorf("concurrent misses ran %d fetches, want 1", n) + } +} + +// ctxSeriesReader blocks until released and fails if its fetch context was cancelled. +type ctxSeriesReader struct { + seriesReader + started chan struct{} + release chan struct{} +} + +func (r *ctxSeriesReader) GetStatsSeries(ctx context.Context, _, _ time.Time, _ []string) (*api.StatsSeries, error) { + r.calls.Add(1) + close(r.started) + <-r.release + if err := ctx.Err(); err != nil { + return nil, err + } + return &api.StatsSeries{Hours: []api.StatsSeriesHour{{Status: "complete"}}}, nil +} + +func TestSharedFetchSurvivesLeaderCancel(t *testing.T) { + c, _ := newTestClient(t) + inner := &ctxSeriesReader{started: make(chan struct{}), release: make(chan struct{})} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + leaderErr := make(chan error, 1) + go func() { + _, err := cr.GetStatsSeries(leaderCtx, time.UnixMilli(0), time.UnixMilli(3600000), nil) + leaderErr <- err + }() + <-inner.started + waiter := make(chan *api.StatsSeries, 1) + go func() { + got, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil) + if err != nil { + t.Error(err) + } + waiter <- got + }() + time.Sleep(20 * time.Millisecond) // let the waiter join the in-flight fetch + cancelLeader() + if err := <-leaderErr; !errors.Is(err, context.Canceled) { + t.Errorf("leader err = %v, want its own cancellation", err) + } + close(inner.release) + if got := <-waiter; got == nil || len(got.Hours) != 1 { + t.Errorf("waiter got %+v; the leader's cancellation leaked into the shared fetch", got) + } + if n := inner.calls.Load(); n != 1 { + t.Errorf("fetches = %d, want 1", n) + } +} + +func TestSharedFetchWaiterHonoursOwnDeadline(t *testing.T) { + c, _ := newTestClient(t) + inner := &ctxSeriesReader{started: make(chan struct{}), release: make(chan struct{})} + defer close(inner.release) + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + go func() { + _, _ = cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil) + }() + <-inner.started + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := cr.GetStatsSeries(ctx, time.UnixMilli(0), time.UnixMilli(3600000), nil); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("err = %v, want the waiter's deadline", err) + } + if time.Since(start) > time.Second { + t.Error("waiter stayed blocked on the leader") + } +} + +func TestSharedFetchReturnsIndependentCopies(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{gate: make(chan struct{})} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + results := make(chan *api.StatsSeries, 2) + var wg sync.WaitGroup + for range 2 { + wg.Go(func() { + got, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil) + if err != nil { + t.Error(err) + return + } + got.Since = time.Now().UnixNano() // a handler decorating its own response + results <- got + }) + } + time.Sleep(20 * time.Millisecond) + close(inner.gate) + wg.Wait() + a, b := <-results, <-results + if a == b { + t.Fatal("concurrent callers share one response object") + } + if inner.calls.Load() != 1 { + t.Errorf("fetches = %d, want 1", inner.calls.Load()) + } +} diff --git a/internal/cache/signal_test.go b/internal/cache/signal_test.go new file mode 100644 index 00000000..5e8f3244 --- /dev/null +++ b/internal/cache/signal_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type signalStatsReader struct { + api.Reader + calls int64 +} + +func (r *signalStatsReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } + +func (r *signalStatsReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { + r.calls++ + return &api.SignalStats{Receptions: r.calls}, nil +} + +func TestSignalCacheSeparatesWindowAndRegion(t *testing.T) { + c, mr := newTestClient(t) + inner := &signalStatsReader{} + r := NewCachedReader(inner, c, CacheTTLs{Stats: time.Minute}) + for _, tc := range []struct { + iatas []string + since, until, want int64 + }{ + {nil, 0, 10, 1}, {[]string{}, 0, 10, 1}, {[]string{""}, 0, 10, 2}, {[]string{"YVR"}, 0, 10, 3}, + {[]string{"YVR", "YYJ"}, 0, 10, 4}, {[]string{"YYJ", "YVR", "YYJ"}, 0, 10, 4}, + {nil, 1, 10, 5}, {nil, 0, 11, 6}, {nil, 0, 10, 1}, + } { + before := slices.Clone(tc.iatas) + got, err := r.GetSignalStats(context.Background(), time.UnixMilli(tc.since), time.UnixMilli(tc.until), tc.iatas) + if err != nil || got.Receptions != tc.want || !slices.Equal(before, tc.iatas) { + t.Fatalf("%+v: %+v %v", tc, got, err) + } + } + mr.FastForward(time.Minute) + got, err := r.GetSignalStats(context.Background(), time.UnixMilli(0), time.UnixMilli(10), nil) + if err != nil || got.Receptions != 7 { + t.Fatalf("expiry: %+v %v", got, err) + } +} diff --git a/internal/config/auth_test.go b/internal/config/auth_test.go new file mode 100644 index 00000000..fb7e02ba --- /dev/null +++ b/internal/config/auth_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadAuth(t *testing.T) { + for _, tc := range []struct { + name, env, want string + file, set bool + }{ + {"no config", "", "", false, false}, + {"YAML", "", "yaml-test-key-0123456789", true, false}, + {"environment overrides", "env-test-key-0123456789", "env-test-key-0123456789", true, true}, + {"empty environment disables", "", "", true, true}, + {"environment without file", "env-test-key-0123456789", "env-test-key-0123456789", false, true}, + {"surrounding whitespace trimmed", " env-test-key-0123456789\n", "env-test-key-0123456789", true, true}, + {"whitespace-only disables", " \t", "", true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("BEACON_API_KEY", tc.env) + if !tc.set { + if err := os.Unsetenv("BEACON_API_KEY"); err != nil { + t.Fatal(err) + } + } + path := filepath.Join(t.TempDir(), "config.yaml") + if tc.file { + if err := os.WriteFile(path, []byte("auth:\n api_key: yaml-test-key-0123456789\n"), 0600); err != nil { + t.Fatal(err) + } + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if cfg.Auth.APIKey != tc.want { + t.Fatal("wrong auth configuration precedence") + } + encoded, err := json.Marshal(cfg) + if err != nil || strings.Contains(string(encoded), "test-key") { + t.Fatal("key exposed in JSON") + } + if strings.Contains(Resolve(cfg).String(), "test-key") { + t.Fatal("key exposed in startup summary") + } + }) + } +} + +func TestLoadAuthRejectsUnusableKey(t *testing.T) { + for _, tc := range []struct{ name, key, want string }{ + {"inner space", "env-test-key 0123456789", "whitespace"}, + {"inner tab", "env-test-key\t0123456789", "whitespace"}, + {"too short", "short-test-key", "at least 16"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("BEACON_API_KEY", tc.key) + for _, file := range []bool{false, true} { + path := filepath.Join(t.TempDir(), "config.yaml") + if file { + if err := os.WriteFile(path, []byte("log:\n level: info\n"), 0600); err != nil { + t.Fatal(err) + } + } + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("file=%t: err = %v, want %q", file, err, tc.want) + } + if strings.Contains(err.Error(), "test-key") { + t.Fatal("key exposed in error") + } + } + }) + } +} diff --git a/internal/config/border.go b/internal/config/border.go index 0978a408..285486d5 100644 --- a/internal/config/border.go +++ b/internal/config/border.go @@ -15,7 +15,7 @@ import ( // map: well-formed Feature, geometry Polygon or MultiPolygon, every coordinate within // lon [-180,180] / lat [-90,90], every ring closed (first point == last point, 4+ points -- // see orb.Ring.Closed). On success, returns the Feature re-marshaled to JSON with its bbox -// computed and set, so reads never need to recompute it (db/migrations/011_add_iata_border.sql). +// computed and set, so reads never need to recompute it (iata_codes.border). // // Coordinate order: GeoJSON packs coordinates [lon, lat], not [lat, lon]. A border authored // with the axes swapped is still well-formed JSON and its numbers can still individually fall @@ -30,6 +30,9 @@ func ValidateBorder(raw []byte) (json.RawMessage, error) { if feat.Type != "Feature" { return nil, fmt.Errorf(`top-level "type" must be "Feature", got %q`, feat.Type) } + if feat.Geometry == nil { + return nil, fmt.Errorf("Feature must have a Polygon or MultiPolygon geometry") + } switch geom := feat.Geometry.(type) { case orb.Polygon: if err := validateBorderPolygon(geom); err != nil { diff --git a/internal/config/channels.go b/internal/config/channels.go new file mode 100644 index 00000000..09216c30 --- /dev/null +++ b/internal/config/channels.go @@ -0,0 +1,66 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "encoding/hex" + "fmt" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "gopkg.in/yaml.v3" +) + +// HashtagConfig is a hashtag channel; a plain string in YAML is a Beacon-wide one. +type HashtagConfig struct { + Name string `yaml:"name"` + Region string `yaml:"region"` // optional region slug +} + +func (h *HashtagConfig) UnmarshalYAML(node *yaml.Node) error { + if node.Kind == yaml.ScalarNode { + return node.Decode(&h.Name) + } + type plain HashtagConfig + return node.Decode((*plain)(h)) +} + +// ChannelScope places a configured channel under a region, or Beacon-wide when Region is empty. +type ChannelScope struct { + Fingerprint []byte + Region string +} + +func (c *Config) validateChannelKeys() error { + for i, h := range c.ChannelKeys.Hashtags { + if strings.TrimSpace(h.Name) == "" { + return fmt.Errorf("channel_keys.hashtags[%d] needs a name", i) + } + if h.Region != "" && c.region(h.Region) == nil { + return fmt.Errorf("channel #%s region %q is not a configured region", h.Name, h.Region) + } + } + for hash, k := range c.ChannelKeys.Keys { + if k.Region != "" && c.region(k.Region) == nil { + return fmt.Errorf("channel key %s region %q is not a configured region", hash, k.Region) + } + } + return nil +} + +// ChannelScopes lists every configured channel's region placement. Keys that +// aren't valid hex are skipped; startup already warns about them. +func (c *Config) ChannelScopes() []ChannelScope { + var scopes []ChannelScope + for _, h := range c.ChannelKeys.Hashtags { + _, _, fingerprint := keystore.DeriveHashtagKey(h.Name) + scopes = append(scopes, ChannelScope{Fingerprint: fingerprint, Region: h.Region}) + } + for _, k := range c.ChannelKeys.Keys { + if key, err := hex.DecodeString(k.Key); err == nil { + scopes = append(scopes, ChannelScope{Fingerprint: keystore.Fingerprint(key), Region: k.Region}) + } + } + return scopes +} diff --git a/internal/config/channels_test.go b/internal/config/channels_test.go new file mode 100644 index 00000000..48656708 --- /dev/null +++ b/internal/config/channels_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" +) + +func loadText(t *testing.T, text string) (*Config, error) { + t.Helper() + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(text), 0600); err != nil { + t.Fatal(err) + } + return Load(p) +} + +func TestChannelKeysRegionScoping(t *testing.T) { + text := `regions: + - slug: east + iatas: [YOW] +channel_keys: + hashtags: + - meshcore + - name: ottawa-mesh + region: east + keys: + "11": + key: "8b3387e9c5cdea6ac9e5edbaa115cd72" + name: Public +` + cfg, err := loadText(t, text) + if err != nil { + t.Fatal(err) + } + if got := cfg.ChannelKeys.Hashtags; len(got) != 2 || got[0].Name != "meshcore" || got[0].Region != "" || got[1].Name != "ottawa-mesh" || got[1].Region != "east" { + t.Fatalf("hashtags parsed as %+v", got) + } + scopes := cfg.ChannelScopes() + if len(scopes) != 3 { + t.Fatalf("scopes: %+v", scopes) + } + _, _, ottawa := keystore.DeriveHashtagKey("ottawa-mesh") + found := false + for _, s := range scopes { + if bytes.Equal(s.Fingerprint, ottawa) { + found = s.Region == "east" + } + } + if !found { + t.Fatal("region-scoped hashtag missing its region", scopes) + } + + for name, bad := range map[string]string{ + "unknown hashtag region": strings.Replace(text, "region: east", "region: west", 1), + "empty hashtag": strings.Replace(text, "- meshcore", `- ""`, 1), + "unknown key region": text + " region: west\n", + } { + if _, err := loadText(t, bad); err == nil { + t.Errorf("%s accepted", name) + } + } +} + +func TestMeshMapperChannelsConfig(t *testing.T) { + valid := "meshmapper:\n channels:\n enabled: true\n" + for text, bad := range map[string]bool{ + valid: false, + valid + " refresh_interval: 24h\n": false, + valid + " refresh_interval: 168h\n": false, + valid + " refresh_interval: 23h\n": true, + valid + " refresh_interval: 169h\n": true, + } { + if _, err := loadText(t, text); (err != nil) != bad { + t.Errorf("%q: error=%v", text, err) + } + } + if (MeshMapperChannelsConfig{}).Interval() != MinChannelsRefresh { + t.Fatal("wrong default") + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 82c4bb81..94b109e8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,39 +7,101 @@ package config import ( "fmt" + "net/netip" + "net/url" "os" "path/filepath" + "slices" + "strings" "time" + "unicode" "gopkg.in/yaml.v3" ) // Config is the top-level structure of the Beacon config file. type Config struct { + Auth AuthConfig `yaml:"auth"` + Backup BackupConfig `yaml:"backup"` + Log LogConfig `yaml:"log"` + Server ServerConfig `yaml:"server"` IATAs map[string]IATAConfig `yaml:"iatas"` Regions []RegionConfig `yaml:"regions"` ChannelKeys ChannelKeysConfig `yaml:"channel_keys"` Telemetry TelemetryConfig `yaml:"telemetry"` WebSocket WebSocketConfig `yaml:"websocket"` Packets PacketsConfig `yaml:"packets"` + Routes RoutesConfig `yaml:"routes"` Ingest IngestFilterConfig `yaml:"ingest"` Scopes []ScopeConfig `yaml:"scopes"` + MeshMapper MeshMapperConfig `yaml:"meshmapper"` Cache CacheConfig `yaml:"cache"` CORS CORSConfig `yaml:"cors"` + RateLimit RateLimitConfig `yaml:"ratelimit"` Background BackgroundConfig `yaml:"background"` Presence PresenceConfig `yaml:"presence"` Nodes NodesConfig `yaml:"nodes"` + Observers ObserversConfig `yaml:"observers"` + Analytics AnalyticsConfig `yaml:"analytics"` +} + +// BackupConfig enables protected downloads. Disabled by default. +type BackupConfig struct { + Enabled bool `yaml:"enabled"` +} + +// AuthConfig holds the operator key for the protected admin subtree. +// The key is excluded from JSON; it must not be exposed by configuration APIs. +type AuthConfig struct { + APIKey string `yaml:"api_key" json:"-"` +} + +// LogConfig controls application verbosity and stderr output format. +type LogConfig struct { + Level string `yaml:"level"` + Format string `yaml:"format"` +} + +// ServerConfig controls which direct peers may supply the client address. +type ServerConfig struct { + // TrustedProxies accepts IPv4/IPv6 CIDRs; an empty list trusts no proxy. + // netip.Prefix validates each CIDR while the configuration is loaded. + TrustedProxies []netip.Prefix `yaml:"trusted_proxies"` +} + +func (c *ServerConfig) UnmarshalYAML(node *yaml.Node) error { + // Pointers retain null list entries, which yaml would otherwise discard. + // Leave them as invalid prefixes for Load's validation below. + var raw struct { + TrustedProxies []*netip.Prefix `yaml:"trusted_proxies"` + } + if err := node.Decode(&raw); err != nil { + return err + } + c.TrustedProxies = make([]netip.Prefix, len(raw.TrustedProxies)) + for i, prefix := range raw.TrustedProxies { + if prefix != nil { + c.TrustedProxies[i] = *prefix + } + } + return nil } // ResolvedConfig holds all runtime configuration with defaults applied. type ResolvedConfig struct { - TelemetryResolution time.Duration - TelemetryRetention time.Duration - PacketRetention time.Duration - MaxConnsPerIP int - ViewRefreshInterval time.Duration - ReconfirmInterval time.Duration - CleanupInterval time.Duration + RateLimit ResolvedRateLimitConfig + TelemetryResolution time.Duration + TelemetryRetention time.Duration + RollupRetention time.Duration + PacketRetention time.Duration + RouteRetention time.Duration + RouteGrace time.Duration + RouteMinObservations int + MaxConnsPerIP int + MaxConnectsPerMinute int + ViewRefreshInterval time.Duration + ReconfirmInterval time.Duration + CleanupInterval time.Duration PresenceFlushInterval time.Duration PresencePacketTTL time.Duration @@ -51,8 +113,23 @@ type ResolvedConfig struct { // NodeStaleThreshold and NodeDeleteAfter mirror ClockDriftThreshold's "0 means unset, // resolve to a default" pattern -- see NodesConfig. - NodeStaleThreshold time.Duration - NodeDeleteAfter time.Duration + NodeStaleThreshold time.Duration + NodeDeleteAfter time.Duration + ObserverDeleteAfter time.Duration +} + +// RateLimitConfig controls the per-client REST API request budget. +type RateLimitConfig struct { + Enabled *bool `yaml:"enabled"` // Defaults to true; false disables both windows. + RequestsPerMinute int `yaml:"requests_per_minute"` + Burst int `yaml:"burst"` // One-second window cap, not token-bucket capacity. +} + +// ResolvedRateLimitConfig has defaults applied and no optional values. +type ResolvedRateLimitConfig struct { + Enabled bool + RequestsPerMinute int + Burst int } // PresenceConfig controls coalescing of presence bookkeeping writes @@ -126,7 +203,7 @@ type CacheConfig struct { type CacheTTLsConfig struct { // Stats controls the TTL for aggregated network statistics endpoints // (overview, observations, payload breakdown, top nodes/observers, radio presets, scope stats). - // These are backed by materialized views refreshed hourly, so values under 1m are rarely useful. + // Rollup-backed keys also change with the rollup revision. Stats duration `yaml:"stats"` // Reference controls the TTL for mostly-static reference data @@ -147,13 +224,14 @@ type CacheTTLsConfig struct { // Name can be provided with or without the # or $ prefix. // Beacon normalizes plain names by prepending #. type ScopeConfig struct { - Name string `yaml:"name"` // e.g. "bc", "#west", "$private" + Name string `yaml:"name"` // e.g. "bc", "#west", "$private" + Region string `yaml:"region"` // configured region slug; lists the scope under that region's IATAs } // TelemetryConfig controls observer telemetry storage behaviour. type TelemetryConfig struct { // Retention is how long telemetry rows are kept before the cleanup job removes them. - // Defaults to 672h (4 weeks) if not set. + // Defaults to 744h (31 days) if not set. Retention duration `yaml:"retention"` // Resolution is how frequently a telemetry snapshot is stored per observer. @@ -168,17 +246,45 @@ type WebSocketConfig struct { // MaxConnectionsPerIP is the maximum number of concurrent WebSocket // connections allowed from a single IP address. Defaults to 5 if not set. MaxConnectionsPerIP int `yaml:"max_connections_per_ip"` + // MaxConnectsPerMinute limits upgrade attempts, including failed handshakes. + // Zero/omitted defaults to 10; IPv6 addresses share a /64 attempt budget. + MaxConnectsPerMinute int `yaml:"max_connects_per_minute"` + // AllowedOrigins are extra origins allowed to open /ws: exact, or scheme://*.domain[:port] + // for every subdomain of domain (never the apex). Defaults to same-host only. + AllowedOrigins []string `yaml:"allowed_origins"` } // PacketsConfig controls packet retention behaviour. type PacketsConfig struct { - // Retention is how long packet and observation rows are kept. - // Defaults to 720h (30 days) if not set. + // Retention covers packets, observations and channel messages. Defaults to 168h. Retention duration `yaml:"retention"` } +// AnalyticsConfig controls hourly rollup retention behaviour. +type AnalyticsConfig struct { + // RollupRetention is how long hourly rollups are kept and the longest window /stats/series accepts. Defaults to 2160h (90 days). + RollupRetention duration `yaml:"rollup_retention"` +} + +// RoutesConfig controls known-route retention behaviour. +type RoutesConfig struct { + // Retention is how long a route is kept after it was last observed. + // Defaults to 336h (14 days) if not set. + Retention duration `yaml:"retention"` + // Grace is how long a route observed fewer than MinObservations times is + // kept. Defaults to 168h (7 days) if not set. + Grace duration `yaml:"grace"` + // MinObservations is the observation count below which Grace applies + // instead of Retention. Defaults to 3 if not set. + MinObservations int `yaml:"min_observations"` +} + // NodesConfig controls node-derived signal thresholds. type NodesConfig struct { + // MarkForeign annotates repeaters outside the union of IATA borders, with + // imported MeshMapper boundaries replacing border files. Disabled by default; + // enabling it requires a border file or meshmapper.zones. + MarkForeign bool `yaml:"mark_foreign"` // ClockDriftThreshold is the |device clock - server clock| magnitude, measured from a // repeater/room server's ADVERT timestamp, above which the node API reports // clockOutOfSync=true for that node. Defaults to 5m if not set. @@ -187,8 +293,15 @@ type NodesConfig struct { // stale=true for it. Defaults to 24h if not set. StaleThreshold duration `yaml:"stale_threshold"` // DeleteAfter is how long since a node's last_seen before the cleanup job deletes the - // node entirely. Defaults to the same 30-day default as packets.retention if not set -- - // independently configurable from it, just the same starting point. + // node entirely. Defaults to 720h (30 days) if not set. + DeleteAfter duration `yaml:"delete_after"` +} + +// ObserversConfig controls optional observer age-out. +type ObserversConfig struct { + // DeleteAfter is how long an observer must be unseen before it can be deleted. + // Omitted or nonpositive disables age-out. Retained packet observations, + // telemetry and ownership records always protect the observer from deletion. DeleteAfter duration `yaml:"delete_after"` } @@ -216,9 +329,9 @@ func (d *duration) UnmarshalYAML(value *yaml.Node) error { // channel_hash = SHA256(secret)[0]. Explicit keys are provided as hex strings // keyed by the channel hash hex (e.g. "11" for 0x11). type ChannelKeysConfig struct { - // Hashtags is a list of hashtag names (without the # prefix). + // Hashtags lists hashtag names (without the # prefix), each optionally scoped to a region. // Beacon derives the PSK and channel hash automatically. - Hashtags []string `yaml:"hashtags"` + Hashtags []HashtagConfig `yaml:"hashtags"` // Keys maps channel hash hex → explicit key config. Keys map[string]ExplicitKeyConfig `yaml:"keys"` @@ -226,8 +339,9 @@ type ChannelKeysConfig struct { // ExplicitKeyConfig holds an explicit channel key and optional display name. type ExplicitKeyConfig struct { - Key string `yaml:"key"` // hex-encoded key bytes - Name string `yaml:"name"` // optional display name + Key string `yaml:"key"` // hex-encoded key bytes + Name string `yaml:"name"` // optional display name + Region string `yaml:"region"` // optional region slug; empty lists the channel Beacon-wide } // IATAConfig holds optional overrides for a known IATA code. @@ -279,6 +393,10 @@ func Load(path string) (*Config, error) { data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { + cfg.Auth.APIKey = os.Getenv("BEACON_API_KEY") + if err := cfg.validateAPIKey(); err != nil { + return nil, err + } return cfg, nil } return nil, err @@ -286,7 +404,50 @@ func Load(path string) (*Config, error) { if err := yaml.Unmarshal(data, cfg); err != nil { return nil, err } + if value, set := os.LookupEnv("BEACON_API_KEY"); set { + cfg.Auth.APIKey = value + } + if err := cfg.validateAPIKey(); err != nil { + return nil, err + } + if err := cfg.validateDurations(); err != nil { + return nil, err + } + for i, prefix := range cfg.Server.TrustedProxies { + if !prefix.IsValid() { + return nil, fmt.Errorf("server.trusted_proxies[%d] must be a valid CIDR", i) + } + } + if cfg.RateLimit.RequestsPerMinute < 0 || cfg.RateLimit.Burst < 0 { + return nil, fmt.Errorf("ratelimit.requests_per_minute and ratelimit.burst must be positive or zero for defaults") + } + if cfg.WebSocket.MaxConnectsPerMinute < 0 { + return nil, fmt.Errorf("websocket.max_connects_per_minute must be positive or zero for the default") + } + for i, origin := range cfg.WebSocket.AllowedOrigins { + if err := validateOrigin(origin); err != nil { + return nil, fmt.Errorf("websocket.allowed_origins[%d]: %w", i, err) + } + } + if d := cfg.Packets.Retention.Duration; d != 0 && d < 24*time.Hour { + return nil, fmt.Errorf("packets.retention must be at least 24h") + } + if d := cfg.Analytics.RollupRetention.Duration; d != 0 && d < 24*time.Hour { + return nil, fmt.Errorf("analytics.rollup_retention must be at least 24h") + } configDir := filepath.Dir(path) + if err := cfg.validateMeshMapper(); err != nil { + return nil, err + } + if err := cfg.validateChannelKeys(); err != nil { + return nil, err + } + if err := cfg.validateScopes(); err != nil { + return nil, err + } + if cfg.Nodes.MarkForeign && !cfg.MeshMapper.Zones.Enabled && !cfg.hasBorderFile() { + return nil, fmt.Errorf("nodes.mark_foreign requires an iatas.*.borderFile or meshmapper.zones") + } for iata, details := range cfg.IATAs { if details.BorderFile != "" && !filepath.IsAbs(details.BorderFile) { details.BorderFile = filepath.Join(configDir, details.BorderFile) @@ -296,16 +457,100 @@ func Load(path string) (*Config, error) { return cfg, nil } +// minAPIKeyLength keeps the admin key out of guessing range. +const minAPIKeyLength = 16 + +// validateAPIKey trims the key, since Bearer tokens can't carry whitespace; errors never echo it. +func (c *Config) validateAPIKey() error { + c.Auth.APIKey = strings.TrimSpace(c.Auth.APIKey) + switch key := c.Auth.APIKey; { + case key == "": + return nil + case strings.IndexFunc(key, unicode.IsSpace) >= 0: + return fmt.Errorf("auth.api_key / BEACON_API_KEY must not contain whitespace") + case len(key) < minAPIKeyLength: + return fmt.Errorf("auth.api_key / BEACON_API_KEY must be at least %d characters", minAPIKeyLength) + } + return nil +} + +// validateDurations rejects negative intervals, which would otherwise panic tickers or invert cutoffs. +// observers.delete_after is exempt: nonpositive disables it. +func (c *Config) validateDurations() error { + for _, d := range []struct { + key string + value duration + }{ + {"presence.flush_interval", c.Presence.FlushInterval}, + {"presence.packet_ttl", c.Presence.PacketTTL}, + {"background.view_refresh", c.Background.ViewRefresh}, + {"background.reconfirm", c.Background.Reconfirm}, + {"background.cleanup", c.Background.Cleanup}, + {"cache.ttl", c.Cache.TTL}, + {"cache.ttls.stats", c.Cache.TTLs.Stats}, + {"cache.ttls.reference", c.Cache.TTLs.Reference}, + {"cache.ttls.nodes", c.Cache.TTLs.Nodes}, + {"cache.ttls.observers", c.Cache.TTLs.Observers}, + {"telemetry.retention", c.Telemetry.Retention}, + {"telemetry.resolution", c.Telemetry.Resolution}, + {"routes.retention", c.Routes.Retention}, + {"routes.grace", c.Routes.Grace}, + {"nodes.clock_drift_threshold", c.Nodes.ClockDriftThreshold}, + {"nodes.stale_threshold", c.Nodes.StaleThreshold}, + {"nodes.delete_after", c.Nodes.DeleteAfter}, + {"meshmapper.scopes.refresh_interval", c.MeshMapper.Scopes.RefreshInterval}, + {"meshmapper.zones.refresh_interval", c.MeshMapper.Zones.RefreshInterval}, + {"meshmapper.channels.refresh_interval", c.MeshMapper.Channels.RefreshInterval}, + } { + if d.value.Duration < 0 { + return fmt.Errorf("%s must not be negative (omit it or use 0 for the default)", d.key) + } + } + return nil +} + +// validateOrigin allows exact origins or a leading "*." subdomain wildcard; the WebSocket +// library treats entries as path.Match patterns, so every other pattern character is rejected. +func validateOrigin(origin string) error { + u, err := url.Parse(origin) + if err != nil { + return fmt.Errorf("invalid origin %q: %w", origin, err) + } + host := u.Hostname() + // At least two labels after the wildcard, so it never spans a whole TLD. + if rest, ok := strings.CutPrefix(host, "*."); ok { + if labels := strings.Split(rest, "."); len(labels) >= 2 && !slices.Contains(labels, "") { + host = rest + } + } + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || + u.Path != "" || u.RawQuery != "" || u.Fragment != "" || u.Opaque != "" || + strings.Contains(host, "*") || strings.ContainsAny(origin, `?[]\`) { + return fmt.Errorf("origin %q must be scheme://host[:port] or scheme://*.domain[:port] with an http or https scheme", origin) + } + return nil +} + // Resolve returns a ResolvedConfig with defaults applied for any zero values. func Resolve(cfg *Config) ResolvedConfig { r := ResolvedConfig{ - TelemetryResolution: cfg.Telemetry.Resolution.Duration, - TelemetryRetention: cfg.Telemetry.Retention.Duration, - PacketRetention: cfg.Packets.Retention.Duration, - MaxConnsPerIP: cfg.WebSocket.MaxConnectionsPerIP, - ViewRefreshInterval: cfg.Background.ViewRefresh.Duration, - ReconfirmInterval: cfg.Background.Reconfirm.Duration, - CleanupInterval: cfg.Background.Cleanup.Duration, + RateLimit: ResolvedRateLimitConfig{ + Enabled: cfg.RateLimit.Enabled == nil || *cfg.RateLimit.Enabled, + RequestsPerMinute: cfg.RateLimit.RequestsPerMinute, + Burst: cfg.RateLimit.Burst, + }, + TelemetryResolution: cfg.Telemetry.Resolution.Duration, + TelemetryRetention: cfg.Telemetry.Retention.Duration, + RollupRetention: cfg.Analytics.RollupRetention.Duration, + PacketRetention: cfg.Packets.Retention.Duration, + RouteRetention: cfg.Routes.Retention.Duration, + RouteGrace: cfg.Routes.Grace.Duration, + RouteMinObservations: cfg.Routes.MinObservations, + MaxConnsPerIP: cfg.WebSocket.MaxConnectionsPerIP, + MaxConnectsPerMinute: cfg.WebSocket.MaxConnectsPerMinute, + ViewRefreshInterval: cfg.Background.ViewRefresh.Duration, + ReconfirmInterval: cfg.Background.Reconfirm.Duration, + CleanupInterval: cfg.Background.Cleanup.Duration, PresenceFlushInterval: cfg.Presence.FlushInterval.Duration, PresencePacketTTL: cfg.Presence.PacketTTL.Duration, @@ -313,19 +558,41 @@ func Resolve(cfg *Config) ResolvedConfig { ClockDriftThreshold: cfg.Nodes.ClockDriftThreshold.Duration, NodeStaleThreshold: cfg.Nodes.StaleThreshold.Duration, NodeDeleteAfter: cfg.Nodes.DeleteAfter.Duration, + ObserverDeleteAfter: cfg.Observers.DeleteAfter.Duration, + } + if r.RateLimit.RequestsPerMinute == 0 { + r.RateLimit.RequestsPerMinute = 300 + } + if r.RateLimit.Burst == 0 { + r.RateLimit.Burst = r.RateLimit.RequestsPerMinute } if r.TelemetryResolution == 0 { r.TelemetryResolution = time.Hour } if r.TelemetryRetention == 0 { - r.TelemetryRetention = 28 * 24 * time.Hour + r.TelemetryRetention = 31 * 24 * time.Hour + } + if r.RollupRetention == 0 { + r.RollupRetention = 90 * 24 * time.Hour } if r.PacketRetention == 0 { - r.PacketRetention = 30 * 24 * time.Hour + r.PacketRetention = 7 * 24 * time.Hour + } + if r.RouteRetention == 0 { + r.RouteRetention = 14 * 24 * time.Hour + } + if r.RouteGrace == 0 { + r.RouteGrace = 7 * 24 * time.Hour + } + if r.RouteMinObservations == 0 { + r.RouteMinObservations = 3 } if r.MaxConnsPerIP == 0 { r.MaxConnsPerIP = 5 } + if r.MaxConnectsPerMinute == 0 { + r.MaxConnectsPerMinute = 10 + } if r.ViewRefreshInterval == 0 { r.ViewRefreshInterval = time.Hour } @@ -348,8 +615,6 @@ func Resolve(cfg *Config) ResolvedConfig { r.NodeStaleThreshold = 24 * time.Hour } if r.NodeDeleteAfter == 0 { - // Same default as packets.retention (30 days) -- independently configurable, just - // the same starting point, not tied to whatever PacketRetention resolves to. r.NodeDeleteAfter = 30 * 24 * time.Hour } return r @@ -357,10 +622,11 @@ func Resolve(cfg *Config) ResolvedConfig { func (r ResolvedConfig) String() string { return fmt.Sprintf( - "telemetryResolution=%s telemetryRetention=%s packetRetention=%s maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s", - r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, - r.MaxConnsPerIP, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, + "telemetryResolution=%s telemetryRetention=%s rollupRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d maxConnectsPerMinute=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d", + r.TelemetryResolution, r.TelemetryRetention, r.RollupRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, + r.MaxConnsPerIP, r.MaxConnectsPerMinute, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold, - r.NodeStaleThreshold, r.NodeDeleteAfter, + r.NodeStaleThreshold, r.NodeDeleteAfter, r.ObserverDeleteAfter, + r.RateLimit.Enabled, r.RateLimit.RequestsPerMinute, r.RateLimit.Burst, ) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a68e18b2..f7716de2 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -5,11 +5,56 @@ package config import ( "os" + "path/filepath" "strings" "testing" "time" ) +func TestLoadLogConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte("log:\n level: warn\n format: json\n"), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if cfg.Log.Level != "warn" || cfg.Log.Format != "json" { + t.Fatalf("log config ignored: %+v", cfg.Log) + } +} + +func TestLoadTrustedProxies(t *testing.T) { + for _, tc := range []struct { + name, yaml string + count int + wantError bool + }{ + {"omitted", "{}", 0, false}, + {"empty", "server: {trusted_proxies: []}", 0, false}, + {"CIDRs", "server: {trusted_proxies: ['192.0.2.0/24', '2001:db8::/32']}", 2, false}, + {"bare IP", "server: {trusted_proxies: ['192.0.2.1']}", 0, true}, + {"invalid CIDR", "server: {trusted_proxies: ['192.0.2.0/99']}", 0, true}, + {"empty entry", "server: {trusted_proxies: ['', '192.0.2.0/24']}", 0, true}, + {"null entry", "server: {trusted_proxies: [null, '192.0.2.0/24']}", 0, true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if (err != nil) != tc.wantError { + t.Fatalf("Load error = %v, want error = %v", err, tc.wantError) + } + if err == nil && len(cfg.Server.TrustedProxies) != tc.count { + t.Fatalf("got %d proxy prefixes, want %d", len(cfg.Server.TrustedProxies), tc.count) + } + }) + } +} + func TestLoad_FileNotFound(t *testing.T) { cfg, err := Load("/nonexistent/path/config.yaml") if err != nil { @@ -38,6 +83,8 @@ regions: name: British Columbia display_order: 1 iatas: [YVR] +observers: + delete_after: 720h `) f.Close() @@ -54,6 +101,9 @@ regions: if cfg.Regions[0].Slug != "bc" { t.Errorf("expected slug bc, got %s", cfg.Regions[0].Slug) } + if Resolve(cfg).ObserverDeleteAfter != 30*24*time.Hour { + t.Error("observers.delete_after was not loaded from YAML") + } } func TestLoad_InvalidYAML(t *testing.T) { @@ -77,11 +127,11 @@ func TestResolve_Defaults(t *testing.T) { if r.TelemetryResolution != time.Hour { t.Errorf("expected TelemetryResolution 1h, got %v", r.TelemetryResolution) } - if r.TelemetryRetention != 28*24*time.Hour { - t.Errorf("expected TelemetryRetention 672h, got %v", r.TelemetryRetention) + if r.TelemetryRetention != 31*24*time.Hour { + t.Errorf("expected TelemetryRetention 744h, got %v", r.TelemetryRetention) } - if r.PacketRetention != 30*24*time.Hour { - t.Errorf("expected PacketRetention 720h, got %v", r.PacketRetention) + if r.PacketRetention != 7*24*time.Hour { + t.Errorf("expected PacketRetention 168h, got %v", r.PacketRetention) } if r.MaxConnsPerIP != 5 { t.Errorf("expected MaxConnsPerIP 5, got %d", r.MaxConnsPerIP) @@ -102,7 +152,10 @@ func TestResolve_Defaults(t *testing.T) { t.Errorf("expected NodeStaleThreshold 24h, got %v", r.NodeStaleThreshold) } if r.NodeDeleteAfter != 30*24*time.Hour { - t.Errorf("expected NodeDeleteAfter 720h (same default as PacketRetention), got %v", r.NodeDeleteAfter) + t.Errorf("expected NodeDeleteAfter 720h, got %v", r.NodeDeleteAfter) + } + if r.ObserverDeleteAfter != 0 { + t.Errorf("observer deletion must be disabled by default, got %v", r.ObserverDeleteAfter) } } @@ -115,6 +168,7 @@ func TestResolve_ExplicitValues(t *testing.T) { cfg.Background.ViewRefresh.Duration = 2 * time.Hour cfg.Background.Reconfirm.Duration = 3 * time.Hour cfg.Background.Cleanup.Duration = 4 * time.Hour + cfg.Observers.DeleteAfter.Duration = 45 * 24 * time.Hour r := Resolve(cfg) if r.TelemetryResolution != 30*time.Minute { @@ -126,6 +180,9 @@ func TestResolve_ExplicitValues(t *testing.T) { if r.ViewRefreshInterval != 2*time.Hour { t.Errorf("expected 2h, got %v", r.ViewRefreshInterval) } + if r.ObserverDeleteAfter != 45*24*time.Hour { + t.Errorf("expected observer delete_after 1080h, got %v", r.ObserverDeleteAfter) + } } func TestResolvedConfig_String(t *testing.T) { @@ -141,3 +198,106 @@ func TestResolvedConfig_String(t *testing.T) { t.Error("expected maxConnsPerIP in string") } } + +func TestResolve_RouteDefaults(t *testing.T) { + r := Resolve(&Config{}) + if r.RouteRetention != 336*time.Hour { + t.Errorf("RouteRetention = %s, want 336h", r.RouteRetention) + } + if r.RouteGrace != 168*time.Hour { + t.Errorf("RouteGrace = %s, want 168h", r.RouteGrace) + } + if r.RouteMinObservations != 3 { + t.Errorf("RouteMinObservations = %d, want 3", r.RouteMinObservations) + } +} + +func TestResolve_RollupRetentionDefaults(t *testing.T) { + r := Resolve(&Config{}) + if r.RollupRetention != 90*24*time.Hour { + t.Errorf("expected RollupRetention 2160h (90 days), got %v", r.RollupRetention) + } +} + +func TestResolve_TelemetryRetentionDefault(t *testing.T) { + r := Resolve(&Config{}) + if r.TelemetryRetention != 31*24*time.Hour { + t.Errorf("expected TelemetryRetention 744h (31 days), got %v", r.TelemetryRetention) + } +} + +func TestResolve_RollupRetentionExplicitOverride(t *testing.T) { + cfg := &Config{} + cfg.Analytics.RollupRetention.Duration = 60 * 24 * time.Hour + r := Resolve(cfg) + if r.RollupRetention != 60*24*time.Hour { + t.Errorf("expected RollupRetention 1440h (60 days), got %v", r.RollupRetention) + } +} + +func TestLoad_PacketsRetentionTooShort(t *testing.T) { + f, err := os.CreateTemp("", "beacon-config-*.yaml") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer os.Remove(f.Name()) + + _, _ = f.WriteString("packets:\n retention: 1h\n") + f.Close() + + _, err = Load(f.Name()) + if err == nil { + t.Fatal("expected error for packets.retention < 24h, got nil") + } + if !strings.Contains(err.Error(), "packets.retention must be at least 24h") { + t.Errorf("expected error about packets.retention, got: %v", err) + } +} + +func TestLoad_AnalyticsRollupRetentionTooShort(t *testing.T) { + f, err := os.CreateTemp("", "beacon-config-*.yaml") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer os.Remove(f.Name()) + + _, _ = f.WriteString("analytics:\n rollup_retention: 1h\n") + f.Close() + + _, err = Load(f.Name()) + if err == nil { + t.Fatal("expected error for analytics.rollup_retention < 24h, got nil") + } + if !strings.Contains(err.Error(), "analytics.rollup_retention must be at least 24h") { + t.Errorf("expected error about analytics.rollup_retention, got: %v", err) + } +} + +func TestLoad_NegativeDurationsRejected(t *testing.T) { + for _, key := range []string{ + "background:\n cleanup: -1h", "background:\n view_refresh: -1h", "background:\n reconfirm: -1m", + "presence:\n flush_interval: -1s", "presence:\n packet_ttl: -1s", + "telemetry:\n resolution: -1h", "telemetry:\n retention: -1h", + "routes:\n retention: -1h", "routes:\n grace: -1h", + "nodes:\n clock_drift_threshold: -1m", "nodes:\n stale_threshold: -1h", "nodes:\n delete_after: -1h", + "cache:\n ttl: -1m", "cache:\n ttls:\n stats: -1m", + "meshmapper:\n scopes:\n refresh_interval: -1h", + } { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(key+"\n"), 0600); err != nil { + t.Fatal(err) + } + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), "must not be negative") { + t.Errorf("%q: err = %v, want a negative-duration error", key, err) + } + } + // Nonpositive still means "disabled" here. + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte("observers:\n delete_after: -1h\n"), 0600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err != nil { + t.Errorf("observers.delete_after: %v", err) + } +} diff --git a/internal/config/foreign_borders.go b/internal/config/foreign_borders.go new file mode 100644 index 00000000..eb5c410c --- /dev/null +++ b/internal/config/foreign_borders.go @@ -0,0 +1,84 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "encoding/json" + "fmt" + "maps" + "os" + "slices" + + "github.com/MeshCore-Beacon/beacon-server/internal/borders" + "github.com/paulmach/orb" + "github.com/paulmach/orb/geojson" +) + +func (c *Config) hasBorderFile() bool { + for _, details := range c.IATAs { + if details.BorderFile != "" { + return true + } + } + return false +} + +// LoadLocalBorders reads the validated border files used for foreign marking, +// keyed by IATA. IATA airport coordinates are not boundaries. Nil when disabled. +func LoadLocalBorders(cfg *Config) (map[string]json.RawMessage, error) { + if !cfg.Nodes.MarkForeign { + return nil, nil + } + files := map[string]json.RawMessage{} + for code, details := range cfg.IATAs { + if details.BorderFile == "" { + continue + } + raw, err := os.ReadFile(details.BorderFile) + if err != nil { + return nil, fmt.Errorf("nodes.mark_foreign: reading border for %s: %w", code, err) + } + validated, err := ValidateBorder(raw) + if err != nil { + return nil, fmt.Errorf("nodes.mark_foreign: invalid border for %s: %w", code, err) + } + files[code] = validated + } + // Fail at startup rather than on the first rebuild. + if _, err := BuildLocalBorders(files, nil); err != nil { + return nil, err + } + return files, nil +} + +// BuildLocalBorders unions validated borders; an imported boundary replaces +// the same IATA's file. Nil when there are none. +func BuildLocalBorders(files, imported map[string]json.RawMessage) (*borders.Local, error) { + merged := maps.Clone(files) + if merged == nil { + merged = map[string]json.RawMessage{} + } + maps.Copy(merged, imported) + if len(merged) == 0 { + return nil, nil + } + var polygons []orb.Polygon + for _, code := range slices.Sorted(maps.Keys(merged)) { + feature, err := geojson.UnmarshalFeature(merged[code]) + if err != nil { + return nil, fmt.Errorf("nodes.mark_foreign: decoding border for %s: %w", code, err) + } + switch geometry := feature.Geometry.(type) { + case orb.Polygon: + polygons = append(polygons, geometry) + case orb.MultiPolygon: + polygons = append(polygons, geometry...) + } + } + local, err := borders.New(polygons) + if err != nil { + return nil, fmt.Errorf("nodes.mark_foreign: %w", err) + } + return local, nil +} diff --git a/internal/config/foreign_borders_test.go b/internal/config/foreign_borders_test.go new file mode 100644 index 00000000..e39e6b77 --- /dev/null +++ b/internal/config/foreign_borders_test.go @@ -0,0 +1,96 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestValidateBorderNoGeometry(t *testing.T) { + for _, raw := range []string{`{"type":"Feature","geometry":null}`, `{"type":"Feature"}`} { + if _, err := ValidateBorder([]byte(raw)); err == nil { + t.Fatal("missing geometry accepted") + } + } +} + +func TestLoadLocalBorders(t *testing.T) { + if got, err := LoadLocalBorders(&Config{}); err != nil || got != nil { + t.Fatalf("disabled: %v %v", got, err) + } + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + for _, tc := range []struct { + text string + bad bool + }{ + {"nodes:\n mark_foreign: true\n", true}, + {"nodes:\n mark_foreign: true\nregions:\n - slug: a\n iatas: [AAA]\nmeshmapper:\n zones:\n enabled: true\n", false}, + } { + if err := os.WriteFile(path, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); (err != nil) != tc.bad { + t.Fatalf("%q: %v", tc.text, err) + } + } + border := filepath.Join(dir, "region.json") + raw := `{"type":"Feature","geometry":{"type":"MultiPolygon","coordinates":[[[[10,10],[20,10],[20,20],[10,20],[10,10]]],[[[30,10],[40,10],[40,20],[30,20],[30,10]]]]}}` + if err := os.WriteFile(border, []byte(raw), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("nodes:\n mark_foreign: true\niatas:\n AAA:\n borderFile: region.json\n BBB:\n name: Airport without an operating border\n"), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + files, err := LoadLocalBorders(cfg) + if err != nil || len(files) != 1 { + t.Fatal(files, err) + } + local, err := BuildLocalBorders(files, nil) + if err != nil { + t.Fatal(err) + } + if *local.PossiblyForeign(2, new(15.0), new(15.0)) || *local.PossiblyForeign(2, new(15.0), new(35.0)) || !*local.PossiblyForeign(2, new(25.0), new(25.0)) { + t.Fatal("polygon union not used") + } + imported := map[string]json.RawMessage{"AAA": json.RawMessage(`{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[50,50],[60,50],[60,60],[50,60],[50,50]]]}}`)} + replaced, err := BuildLocalBorders(files, imported) + if err != nil { + t.Fatal(err) + } + if !*replaced.PossiblyForeign(2, new(15.0), new(15.0)) || *replaced.PossiblyForeign(2, new(55.0), new(55.0)) { + t.Fatal("imported boundary did not replace the file border") + } + if len(files) != 1 || string(files["AAA"]) == string(imported["AAA"]) { + t.Fatal("build changed its inputs") + } + if empty, err := BuildLocalBorders(nil, nil); err != nil || empty != nil { + t.Fatal(empty, err) + } + for _, bad := range []string{`not json`, `{"type":"Feature","geometry":null}`, `{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[179,10],[-179,10],[-179,20],[179,20],[179,10]]]}}`} { + if err := os.WriteFile(border, []byte(bad), 0600); err != nil { + t.Fatal(err) + } + if _, err := LoadLocalBorders(cfg); err == nil || !strings.Contains(err.Error(), "nodes.mark_foreign") { + t.Fatalf("bad boundary did not fail clearly: %v", err) + } + } + if err := os.Remove(border); err != nil { + t.Fatal(err) + } + if _, err := LoadLocalBorders(cfg); err == nil { + t.Fatal("missing file accepted") + } + if *local.PossiblyForeign(2, new(15.0), new(15.0)) { + t.Fatal("running classifier changed with its source file") + } +} diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go new file mode 100644 index 00000000..8f8a0c70 --- /dev/null +++ b/internal/config/meshmapper.go @@ -0,0 +1,91 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "fmt" + "log/slog" + "time" +) + +// MeshMapper rate-limits get_zones/get_geojson to one request per target per +// 23.5h and get_scopes to one per region per 55m; minimums round up. +const ( + MinZonesRefresh = 24 * time.Hour + MinScopesRefresh = time.Hour + MinChannelsRefresh = 24 * time.Hour // get_channels.php: one per region per 23.5h +) + +type MeshMapperConfig struct { + Scopes MeshMapperScopesConfig `yaml:"scopes"` + Zones MeshMapperZonesConfig `yaml:"zones"` + Channels MeshMapperChannelsConfig `yaml:"channels"` +} + +// MeshMapperChannelsConfig imports each known IATA's public hashtag channels. +type MeshMapperChannelsConfig struct { + Enabled bool `yaml:"enabled"` + RefreshInterval duration `yaml:"refresh_interval"` +} + +func (c MeshMapperChannelsConfig) Interval() time.Duration { + if c.RefreshInterval.Duration == 0 { + return MinChannelsRefresh + } + return c.RefreshInterval.Duration +} + +// MeshMapperZonesConfig imports each known IATA's published boundary, overriding borderFile. +type MeshMapperZonesConfig struct { + Enabled bool `yaml:"enabled"` + ImportGroups bool `yaml:"import_groups"` // zone groups become regions + RefreshInterval duration `yaml:"refresh_interval"` +} + +func (c MeshMapperZonesConfig) Interval() time.Duration { + if c.RefreshInterval.Duration == 0 { + return 24 * time.Hour + } + return c.RefreshInterval.Duration +} + +// MeshMapperScopesConfig augments, but never replaces, the manual scopes list. +// Sources are discovered per known IATA from the zone list. +type MeshMapperScopesConfig struct { + Enabled bool `yaml:"enabled"` + Sources map[string]string `yaml:"sources"` // deprecated: ignored, kept to warn + RefreshInterval duration `yaml:"refresh_interval"` +} + +func (c MeshMapperScopesConfig) Interval() time.Duration { + if c.RefreshInterval.Duration == 0 { + return time.Hour + } + return c.RefreshInterval.Duration +} + +func (c *Config) validateMeshMapper() error { + if z := c.MeshMapper.Zones; z.ImportGroups && !z.Enabled { + return fmt.Errorf("meshmapper.zones.import_groups requires meshmapper.zones.enabled") + } + if z := c.MeshMapper.Zones; z.Enabled { + if z.Interval() < MinZonesRefresh || z.Interval() > 7*24*time.Hour { + return fmt.Errorf("meshmapper.zones.refresh_interval must be between 24h and 168h") + } + } + if ch := c.MeshMapper.Channels; ch.Enabled && (ch.Interval() < MinChannelsRefresh || ch.Interval() > 7*24*time.Hour) { + return fmt.Errorf("meshmapper.channels.refresh_interval must be between 24h and 168h") + } + s := c.MeshMapper.Scopes + if !s.Enabled { + return nil + } + if s.Interval() < MinScopesRefresh || s.Interval() > 24*time.Hour { + return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 1h and 24h") + } + if len(s.Sources) > 0 { + slog.Warn("meshmapper.scopes.sources is ignored; sources are discovered from the MeshMapper zone list", "component", "config") + } + return nil +} diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go new file mode 100644 index 00000000..51279593 --- /dev/null +++ b/internal/config/meshmapper_test.go @@ -0,0 +1,78 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestMeshMapperConfig(t *testing.T) { + valid := "meshmapper:\n scopes:\n enabled: true\n" + for _, tc := range []struct { + name, text string + bad bool + }{ + {"valid", valid, false}, + {"daily", valid + " refresh_interval: 24h\n", false}, + {"legacy sources ignored", valid + " sources:\n yow: http://127.0.0.1/\n", false}, + {"hourly", valid + " refresh_interval: 1h\n", false}, + {"under the rate limit", valid + " refresh_interval: 55m\n", true}, + {"fast", valid + " refresh_interval: 1s\n", true}, + {"slow", valid + " refresh_interval: 25h\n", true}, + {"negative", valid + " refresh_interval: -1h\n", true}, + {"disabled", strings.Replace(valid, "enabled: true", "enabled: false", 1), false}, + } { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + _, err := Load(p) + if (err != nil) != tc.bad { + t.Fatalf("error=%v", err) + } + }) + } + if (MeshMapperScopesConfig{}).Interval() != time.Hour { + t.Fatal("wrong default") + } +} + +func TestMeshMapperZonesConfig(t *testing.T) { + valid := "regions:\n - slug: ottawa\n iatas: [YOW, YOW]\n - slug: toronto\n iatas: [YYZ]\nmeshmapper:\n zones:\n enabled: true\n" + for _, tc := range []struct { + name, text string + bad bool + interval time.Duration + }{ + {"default", valid, false, 24 * time.Hour}, + {"daily", valid + " refresh_interval: 24h\n", false, 24 * time.Hour}, + {"weekly", valid + " refresh_interval: 168h\n", false, 168 * time.Hour}, + {"under the rate limit", valid + " refresh_interval: 23h30m\n", true, 0}, + {"hourly", valid + " refresh_interval: 1h\n", true, 0}, + {"too rare", valid + " refresh_interval: 169h\n", true, 0}, + {"no regions", "meshmapper:\n zones:\n enabled: true\n", false, 24 * time.Hour}, + {"groups", valid + " import_groups: true\n", false, 24 * time.Hour}, + {"groups without zones", "meshmapper:\n zones:\n import_groups: true\n", true, 0}, + {"disabled", "meshmapper:\n zones:\n enabled: false\n", false, 24 * time.Hour}, + } { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(p) + if (err != nil) != tc.bad { + t.Fatalf("error=%v", err) + } + if err == nil && cfg.MeshMapper.Zones.Interval() != tc.interval { + t.Fatal(cfg.MeshMapper.Zones.Interval()) + } + }) + } +} diff --git a/internal/config/rate_limit_test.go b/internal/config/rate_limit_test.go new file mode 100644 index 00000000..9e4e135b --- /dev/null +++ b/internal/config/rate_limit_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRateLimitConfig(t *testing.T) { + for _, tc := range []struct { + name, yaml string + want ResolvedRateLimitConfig + wantError bool + }{ + {"omitted", "{}", ResolvedRateLimitConfig{true, 300, 300}, false}, + {"disabled", "ratelimit: {enabled: false}", ResolvedRateLimitConfig{false, 300, 300}, false}, + {"custom", "ratelimit: {enabled: true, requests_per_minute: 900, burst: 100}", ResolvedRateLimitConfig{true, 900, 100}, false}, + {"burst default follows budget", "ratelimit: {requests_per_minute: 600}", ResolvedRateLimitConfig{true, 600, 600}, false}, + {"zero defaults", "ratelimit: {requests_per_minute: 0, burst: 0}", ResolvedRateLimitConfig{true, 300, 300}, false}, + {"negative minute", "ratelimit: {requests_per_minute: -1}", ResolvedRateLimitConfig{}, true}, + {"negative burst", "ratelimit: {burst: -1}", ResolvedRateLimitConfig{}, true}, + {"invalid disabled config", "ratelimit: {enabled: false, burst: -1}", ResolvedRateLimitConfig{}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if (err != nil) != tc.wantError { + t.Fatalf("Load error = %v, want error = %v", err, tc.wantError) + } + if err == nil && Resolve(cfg).RateLimit != tc.want { + t.Fatalf("resolved limits = %+v, want %+v", Resolve(cfg).RateLimit, tc.want) + } + }) + } + cfg, err := Load(filepath.Join(t.TempDir(), "missing.yaml")) + if err != nil || Resolve(cfg).RateLimit != (ResolvedRateLimitConfig{true, 300, 300}) { + t.Fatalf("missing config did not enable defaults: %v", err) + } +} diff --git a/internal/config/scopes.go b/internal/config/scopes.go new file mode 100644 index 00000000..87c05a56 --- /dev/null +++ b/internal/config/scopes.go @@ -0,0 +1,52 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "fmt" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func (c *Config) validateScopes() error { + for i, s := range c.Scopes { + if strings.TrimSpace(s.Name) == "" { + return fmt.Errorf("scopes[%d].name is required", i) + } + if s.Region == "" { + return fmt.Errorf("scope %q needs a region", s.Name) + } + if c.region(s.Region) == nil { + return fmt.Errorf("scope %q region %q is not a configured region", s.Name, s.Region) + } + } + return nil +} + +func (c *Config) region(slug string) *RegionConfig { + for i := range c.Regions { + if c.Regions[i].Slug == slug { + return &c.Regions[i] + } + } + return nil +} + +// ManualScopeMembers maps each region IATA to the normalized names of its configured scopes. +func (c *Config) ManualScopeMembers() map[string][]string { + members := map[string][]string{} + for _, s := range c.Scopes { + region := c.region(s.Region) + if region == nil { + continue + } + name := scopestore.FromName(s.Name).Name + for _, iata := range region.IATAs { + iata = strings.ToUpper(iata) + members[iata] = append(members[iata], name) + } + } + return members +} diff --git a/internal/config/scopes_test.go b/internal/config/scopes_test.go new file mode 100644 index 00000000..6be61e4f --- /dev/null +++ b/internal/config/scopes_test.go @@ -0,0 +1,43 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestScopesConfig(t *testing.T) { + valid := "regions:\n - slug: ottawa\n iatas: [YOW, YGK]\n - slug: bc\n iatas: [YVR]\nscopes:\n - name: on\n region: ottawa\n - name: \"#west\"\n region: bc\n - name: \"$ottawa\"\n region: ottawa\n" + for _, tc := range []struct { + name, text string + bad bool + }{ + {"valid", valid, false}, + {"missing region", strings.Replace(valid, " region: bc\n", "", 1), true}, + {"unknown region", strings.Replace(valid, "region: bc", "region: prairies", 1), true}, + {"empty name", strings.Replace(valid, "name: on", "name: \"\"", 1), true}, + } { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(p) + if (err != nil) != tc.bad { + t.Fatalf("error=%v", err) + } + if tc.bad { + return + } + want := map[string][]string{"YOW": {"#on", "$ottawa"}, "YGK": {"#on", "$ottawa"}, "YVR": {"#west"}} + if got := cfg.ManualScopeMembers(); !reflect.DeepEqual(got, want) { + t.Fatalf("ManualScopeMembers = %v, want %v", got, want) + } + }) + } +} diff --git a/internal/config/seed.go b/internal/config/seed.go index 2fbbe558..e08f259b 100644 --- a/internal/config/seed.go +++ b/internal/config/seed.go @@ -5,12 +5,12 @@ package config import ( "context" - "crypto/sha256" "encoding/json" "fmt" - "log" + "log/slog" "os" - "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) // Seeder is the database interface required to seed config data on startup. @@ -19,14 +19,15 @@ type Seeder interface { UpsertIATADetails(ctx context.Context, iata string, name string, lat, lng *float64) error UpsertIATABorder(ctx context.Context, iata string, border json.RawMessage) error UpsertRegion(ctx context.Context, slug, name, description string, displayOrder int, centerLat, centerLng *float64, zoomLevel *int) (int32, error) - UpsertRegionIATA(ctx context.Context, regionID int32, iata string) error + SetRegionIATAs(ctx context.Context, regionID int32, iatas []string) error UpsertTransportScope(ctx context.Context, name, displayName string, transportKey, keyFingerprint []byte) error + SetChannelConfigScopes(ctx context.Context, fingerprints [][]byte, regions []string) error } // Seed applies config-defined regions, IATA overrides to the database. // It is safe to call on every startup — all operations are upserts. func Seed(ctx context.Context, cfg *Config, db Seeder) error { - log.Printf("config: seeding %d IATAs, %d regions, %d scopes", len(cfg.IATAs), len(cfg.Regions), len(cfg.Scopes)) + slog.Info(fmt.Sprintf("config: seeding %d IATAs, %d regions, %d scopes", len(cfg.IATAs), len(cfg.Regions), len(cfg.Scopes)), "component", "config") // IATA overrides for iata, details := range cfg.IATAs { if err := db.UpsertIATADetails(ctx, iata, details.Name, details.Lat, details.Lng); err != nil { @@ -56,36 +57,27 @@ func Seed(ctx context.Context, cfg *Config, db Seeder) error { if err := db.UpsertIATA(ctx, iata); err != nil { return err } - if err := db.UpsertRegionIATA(ctx, id, iata); err != nil { - return err - } } + // Config owns the member list, including when it takes over an imported slug. + if err := db.SetRegionIATAs(ctx, id, r.IATAs); err != nil { + return err + } + } + // Channel region placement + var fingerprints [][]byte + var regions []string + for _, scope := range cfg.ChannelScopes() { + fingerprints, regions = append(fingerprints, scope.Fingerprint), append(regions, scope.Region) + } + if err := db.SetChannelConfigScopes(ctx, fingerprints, regions); err != nil { + return err } // Transport Codes for _, s := range cfg.Scopes { - name := normalizeScopeName(s.Name) - key := deriveScopeKey(name) - h := sha256.Sum256(key) - fingerprint := h[:8] - if err := db.UpsertTransportScope(ctx, name, "", key, fingerprint); err != nil { + entry := scopestore.FromName(s.Name) + if err := db.UpsertTransportScope(ctx, entry.Name, "", entry.TransportKey, entry.KeyFingerprint); err != nil { return err } } return nil } - -// normalizeScopeName ensures the scope name has a # or $ prefix. -// Plain names get # prepended: "bc" → "#bc". -func normalizeScopeName(name string) string { - if strings.HasPrefix(name, "#") || strings.HasPrefix(name, "$") { - return name - } - return "#" + name -} - -// deriveScopeKey derives the 16-byte transport key from a normalized scope name. -// key = SHA256(name)[:16] -func deriveScopeKey(name string) []byte { - h := sha256.Sum256([]byte(name)) - return h[:16] -} diff --git a/internal/config/seed_test.go b/internal/config/seed_test.go index 4f2fa0c0..c09aa11f 100644 --- a/internal/config/seed_test.go +++ b/internal/config/seed_test.go @@ -10,15 +10,18 @@ import ( "errors" "os" "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) type stubSeeder struct { - regions map[string]int32 - iatas []string - regionIATAs map[int32][]string - scopes []string - borders map[string]json.RawMessage - upsertErr error + regions map[string]int32 + iatas []string + regionIATAs map[int32][]string + channelRegions []string + scopes []string + borders map[string]json.RawMessage + upsertErr error } func newStubSeeder() *stubSeeder { @@ -50,8 +53,13 @@ func (s *stubSeeder) UpsertRegion(_ context.Context, slug, _, _ string, _ int, _ return id, s.upsertErr } -func (s *stubSeeder) UpsertRegionIATA(_ context.Context, regionID int32, iata string) error { - s.regionIATAs[regionID] = append(s.regionIATAs[regionID], iata) +func (s *stubSeeder) SetRegionIATAs(_ context.Context, regionID int32, iatas []string) error { + s.regionIATAs[regionID] = iatas + return s.upsertErr +} + +func (s *stubSeeder) SetChannelConfigScopes(_ context.Context, fingerprints [][]byte, regions []string) error { + s.channelRegions = regions return s.upsertErr } @@ -198,39 +206,39 @@ func TestSeed_DBError(t *testing.T) { } func TestNormalizeScopeName_WithHash(t *testing.T) { - if normalizeScopeName("#bc") != "#bc" { + if scopestore.FromName("#bc").Name != "#bc" { t.Error("expected #bc unchanged") } } func TestNormalizeScopeName_WithDollar(t *testing.T) { - if normalizeScopeName("$bc") != "$bc" { + if scopestore.FromName("$bc").Name != "$bc" { t.Error("expected $bc unchanged") } } func TestNormalizeScopeName_WithoutPrefix(t *testing.T) { - if normalizeScopeName("bc") != "#bc" { + if scopestore.FromName("bc").Name != "#bc" { t.Error("expected bc to become #bc") } } func TestNormalizeScopeName_Empty(t *testing.T) { - if normalizeScopeName("") != "#" { + if scopestore.FromName("").Name != "#" { t.Error("expected empty string to become #") } } func TestDeriveScopeKey_Length(t *testing.T) { - key := deriveScopeKey("#bc") + key := scopestore.FromName("#bc").TransportKey if len(key) != 16 { t.Errorf("expected 16 bytes, got %d", len(key)) } } func TestDeriveScopeKey_Deterministic(t *testing.T) { - a := deriveScopeKey("#bc") - b := deriveScopeKey("#bc") + a := scopestore.FromName("#bc").TransportKey + b := scopestore.FromName("#bc").TransportKey if hex.EncodeToString(a) != hex.EncodeToString(b) { t.Error("expected same key for same input") } @@ -238,7 +246,7 @@ func TestDeriveScopeKey_Deterministic(t *testing.T) { func TestDeriveScopeKey_KnownValue(t *testing.T) { // SHA256("#bc")[:16] — pin the exact derivation so changes are caught - key := deriveScopeKey("#bc") + key := scopestore.FromName("#bc").TransportKey got := hex.EncodeToString(key) // generate this once: echo -n "#bc" | sha256sum | cut -c1-32 const want = "84509cfe73d94f7f6a8299e6bcdb8a3c" @@ -248,9 +256,23 @@ func TestDeriveScopeKey_KnownValue(t *testing.T) { } func TestDeriveScopeKey_DifferentInputs(t *testing.T) { - a := deriveScopeKey("#bc") - b := deriveScopeKey("#other") + a := scopestore.FromName("#bc").TransportKey + b := scopestore.FromName("#other").TransportKey if hex.EncodeToString(a) == hex.EncodeToString(b) { t.Error("expected different keys for different inputs") } } + +func TestSeed_ChannelScopes(t *testing.T) { + cfg := &Config{ + Regions: []RegionConfig{{Slug: "bc", IATAs: []string{"YVR"}}}, + ChannelKeys: ChannelKeysConfig{Hashtags: []HashtagConfig{{Name: "meshcore"}, {Name: "vancouver", Region: "bc"}}}, + } + db := newStubSeeder() + if err := Seed(context.Background(), cfg, db); err != nil { + t.Fatal(err) + } + if len(db.channelRegions) != 2 || db.channelRegions[0] != "" || db.channelRegions[1] != "bc" { + t.Fatalf("channel regions = %q", db.channelRegions) + } +} diff --git a/internal/config/ws_connect_test.go b/internal/config/ws_connect_test.go new file mode 100644 index 00000000..38463058 --- /dev/null +++ b/internal/config/ws_connect_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWebSocketConnectConfig(t *testing.T) { + for _, tc := range []struct { + name, yaml string + want int + wantError bool + }{ + {"omitted", "{}", 10, false}, + {"zero default", "websocket: {max_connects_per_minute: 0}", 10, false}, + {"custom", "websocket: {max_connects_per_minute: 30}", 30, false}, + {"negative", "websocket: {max_connects_per_minute: -1}", 0, true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if (err != nil) != tc.wantError { + t.Fatalf("Load error = %v, want error = %v", err, tc.wantError) + } + if err == nil && (Resolve(cfg).MaxConnectsPerMinute != tc.want || Resolve(cfg).MaxConnsPerIP != 5) { + t.Fatalf("unexpected resolved WebSocket limits: %+v", Resolve(cfg)) + } + }) + } +} + +func TestWebSocketAllowedOrigins(t *testing.T) { + for _, tc := range []struct { + name, origin string + wantError bool + }{ + {"https", "https://example.com", false}, + {"http with port", "http://localhost:5173", false}, + {"wildcard", "*", true}, + {"wildcard subdomain", "https://*.example.com", false}, + {"wildcard subdomain with port", "http://*.example.com:8443", false}, + {"wildcard scheme only", "https://*", true}, + {"wildcard over tld", "https://*.com", true}, + {"wildcard label prefix", "https://a*.example.com", true}, + {"wildcard label suffix", "https://*a.example.com", true}, + {"wildcard inner label", "https://x.*.example.com", true}, + {"double wildcard", "https://*.*.example.com", true}, + {"wildcard empty label", "https://*..com", true}, + {"question mark", "https://ex?mple.com", true}, + {"character class", "https://[ab].example.com", true}, + {"backslash", `https://*.example.com\`, true}, + {"no scheme", "example.com", true}, + {"other scheme", "ftp://example.com", true}, + {"path", "https://example.com/", true}, + {"query", "https://example.com?x=1", true}, + {"userinfo", "https://user@example.com", true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + yaml := "websocket: {allowed_origins: [\"" + tc.origin + "\"]}" + if err := os.WriteFile(path, []byte(yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if (err != nil) != tc.wantError { + t.Fatalf("Load error = %v, want error = %v", err, tc.wantError) + } + if err == nil && (len(cfg.WebSocket.AllowedOrigins) != 1 || cfg.WebSocket.AllowedOrigins[0] != tc.origin) { + t.Fatalf("allowed_origins = %v", cfg.WebSocket.AllowedOrigins) + } + }) + } +} diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 1f8866df..87a9ca06 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -17,8 +17,11 @@ package hub import ( "encoding/json" - "log" + "fmt" + "log/slog" "slices" + "sync/atomic" + "time" ) // EventType identifies the kind of server-push event. These match the @@ -35,29 +38,36 @@ const ( // Event is a single fan-out unit. Payload is pre-serialised JSON so the // broadcast loop never touches encoding — it's done once by the ingest path. // -// PayloadResolved is an optional second serialization carrying additional -// fields for clients that opted into them via configure (currently just -// resolvedPath on packetObservation events). Left nil for event types that -// don't have an opt-in variant; the hub falls back to Payload in that case. +// PayloadResolved/PayloadWithKey/PayloadResolvedWithKey are optional variants +// for clients that opted in via configure (resolvedPath, observerPublicKey); +// nil falls back to Payload. Repeat marks later hearings that only +// includeRepeats clients receive. type Event struct { - Type EventType - Payload json.RawMessage - PayloadResolved json.RawMessage + Type EventType + Payload json.RawMessage + PayloadResolved json.RawMessage + PayloadWithKey json.RawMessage + PayloadResolvedWithKey json.RawMessage // Routing metadata used by the hub to match subscriptions. // Populated by the ingest layer before calling Broadcast. IATA string PayloadType uint8 ChannelHash string // hex string, non-empty only for channelMessage events + RouteType uint8 // packetObservation only + ObserverID string // packetObservation and observerStatus only + // Repeat marks a later hearing of a stored observation; only IncludeRepeats clients get it. + Repeat bool } // Scope mirrors the client-side subscribe message. All fields are optional: // nil/empty means "no filter on this dimension" (match everything). -// An empty non-nil slice means "match nothing on this dimension". type Scope struct { IATAs []string PayloadTypes []uint8 + RouteTypes []uint8 ChannelHashes []string + ObserverIDs []string Events []EventType } @@ -72,11 +82,38 @@ type Client struct { subscriptions map[string]Scope // OR semantics: event matches if it matches any scope entry // ResolvePath is a connection-wide opt-in (not per-subscription), set via - // SetResolvePath ("configure" WS messages). Freely toggleable at any + // Configure ("configure" WS messages). Freely toggleable at any // point during the connection's lifetime. Only ever read/written inside // Run(), so it needs no locking despite Client being shared with the WS // goroutines. ResolvePath bool + // IncludeObserverKey works like ResolvePath, for observerPublicKey. + IncludeObserverKey bool + // IncludeRepeats works like ResolvePath, for Repeat events. + IncludeRepeats bool +} + +// ClientOptions are the connection-wide settings a "configure" message sets. +type ClientOptions struct { + ResolvePath bool + IncludeObserverKey bool + IncludeRepeats bool +} + +// payloadFor falls back to a narrower variant if the event lacks one. +func (e Event) payloadFor(c *Client) json.RawMessage { + if c.IncludeObserverKey { + if c.ResolvePath && e.PayloadResolvedWithKey != nil { + return e.PayloadResolvedWithKey + } + if !c.ResolvePath && e.PayloadWithKey != nil { + return e.PayloadWithKey + } + } + if c.ResolvePath && e.PayloadResolved != nil { + return e.PayloadResolved + } + return e.Payload } // matches returns true if the event satisfies at least one of the client's @@ -112,6 +149,12 @@ func scopeMatches(s Scope, e Event) bool { if e.Type == EventChannelMessage && len(s.ChannelHashes) > 0 && !slices.Contains(s.ChannelHashes, e.ChannelHash) { return false } + if e.Type == EventPacketObservation && len(s.RouteTypes) > 0 && !slices.Contains(s.RouteTypes, e.RouteType) { + return false + } + if (e.Type == EventPacketObservation || e.Type == EventObserverStatus) && len(s.ObserverIDs) > 0 && !slices.Contains(s.ObserverIDs, e.ObserverID) { + return false + } return true } @@ -121,10 +164,19 @@ type Hub struct { unsubscribe chan unsubscribeMsg remove chan *Client broadcast chan Event + + // observerKeyClients counts registered clients with IncludeObserverKey set. + observerKeyClients atomic.Int64 + // repeatClients counts registered clients with IncludeRepeats set. + repeatClients atomic.Int64 + // repeatDrops counts repeats refused because the broadcast channel was busy. + repeatDrops atomic.Int64 + repeatDropLogAt atomic.Int64 // unix nanos of the last drop log line + sent *sentPaths } // subscribeMsg carries a client registration, a scope subscription, or a -// configure (resolvePath toggle) request — all three go through this single +// configure request — all three go through this single // channel, not separate ones, specifically so that Go's same-channel FIFO // guarantee orders them relative to NewClient's registration message. A // separate "configure" channel raced against registration: select() has no @@ -138,7 +190,7 @@ type subscribeMsg struct { subscriptionID string isConfigure bool - resolvePath bool + options ClientOptions } type unsubscribeMsg struct { @@ -146,14 +198,6 @@ type unsubscribeMsg struct { subscriptionID string } -// configureMsg carries a connection-wide setting change, decoupled from the -// subscribe/unsubscribe scope mechanics so it can be toggled independently -// and repeatedly over the life of a connection. -type configureMsg struct { - client *Client - resolvePath bool -} - // New creates a Hub. Call Run() in a goroutine before using it. func New() *Hub { return &Hub{ @@ -161,6 +205,7 @@ func New() *Hub { unsubscribe: make(chan unsubscribeMsg, 64), remove: make(chan *Client, 64), broadcast: make(chan Event, 512), + sent: newSentPaths(sentPathsTTL, sentPathsMax), } } @@ -191,13 +236,26 @@ func (h *Hub) RemoveScope(c *Client, id string) { h.unsubscribe <- unsubscribeMsg{client: c, subscriptionID: id} } -// SetResolvePath toggles a client's opt-in to the resolvedPath variant of -// packetObservation events. Unlike scopes, this is a single connection-wide -// flag (not additive/OR'd) and can be flipped on or off at any point during -// the connection's lifetime — takes effect on the next broadcast after the -// hub processes it. -func (h *Hub) SetResolvePath(c *Client, enabled bool) { - h.subscribe <- subscribeMsg{client: c, isConfigure: true, resolvePath: enabled} +// Configure replaces a client's connection-wide options. Unlike scopes, it is +// not additive. +func (h *Hub) Configure(c *Client, opts ClientOptions) { + h.subscribe <- subscribeMsg{client: c, isConfigure: true, options: opts} +} + +// ObserverKeyWanted lets ingest skip the key variants when nobody wants them. +func (h *Hub) ObserverKeyWanted() bool { + return h.observerKeyClients.Load() > 0 +} + +// RepeatsWanted lets ingest skip broadcasting repeats when nobody wants them. +func (h *Hub) RepeatsWanted() bool { + return h.repeatClients.Load() > 0 +} + +// MarkSent records every hearing's path and reports whether it is a later, new path for a +// packet the observer already heard, so broker copies and same-path duplicates never stream. +func (h *Hub) MarkSent(packetHash, observerID, path []byte) bool { + return h.sent.mark(packetHash, observerID, path, time.Now()) } // Remove deregisters a client and closes its Send channel. @@ -211,7 +269,32 @@ func (h *Hub) Broadcast(e Event) { select { case h.broadcast <- e: default: - log.Println("hub: broadcast channel full, dropping event") + slog.Warn("hub: broadcast channel full, dropping event", "component", "hub") + } +} + +// BroadcastRepeat enqueues a repeat event, dropping it once the broadcast channel is half +// full so repeats never crowd out first hearings. +func (h *Hub) BroadcastRepeat(e Event) { + e.Repeat = true + if len(h.broadcast) >= cap(h.broadcast)/2 { + h.dropRepeat() + return + } + select { + case h.broadcast <- e: + default: + h.dropRepeat() + } +} + +// dropRepeat counts a dropped repeat and logs the running total at most once a minute. +func (h *Hub) dropRepeat() { + total := h.repeatDrops.Add(1) + now := time.Now().UnixNano() + last := h.repeatDropLogAt.Load() + if now-last >= int64(time.Minute) && h.repeatDropLogAt.CompareAndSwap(last, now) { + slog.Warn("hub: broadcast channel busy, dropping repeat events", "component", "hub", "dropped_total", total) } } @@ -235,9 +318,25 @@ func (h *Hub) Run() { // Registration with no scope yet (NewClient path). clients[msg.client] = struct{}{} case msg.isConfigure: - // SetResolvePath path — client must already be registered. + // Configure path — client must already be registered. if _, ok := clients[msg.client]; ok { - msg.client.ResolvePath = msg.resolvePath + if msg.client.IncludeObserverKey != msg.options.IncludeObserverKey { + if msg.options.IncludeObserverKey { + h.observerKeyClients.Add(1) + } else { + h.observerKeyClients.Add(-1) + } + } + if msg.client.IncludeRepeats != msg.options.IncludeRepeats { + if msg.options.IncludeRepeats { + h.repeatClients.Add(1) + } else { + h.repeatClients.Add(-1) + } + } + msg.client.ResolvePath = msg.options.ResolvePath + msg.client.IncludeObserverKey = msg.options.IncludeObserverKey + msg.client.IncludeRepeats = msg.options.IncludeRepeats } default: // AddScope path — client must already be registered. @@ -254,33 +353,43 @@ func (h *Hub) Run() { case c := <-h.remove: if _, ok := clients[c]; ok { delete(clients, c) + if c.IncludeObserverKey { + h.observerKeyClients.Add(-1) + } + if c.IncludeRepeats { + h.repeatClients.Add(-1) + } close(c.Send) close(c.laggedCH) } case evt := <-h.broadcast: for c := range clients { - if !c.matches(evt) { + if (evt.Repeat && !c.IncludeRepeats) || !c.matches(evt) { continue } outEvt := evt - if c.ResolvePath && evt.PayloadResolved != nil { - outEvt.Payload = evt.PayloadResolved - } + outEvt.Payload = evt.payloadFor(c) select { case c.Send <- outEvt: default: + // Evict the oldest so the newest still goes out. dropped := 1 select { case <-c.Send: default: } select { + case c.Send <- outEvt: + default: + dropped++ + } + select { case c.laggedCH <- LaggedNotification{DroppedCount: dropped}: default: // laggedCh itself full; write pump will catch up on next drain } - log.Printf("hub: client send buffer full, dropped event type=%s", evt.Type) + slog.Warn(fmt.Sprintf("hub: client send buffer full, dropped event type=%s", evt.Type), "component", "hub") } } } diff --git a/internal/hub/hub_test.go b/internal/hub/hub_test.go index 5ced37be..dbaaf0ef 100644 --- a/internal/hub/hub_test.go +++ b/internal/hub/hub_test.go @@ -240,6 +240,62 @@ func TestHub_Broadcast_FullBuffer_SendsLaggedNotification(t *testing.T) { } } +func TestScopeMatches_RouteTypeFilter(t *testing.T) { + s := Scope{RouteTypes: []uint8{1}} + if !scopeMatches(s, Event{Type: EventPacketObservation, RouteType: 1}) { + t.Error("expected flood observation to match") + } + if scopeMatches(s, Event{Type: EventPacketObservation, RouteType: 2}) { + t.Error("expected direct observation not to match") + } + if !scopeMatches(s, Event{Type: EventNodeUpdate}) { + t.Error("routeTypes should only filter packetObservation") + } +} + +func TestScopeMatches_ObserverFilter(t *testing.T) { + s := Scope{ObserverIDs: []string{"obs-a"}} + for _, typ := range []EventType{EventPacketObservation, EventObserverStatus} { + if !scopeMatches(s, Event{Type: typ, ObserverID: "obs-a"}) { + t.Errorf("%s from obs-a should match", typ) + } + if scopeMatches(s, Event{Type: typ, ObserverID: "obs-b"}) { + t.Errorf("%s from obs-b should not match", typ) + } + } + if !scopeMatches(s, Event{Type: EventChannelMessage}) { + t.Error("observerIds should not filter channelMessage") + } +} + +func TestHub_Broadcast_FullBuffer_KeepsNewestEvent(t *testing.T) { + h := runHub(t) + c := h.NewClient() + h.AddScope(c, "sub1", Scope{Events: []EventType{EventPacketObservation}}) + time.Sleep(10 * time.Millisecond) + + for i := 0; i < cap(c.Send); i++ { + h.Broadcast(Event{Type: EventPacketObservation, IATA: "YVR", Payload: json.RawMessage(`"old"`)}) + } + h.Broadcast(Event{Type: EventPacketObservation, IATA: "YVR", Payload: json.RawMessage(`"new"`)}) + + select { + case notif := <-c.LaggedCH(): + if notif.DroppedCount != 1 { + t.Errorf("DroppedCount = %d, want 1", notif.DroppedCount) + } + case <-time.After(200 * time.Millisecond): + t.Fatal("expected lagged notification, timed out") + } + var last Event + for len(c.Send) > 0 { + last = <-c.Send + } + if string(last.Payload) != `"new"` { + t.Errorf("newest queued event = %s, want the event that overflowed the buffer", last.Payload) + } +} + func TestClientMatches_NoSubscriptions(t *testing.T) { c := &Client{subscriptions: make(map[string]Scope)} if c.matches(Event{Type: EventPacketObservation}) { @@ -269,7 +325,7 @@ func TestHub_ResolvePath_OptedIn_GetsResolvedPayload(t *testing.T) { h := runHub(t) c := h.NewClient() h.AddScope(c, "sub1", Scope{Events: []EventType{EventPacketObservation}}) - h.SetResolvePath(c, true) + h.Configure(c, ClientOptions{ResolvePath: true}) time.Sleep(10 * time.Millisecond) @@ -294,7 +350,7 @@ func TestHub_ResolvePath_DefaultOff_GetsBasePayload(t *testing.T) { h := runHub(t) c := h.NewClient() h.AddScope(c, "sub1", Scope{Events: []EventType{EventPacketObservation}}) - // no SetResolvePath call — default is off + // no Configure call — default is off time.Sleep(10 * time.Millisecond) @@ -320,7 +376,7 @@ func TestHub_ResolvePath_OptedIn_NoResolvedVariant_FallsBackToBase(t *testing.T) c := h.NewClient() // e.g. nodeUpdate events never carry a PayloadResolved variant h.AddScope(c, "sub1", Scope{Events: []EventType{EventNodeUpdate}}) - h.SetResolvePath(c, true) + h.Configure(c, ClientOptions{ResolvePath: true}) time.Sleep(10 * time.Millisecond) @@ -367,15 +423,182 @@ func TestHub_ResolvePath_ToggleableLive(t *testing.T) { t.Errorf("expected base payload before opting in, got %s", got) } - h.SetResolvePath(c, true) + h.Configure(c, ClientOptions{ResolvePath: true}) time.Sleep(10 * time.Millisecond) if got := broadcastAndRead(); got != `{"resolvedPath":[{"confidence":"high"}]}` { t.Errorf("expected resolved payload after opting in, got %s", got) } - h.SetResolvePath(c, false) + h.Configure(c, ClientOptions{}) time.Sleep(10 * time.Millisecond) if got := broadcastAndRead(); got != `{"resolvedPath":null}` { t.Errorf("expected base payload after opting back out, got %s", got) } } + +func TestEvent_PayloadFor(t *testing.T) { + full := Event{ + Payload: json.RawMessage(`base`), + PayloadResolved: json.RawMessage(`resolved`), + PayloadWithKey: json.RawMessage(`key`), + PayloadResolvedWithKey: json.RawMessage(`resolved+key`), + } + baseOnly := Event{Payload: json.RawMessage(`base`), PayloadResolved: json.RawMessage(`resolved`)} + for _, tc := range []struct { + name string + evt Event + opts ClientOptions + want string + }{ + {"default", full, ClientOptions{}, "base"}, + {"resolve", full, ClientOptions{ResolvePath: true}, "resolved"}, + {"key", full, ClientOptions{IncludeObserverKey: true}, "key"}, + {"both", full, ClientOptions{ResolvePath: true, IncludeObserverKey: true}, "resolved+key"}, + {"key missing", baseOnly, ClientOptions{IncludeObserverKey: true}, "base"}, + {"both, key missing", baseOnly, ClientOptions{ResolvePath: true, IncludeObserverKey: true}, "resolved"}, + } { + t.Run(tc.name, func(t *testing.T) { + c := &Client{ResolvePath: tc.opts.ResolvePath, IncludeObserverKey: tc.opts.IncludeObserverKey} + if got := string(tc.evt.payloadFor(c)); got != tc.want { + t.Fatalf("payloadFor = %q, want %q", got, tc.want) + } + }) + } +} + +func TestHub_ObserverKeyWanted_TracksOptIns(t *testing.T) { + h := runHub(t) + waitFor := func(want bool) { + t.Helper() + deadline := time.Now().Add(time.Second) + for h.ObserverKeyWanted() != want { + if time.Now().After(deadline) { + t.Fatalf("ObserverKeyWanted = %t, want %t", !want, want) + } + time.Sleep(time.Millisecond) + } + } + a, b := h.NewClient(), h.NewClient() + if h.ObserverKeyWanted() { + t.Fatal("no client opted in yet") + } + h.Configure(a, ClientOptions{IncludeObserverKey: true}) + h.Configure(a, ClientOptions{IncludeObserverKey: true}) // repeat must not double count + h.Configure(b, ClientOptions{IncludeObserverKey: true}) + waitFor(true) + h.Configure(a, ClientOptions{ResolvePath: true}) + h.Remove(b) + waitFor(false) +} + +func TestHub_Repeat_OnlyReachesOptedInClients(t *testing.T) { + h := runHub(t) + plain, opted := h.NewClient(), h.NewClient() + h.AddScope(plain, "all", Scope{}) + h.AddScope(opted, "all", Scope{}) + h.Configure(opted, ClientOptions{IncludeRepeats: true}) + time.Sleep(10 * time.Millisecond) + + read := func(c *Client) (Event, bool) { + select { + case evt := <-c.Send: + return evt, true + case <-time.After(50 * time.Millisecond): + return Event{}, false + } + } + h.BroadcastRepeat(Event{Type: EventPacketObservation}) + if evt, ok := read(opted); !ok || !evt.Repeat { + t.Fatalf("opted-in client: got %+v, %t", evt, ok) + } + if evt, ok := read(plain); ok { + t.Fatalf("plain client got repeat %+v", evt) + } + h.Broadcast(Event{Type: EventPacketObservation}) + for _, c := range []*Client{plain, opted} { + if evt, ok := read(c); !ok || evt.Repeat { + t.Fatalf("normal event: got %+v, %t", evt, ok) + } + } +} + +func TestHub_RepeatsWanted_TracksOptIns(t *testing.T) { + h := runHub(t) + waitFor := func(want bool) { + t.Helper() + deadline := time.Now().Add(time.Second) + for h.RepeatsWanted() != want { + if time.Now().After(deadline) { + t.Fatalf("RepeatsWanted = %t, want %t", !want, want) + } + time.Sleep(time.Millisecond) + } + } + a, b := h.NewClient(), h.NewClient() + if h.RepeatsWanted() { + t.Fatal("no client opted in yet") + } + h.Configure(a, ClientOptions{IncludeRepeats: true}) + h.Configure(a, ClientOptions{IncludeRepeats: true}) // repeat must not double count + h.Configure(b, ClientOptions{IncludeRepeats: true}) + waitFor(true) + h.Configure(a, ClientOptions{ResolvePath: true}) + h.Remove(b) + waitFor(false) +} + +func TestHub_BroadcastRepeat_DropsFirstWhenBusy(t *testing.T) { + h := New() // not running, so the broadcast channel only fills + h.BroadcastRepeat(Event{Type: EventPacketObservation}) + if len(h.broadcast) != 1 || h.repeatDrops.Load() != 0 { + t.Fatalf("repeat on an idle hub: queued %d, dropped %d", len(h.broadcast), h.repeatDrops.Load()) + } + for len(h.broadcast) < cap(h.broadcast)/2 { + h.Broadcast(Event{Type: EventPacketObservation}) + } + queued := len(h.broadcast) + h.BroadcastRepeat(Event{Type: EventPacketObservation}) + h.BroadcastRepeat(Event{Type: EventPacketObservation}) + if len(h.broadcast) != queued || h.repeatDrops.Load() != 2 { + t.Fatalf("repeats past half full: queued %d (want %d), dropped %d", len(h.broadcast), queued, h.repeatDrops.Load()) + } + h.Broadcast(Event{Type: EventPacketObservation}) + if len(h.broadcast) != queued+1 { + t.Fatal("normal event not enqueued past half full") + } +} + +func TestSentPaths(t *testing.T) { + start := time.Now() + s := newSentPaths(time.Minute, 3) + hash, observer := []byte{1, 2}, []byte{3} + if s.mark(hash, observer, []byte{0xaa}, start) { + t.Fatal("first hearing counted as a repeat") + } + if s.mark(hash, observer, []byte{0xaa}, start.Add(time.Second)) { + t.Fatal("broker copy of the first hearing counted as a repeat") + } + if !s.mark(hash, observer, []byte{0xaa, 0xbb}, start.Add(time.Second)) { + t.Fatal("new path refused") + } + if s.mark(hash, observer, []byte{0xaa, 0xbb}, start.Add(time.Second)) { + t.Fatal("exact copy within the window allowed") + } + if s.mark(hash, []byte{4}, []byte{0xaa}, start.Add(time.Second)) { + t.Fatal("other observer's first hearing counted as a repeat") + } + if s.mark(hash, observer, []byte{0xcc}, start.Add(time.Minute)) { + t.Fatal("hearing after the window not treated as a first hearing") + } + + s = newSentPaths(time.Hour, 3) + for i := range 4 { + s.mark(hash, observer, []byte{byte(i)}, start) + } + if len(s.at) != 3 || len(s.order) != 3 { + t.Fatalf("size bound: %d keys, %d queued", len(s.at), len(s.order)) + } + if _, ok := s.at[s.key(hash, observer, []byte{0})]; ok { + t.Fatal("oldest key not evicted at the size bound") + } +} diff --git a/internal/hub/sent.go b/internal/hub/sent.go new file mode 100644 index 00000000..f6ae3ed9 --- /dev/null +++ b/internal/hub/sent.go @@ -0,0 +1,70 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package hub + +import ( + "hash/maphash" + "sync" + "time" +) + +const ( + sentPathsTTL = 2 * time.Minute + sentPathsMax = 200_000 +) + +// sentPaths remembers recently heard (packet, observer, path) hearings. It is shared by all +// broker workers; keys are hashed so memory stays small at the size bound. +type sentPaths struct { + mu sync.Mutex + seed maphash.Seed + ttl time.Duration + max int + at map[uint64]time.Time + order []sentPath // oldest first +} + +type sentPath struct { + key uint64 + at time.Time +} + +func newSentPaths(ttl time.Duration, max int) *sentPaths { + return &sentPaths{seed: maphash.MakeSeed(), ttl: ttl, max: max, at: make(map[uint64]time.Time)} +} + +// mark records the hearing and reports whether it is a new path for a (packet, observer) already +// heard over another one. Keying the first hearing by pair, not by which worker won the insert, +// means a broker copy of it is never a repeat however the two workers interleave. +func (s *sentPaths) mark(packetHash, observerID, path []byte, now time.Time) bool { + pair, hearing := s.key(packetHash, observerID), s.key(packetHash, observerID, path) + s.mu.Lock() + defer s.mu.Unlock() + firstHearing := s.add(pair, now) + return s.add(hearing, now) && !firstHearing +} + +func (s *sentPaths) key(parts ...[]byte) uint64 { + var h maphash.Hash + h.SetSeed(s.seed) + for _, part := range parts { + h.WriteByte(byte(len(part))) + h.Write(part) + } + return h.Sum64() +} + +// add records key unless already present within the TTL and reports whether it was new. +func (s *sentPaths) add(key uint64, now time.Time) bool { + for len(s.order) > 0 && (now.Sub(s.order[0].at) >= s.ttl || len(s.order) >= s.max) { + delete(s.at, s.order[0].key) + s.order = s.order[1:] + } + if _, ok := s.at[key]; ok { + return false + } + s.at[key] = now + s.order = append(s.order, sentPath{key, now}) + return true +} diff --git a/internal/iatadb/gen/airports.csv b/internal/iatadb/gen/airports.csv index 561e33a3..2070812f 100644 --- a/internal/iatadb/gen/airports.csv +++ b/internal/iatadb/gen/airports.csv @@ -29945,7 +29945,7 @@ 599256,"GB-1333","heliport","Clifton Dykes",54.644211,-2.69254,,"EU","GB","GB-U-A",,"no",,,,,,, 599308,"GB-1334","closed","RAF Long Kesh",54.490669,-6.103835,115,"EU","GB","GB-NIR","Maze, County Down","no",,,,,,"https://en.wikipedia.org/wiki/RAF_Long_Kesh", 599309,"GB-1335","heliport","Lisburn Heliport",54.49328,-6.097745,118,"EU","GB","GB-NIR","Lisburn, County Down","no",,,,,,, -599921,"GB-1336","small_airport","(Duplicete)East Kirkby Airfield",53.13889,-0.00056,39,"EU","GB","GB-ENG","East Kirkby","no",,,,,"https://lincsaviation.co.uk","https://en.m.wikipedia.org/wiki/Lincolnshire_Aviation_Heritage_Centre?utm_source=chatgpt.com","RAF East Kirkby; Lincolnshire Aviation Heritage Centre; Bomber Command; Avro Lancaster NX611 “Just Jane”; Mosquito HJ711; East Lindsey; WW2 airfield; museum; heritage." +599921,"GB-1336","small_airport","(Duplicete)East Kirkby Airfield",53.13889,-0.00056,39,"EU","GB","GB-ENG","East Kirkby","no",,,,,"https://lincsaviation.co.uk","https://en.m.wikipedia.org/wiki/Lincolnshire_Aviation_Heritage_Centre","RAF East Kirkby; Lincolnshire Aviation Heritage Centre; Bomber Command; Avro Lancaster NX611 “Just Jane”; Mosquito HJ711; East Lindsey; WW2 airfield; museum; heritage." 599951,"GB-1337","closed","RAF Woodhall Spa",53.131667,-0.18444,33,"EU","GB","GB-ENG","East Linsdey, Lincolnshire","no",,,,,"https://www.lincstrust.org.uk/get-involved/top-reserves/woodhall-spa-airfield","https://en.wikipedia.org/wiki/RAF_Woodhall_Spa","Tattershall Thorpe, No. 617 Squadron, Dambusters, Bristol Bloodhound Missile site, Thorpe Camp Visitor Centre, Lincolnshire Wildlife Trust, RAF Coningsby satellite" 600636,"GB-1338","heliport","Chequers Helicopter Landing Ground",51.742163,-0.781822,,"EU","GB","GB-ENG","Ellesborough, Bucks.","no",,,,,,, 602208,"GB-1339","heliport","Uxbridge Business Park Heliport",51.554853,-0.483221,112,"EU","GB","GB-ENG","Uxbridge, Greater London","no",,,,,,, @@ -30972,7 +30972,7 @@ 30801,"HRYI","small_airport","Butare Airport",-2.59583,29.7367,5801,"AF","RW","RW-05","Butare","no","HRYI","BTQ","HRYI",,,"https://en.wikipedia.org/wiki/Butare_Airport", 31612,"HRYN","small_airport","Nemba Airport",-2.336167,30.206658,4905,"AF","RW","RW-02","Nemba","no","HRYN",,"HRYN",,,"https://en.wikipedia.org/wiki/Nemba_Airport", 31613,"HRYO","small_airport","Gabiro Airport",-1.5499999523162842,30.382999420166016,4806,"AF","RW","RW-02","Gabiro","no",,,"HRYO",,,, -3237,"HRYR","large_airport","Kigali International Airport",-1.96863,30.1395,4859,"AF","RW","RW-01","Kigali","yes","HRYR","KGL","HRYR",,"https://www.rac.co.rw/?utm_source=chatgpt.com","https://en.wikipedia.org/wiki/Kigali_International_Airport","Gregoire Kayibanda International Airport, Kanombe International Airport" +3237,"HRYR","large_airport","Kigali International Airport",-1.96863,30.1395,4859,"AF","RW","RW-01","Kigali","yes","HRYR","KGL","HRYR",,"https://www.rac.co.rw/","https://en.wikipedia.org/wiki/Kigali_International_Airport","Gregoire Kayibanda International Airport, Kanombe International Airport" 32208,"HRYU","small_airport","Ruhengeri Airport",-1.496873,29.631343,6102,"AF","RW","RW-03","Ruhengeri","no","HRYU","RHG","HRYU",,,"https://en.wikipedia.org/wiki/Ruhengeri_Airport", 3238,"HRZA","medium_airport","Kamembe Airport",-2.462239980697632,28.907899856567383,5192,"AF","RW","RW-04","Kamembe","yes","HRZA","KME","HRZA",,,"https://en.wikipedia.org/wiki/Kamembe_Airport", 43976,"HSAK","small_airport","Akobo Airport",7.7779,33.0027,1352,"AF","SS","SS-JG","Akobo","no",,,"HJAK",,,"https://en.wikipedia.org/wiki/Akobo_Airport","HSAK, Ahobo" diff --git a/internal/ingest/advert_endpoint_update_test.go b/internal/ingest/advert_endpoint_update_test.go new file mode 100644 index 00000000..a0899522 --- /dev/null +++ b/internal/ingest/advert_endpoint_update_test.go @@ -0,0 +1,144 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +type advertEndpointDB struct { + *endpointCaptureDB + found, duplicate bool + lookups, upserts int +} + +func (s *advertEndpointDB) GetNodeByPubkey(context.Context, []byte) (uuid.UUID, error) { + if !s.found { + return uuid.Nil, errors.New("node not found") + } + return s.node.ID, nil +} + +func (s *advertEndpointDB) GetNodesByIDs(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]*api.ResolvedNode, error) { + s.lookups++ + return s.endpointCaptureDB.GetNodesByIDs(ctx, ids) +} + +func (s *advertEndpointDB) UpsertNode(_ context.Context, n UpsertNodeParams, _ RadioSettings) (uuid.UUID, error) { + s.node.Name = &n.Name + s.found = true + s.upserts++ + return s.node.ID, nil +} + +func (s *advertEndpointDB) InsertObservation(context.Context, InsertObservationParams) (bool, int64, error) { + return !s.duplicate, 1, nil +} + +func TestAdvertEndpointUsesUpdatedName(t *testing.T) { + for _, existing := range []bool{false, true} { + t.Run(map[bool]string{false: "first advert", true: "renamed node"}[existing], func(t *testing.T) { + w, base := newTestWorker() + old := "Previous repeater" + store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: base, node: api.ResolvedNode{ID: uuid.New(), Name: &old, PublicKey: "aa"}}, found: existing} + w.db = store + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + w.hub.AddScope(client, "rename", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + defer w.hub.Remove(client) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := buildAdvertPacketWithData(t, append([]byte{meshcore.AdvertTypeRepeater | meshcore.AdvertNameMask}, []byte("Renamed repeater")...), false) + w.handlePacket(ctx, "YYZ", "0102", packetEnvelope(t, packet)) + for { + select { + case event := <-client.Send: + if event.Type != hub.EventPacketObservation { + continue + } + var got packetObservationEvent + if err := json.Unmarshal(event.Payload, &got); err != nil { + t.Fatal(err) + } + hop := got.Observation.ResolvedSource + if hop == nil || len(hop.Nodes) != 1 || hop.Nodes[0].Name == nil || *hop.Nodes[0].Name != "Renamed repeater" { + t.Fatalf("live source did not use the newly saved name: %+v", hop) + } + if store.upserts != 1 || hop.Nodes[0].ID != store.node.ID || hop.Confidence != "high" { + t.Fatal("advert identity or confidence changed") + } + return + case <-ctx.Done(): + t.Fatal("packet event missing") + } + } + }) + } +} + +func TestDuplicateAdvertSkipsLiveEndpointLookup(t *testing.T) { + w, base := newTestWorker() + store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: base, node: api.ResolvedNode{ID: uuid.New(), PublicKey: "aa"}}, found: true, duplicate: true} + w.db = store + w.handlePacket(context.Background(), "YYZ", "0102", packetEnvelope(t, buildAdvertPacket(t, false))) + if store.lookups != 0 || store.upserts != 0 { + t.Fatalf("duplicate observation performed live-only work: lookups=%d upserts=%d", store.lookups, store.upserts) + } +} + +func TestRepeatAdvertUsesCurrentIdentityWithoutReapplyingAdvert(t *testing.T) { + r := newRepeatHarness(t, true) + store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: r.db, node: api.ResolvedNode{ID: uuid.New(), PublicKey: "aa"}}} + r.w.db = store + packet := buildAdvertPacketWithData(t, append([]byte{meshcore.AdvertTypeRepeater | meshcore.AdvertNameMask}, []byte("Advert name")...), false) + hear := func(path byte) []packetObservationEvent { + packet.Path, packet.PathLength = []byte{path}, 1 + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(t, packet)) + r.w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + var events []packetObservationEvent + for { + select { + case event := <-r.client.Send: + if event.Type == hub.EventObserverStatus { + return events + } + var payload packetObservationEvent + if err := json.Unmarshal(event.Payload, &payload); err != nil { + t.Fatal(err) + } + events = append(events, payload) + case <-r.ctx.Done(): + t.Fatal("repeat marker missing") + } + } + } + first := hear(0x11) + if len(first) != 1 || first[0].Packet.IsRepeat || store.upserts != 1 { + t.Fatal("first advert did not produce one stored hearing") + } + // A later advert has updated the identity before this older packet is heard again. + current := "Current name" + store.node.Name, store.duplicate = ¤t, true + repeated := hear(0x22) + if len(repeated) != 1 || !repeated[0].Packet.IsRepeat || repeated[0].Observation.ResolvedSource == nil { + t.Fatal("new-path repeat lost its resolved source") + } + if got := repeated[0].Observation.ResolvedSource.Nodes[0].Name; got == nil || *got != current || store.upserts != 1 { + t.Fatal("repeat reapplied the old advert or returned a stale identity") + } + lookups := store.lookups + if got := hear(0x22); len(got) != 0 || store.lookups != lookups || store.upserts != 1 { + t.Fatal("suppressed repeat performed live-only work") + } +} diff --git a/internal/ingest/backfill.go b/internal/ingest/backfill.go index c0469129..65402ec7 100644 --- a/internal/ingest/backfill.go +++ b/internal/ingest/backfill.go @@ -7,7 +7,7 @@ import ( "context" "encoding/hex" "fmt" - "log" + "log/slog" "strings" "time" @@ -29,9 +29,8 @@ type DecryptGroupTextResult struct { ChannelID int ChannelHash []byte Entry keystore.Entry - // NewMessage is false if InsertChannelMessage found the message already existed - // (packet_hash is unique per message) -- e.g. re-running the backfill twice. - NewMessage bool + // Message is nil when the packet already has a stored channel message. + Message *InsertedChannelMessage } // DecryptGroupText attempts to decrypt a GRP_TXT payload against keys and, on success, @@ -42,7 +41,7 @@ type DecryptGroupTextResult struct { // decides what to do with that. The live path falls back to UpsertChannelHashOnly itself; // BackfillChannelMessages, which only scans packets already recorded that way, just leaves // them as they are and tries again next boot. -func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHash, rawPayload []byte) (*DecryptGroupTextResult, error) { +func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHash, rawPayload []byte, historical bool) (*DecryptGroupTextResult, error) { grpTxt, err := meshcore.GroupTextFromBytes(rawPayload) if err != nil { return nil, err @@ -71,17 +70,18 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa PacketHash: packetHash, SenderName: strings.ReplaceAll(strings.ToValidUTF8(payload.Sender, "\uFFFD"), "\x00", ""), SentAt: time.Unix(int64(payload.Timestamp), 0), + Historical: historical, Content: strings.ReplaceAll(strings.ToValidUTF8(payload.Text, "\uFFFD"), "\x00", ""), } newMsg, err := db.InsertChannelMessage(ctx, params) if err != nil { return nil, fmt.Errorf("insert channel message: %w", err) } - if newMsg { + if newMsg != nil { // Non-fatal: the message is stored either way, so just log and continue -- matches // the live ingest path's existing behavior of not treating this as a hard failure. if err := db.SetPacketDecrypted(ctx, packetHash); err != nil { - log.Printf("ingest: failed to set packet decrypted for %s: %v", hex.EncodeToString(packetHash), err) + slog.Error(fmt.Sprintf("ingest: failed to set packet decrypted for %s", hex.EncodeToString(packetHash)), "component", "ingest", "error", err) } } return &DecryptGroupTextResult{ @@ -89,7 +89,7 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa ChannelID: channelID, ChannelHash: channelHashBytes, Entry: usedEntry, - NewMessage: newMsg, + Message: newMsg, }, nil } @@ -108,16 +108,29 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) ( if err != nil { return 0, err } + return backfill(ctx, db, keys, packets), nil +} + +// BackfillChannelHashes retries only the channels whose keys were just imported at runtime. +func BackfillChannelHashes(ctx context.Context, db DB, keys ChannelKeyStore, hashes [][]byte) (int, error) { + packets, err := db.ListUndecryptedGroupTextPacketsByHash(ctx, hashes) + if err != nil { + return 0, err + } + return backfill(ctx, db, keys, packets), nil +} + +func backfill(ctx context.Context, db DB, keys ChannelKeyStore, packets []UndecryptedPacket) int { decrypted := 0 for _, p := range packets { - result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload) + result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload, true) if err != nil { - log.Printf("ingest: backfill: decrypt failed for packet %s: %v", hex.EncodeToString(p.PacketHash), err) + slog.Error(fmt.Sprintf("ingest: backfill: decrypt failed for packet %s", hex.EncodeToString(p.PacketHash)), "component", "ingest", "error", err) continue } - if result != nil && result.NewMessage { + if result != nil && result.Message != nil { decrypted++ } } - return decrypted, nil + return decrypted } diff --git a/internal/ingest/backfill_test.go b/internal/ingest/backfill_test.go index 748332ee..7088b2bb 100644 --- a/internal/ingest/backfill_test.go +++ b/internal/ingest/backfill_test.go @@ -39,7 +39,7 @@ func TestDecryptGroupText_Success(t *testing.T) { }} raw := encryptedGroupText(t, channelHash, psk, "ded", "hello") - result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x01}, raw) + result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x01}, raw, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -49,8 +49,8 @@ func TestDecryptGroupText_Success(t *testing.T) { if result.Payload.Sender != "ded" || result.Payload.Text != "hello" { t.Errorf("expected decrypted sender=ded text=hello, got sender=%s text=%s", result.Payload.Sender, result.Payload.Text) } - if !result.NewMessage { - t.Error("expected NewMessage true when InsertChannelMessage reports a new insert") + if result.Message == nil { + t.Error("expected a message when InsertChannelMessage reports a new insert") } if db.upsertChannelCalls != 1 { t.Errorf("expected UpsertChannel to be called once, got %d", db.upsertChannelCalls) @@ -64,7 +64,7 @@ func TestDecryptGroupText_NoMatchingKey(t *testing.T) { raw := encryptedGroupText(t, channelHash, make([]byte, 16), "ded", "hello") keys := &mapKeys{entries: map[byte][]keystore.Entry{}} - result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x02}, raw) + result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x02}, raw, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -88,7 +88,7 @@ func TestDecryptGroupText_WrongKeyForHash(t *testing.T) { channelHash: {{Key: wrongKey, Fingerprint: []byte{0xBB}}}, }} - result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x03}, raw) + result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x03}, raw, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -100,7 +100,7 @@ func TestDecryptGroupText_WrongKeyForHash(t *testing.T) { func TestDecryptGroupText_MalformedPayload(t *testing.T) { db := &stubDB{} keys := &mapKeys{} - _, err := DecryptGroupText(context.Background(), db, keys, []byte{0x04}, []byte{}) + _, err := DecryptGroupText(context.Background(), db, keys, []byte{0x04}, []byte{}, false) if err == nil { t.Fatal("expected an error for an empty/malformed payload") } @@ -135,6 +135,23 @@ func TestBackfillChannelMessages_DecryptsAndCounts(t *testing.T) { } } +func TestBackfillChannelHashes_OnlyScansNewHashes(t *testing.T) { + psk := make([]byte, 16) + db := &stubDB{ + insertChannelMessageResult: true, + undecryptedByHash: []UndecryptedPacket{{PacketHash: []byte{0x01}, RawPayload: encryptedGroupText(t, 0x44, psk, "ded", "hi")}}, + undecryptedPackets: []UndecryptedPacket{{PacketHash: []byte{0x02}, RawPayload: encryptedGroupText(t, 0x44, psk, "x", "y")}}, + } + keys := &mapKeys{entries: map[byte][]keystore.Entry{0x44: {{Key: psk, Fingerprint: []byte{0xAA}}}}} + n, err := BackfillChannelHashes(context.Background(), db, keys, [][]byte{{0x44}}) + if err != nil || n != 1 { + t.Fatalf("n=%d err=%v", n, err) + } + if len(db.backfillHashes) != 1 || db.backfillHashes[0][0] != 0x44 { + t.Fatalf("scanned hashes %x", db.backfillHashes) + } +} + func TestBackfillChannelMessages_NoUndecryptedPackets(t *testing.T) { db := &stubDB{} keys := &mapKeys{} @@ -146,3 +163,24 @@ func TestBackfillChannelMessages_NoUndecryptedPackets(t *testing.T) { t.Errorf("expected 0, got %d", n) } } + +func TestBackfillMarksMessagesHistorical(t *testing.T) { + psk := make([]byte, 16) + db := &stubDB{ + insertChannelMessageResult: true, + undecryptedPackets: []UndecryptedPacket{{PacketHash: []byte{0x01}, RawPayload: encryptedGroupText(t, 0x44, psk, "ded", "hi")}}, + } + keys := &mapKeys{entries: map[byte][]keystore.Entry{0x44: {{Key: psk, Fingerprint: []byte{0xAA}}}}} + if n, err := BackfillChannelMessages(context.Background(), db, keys); err != nil || n != 1 { + t.Fatalf("n=%d err=%v", n, err) + } + if len(db.channelMessages) != 1 || !db.channelMessages[0].Historical { + t.Errorf("backfilled messages %+v; want Historical so their rolled hours are re-rolled", db.channelMessages) + } + if _, err := DecryptGroupText(context.Background(), db, keys, []byte{0x02}, encryptedGroupText(t, 0x44, psk, "x", "live"), false); err != nil { + t.Fatal(err) + } + if db.channelMessages[1].Historical { + t.Error("live messages must not queue re-rolls") + } +} diff --git a/internal/ingest/capability.go b/internal/ingest/capability.go index 240c3814..22f7ce56 100644 --- a/internal/ingest/capability.go +++ b/internal/ingest/capability.go @@ -5,28 +5,47 @@ package ingest import ( "context" + "sync" "github.com/google/uuid" ) -// runCapabilityDetection checks hash sizes and flips firmware capability flags. -// Called only when the observation INSERT succeeded (no dedup conflict). -// -// Rules (from design doc): -// - hash_size == 1: do nothing (proves nothing about firmware) -// - duplicate hash prefixes within the path: skip entirely -// - non-trace + hash_size 2 or 3 → supports_multibyte_paths = TRUE -// - trace (0x09) + hash_size 2 or 4 → supports_multibyte_traces = TRUE +const capabilityCacheLimit = 4096 + +type capabilityCache struct { + mu sync.Mutex + nodes map[uuid.UUID]uint8 +} + func (w *Worker) runCapabilityDetection(ctx context.Context, payloadType uint8, hashSize uint8, resolvedNodeIDs []uuid.UUID) { - if hashSize < 2 { + var flag uint8 + switch { + case payloadType != 0x09 && (hashSize == 2 || hashSize == 3): + flag = 1 + case payloadType == 0x09 && (hashSize == 2 || hashSize == 4): + flag = 2 + default: return } for _, nodeID := range resolvedNodeIDs { - switch { - case payloadType != 0x09 && (hashSize == 2 || hashSize == 3): - _ = w.db.SetNodeCapability(ctx, nodeID, true, false) - case payloadType == 0x09 && (hashSize == 2 || hashSize == 4): - _ = w.db.SetNodeCapability(ctx, nodeID, false, true) + w.capabilities.mu.Lock() + recorded := w.capabilities.nodes[nodeID]&flag != 0 + w.capabilities.mu.Unlock() + if recorded { + continue + } + if err := w.db.SetNodeCapability(ctx, nodeID, flag == 1, flag == 2); err != nil { + continue + } + w.capabilities.mu.Lock() + if w.capabilities.nodes == nil { + w.capabilities.nodes = make(map[uuid.UUID]uint8) + } + // Eviction only costs another write; capabilities never downgrade. + if len(w.capabilities.nodes) >= capabilityCacheLimit { + clear(w.capabilities.nodes) } + w.capabilities.nodes[nodeID] |= flag + w.capabilities.mu.Unlock() } } diff --git a/internal/ingest/capability_test.go b/internal/ingest/capability_test.go new file mode 100644 index 00000000..2ebfb42d --- /dev/null +++ b/internal/ingest/capability_test.go @@ -0,0 +1,72 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "errors" + "testing" + + "github.com/google/uuid" +) + +func TestCapabilityAlreadyRecorded(t *testing.T) { + w, db := newTestWorker() + id := uuid.New() + for range 3 { + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{id, id}) + } + if got := len(db.setCapabilityCalls); got != 1 { + t.Fatalf("recorded the same capability %d times, want 1", got) + } + w.runCapabilityDetection(context.Background(), 9, 2, []uuid.UUID{id}) + w.runCapabilityDetection(context.Background(), 9, 2, []uuid.UUID{id}) + if got := len(db.setCapabilityCalls); got != 2 || !db.setCapabilityCalls[1].traces { + t.Fatalf("trace capability was not recorded separately: %+v", db.setCapabilityCalls) + } +} + +type failingCapabilityDB struct { + *stubDB + fail bool +} + +func (d *failingCapabilityDB) SetNodeCapability(ctx context.Context, id uuid.UUID, paths, traces bool) error { + _ = d.stubDB.SetNodeCapability(ctx, id, paths, traces) + if d.fail { + return errors.New("write failed") + } + return nil +} + +func TestCapabilityRetriesFailedWrite(t *testing.T) { + w, base := newTestWorker() + db := &failingCapabilityDB{stubDB: base, fail: true} + w.db = db + id := uuid.New() + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{id}) + db.fail = false + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{id}) + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{id}) + if got := len(db.setCapabilityCalls); got != 2 { + t.Fatalf("got %d writes, want the failed write and one successful retry", got) + } +} + +func TestCapabilityCacheEviction(t *testing.T) { + w, db := newTestWorker() + first := uuid.New() + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{first}) + for range capabilityCacheLimit { + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{uuid.New()}) + } + if len(w.capabilities.nodes) > capabilityCacheLimit { + t.Fatal("cache grew beyond limit") + } + before := len(db.setCapabilityCalls) + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{first}) + if len(db.setCapabilityCalls) != before+1 { + t.Fatal("evicted capability was not recorded again") + } +} diff --git a/internal/ingest/channel_scope_test.go b/internal/ingest/channel_scope_test.go new file mode 100644 index 00000000..3ee69334 --- /dev/null +++ b/internal/ingest/channel_scope_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "github.com/meshcore-go/meshcore-go" +) + +type scopeMessageDB struct { + *stubDB + next *InsertedChannelMessage +} + +func (s *scopeMessageDB) InsertChannelMessage(context.Context, InsertChannelMessageParams) (*InsertedChannelMessage, error) { + value := s.next + s.next = nil + return value, nil +} + +func TestChannelMessageScopeLive(t *testing.T) { + for _, status := range []string{"matched", "unscoped", "unknown", "unavailable"} { + t.Run(status, func(t *testing.T) { + w, base := newTestWorker() + stored := &InsertedChannelMessage{ID: 42, ScopeStatus: api.ChannelScopeStatus(status)} + name := "#stored" + if status == "matched" { + stored.Scope = &name + } + w.db = &scopeMessageDB{stubDB: base, next: stored} + psk := make([]byte, 16) + w.keys = &mapKeys{entries: map[byte][]keystore.Entry{0x11: {{Key: psk, Fingerprint: []byte{1}, Name: "fixture"}}}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "scope", hub.Scope{Events: []hub.EventType{hub.EventChannelMessage, hub.EventObserverStatus}}) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeTransportFlood, meshcore.PayloadTypeGrpTxt, 0), Payload: encryptedGroupText(t, 0x11, psk, "Fixture", "test")} + latest := "#different-later-reception" + w.handlePayloadTypeSideEffects(ctx, packet, "YOW", []byte{1}, RadioSettings{}, nil, &latest, nil, 0) + select { + case event := <-client.Send: + var got channelMessageEvent + if err := json.Unmarshal(event.Payload, &got); err != nil { + t.Fatal(err) + } + if got.ID != 42 || got.ScopeStatus != stored.ScopeStatus || (got.Scope == nil) != (stored.Scope == nil) || (got.Scope != nil && *got.Scope != name) { + t.Fatal("live scope did not use stored evidence", got) + } + case <-ctx.Done(): + t.Fatal("no live message") + } + w.handlePayloadTypeSideEffects(ctx, packet, "YOW", []byte{1}, RadioSettings{}, nil, &latest, nil, 0) + w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + select { + case event := <-client.Send: + if event.Type == hub.EventChannelMessage { + t.Fatal("duplicate broadcast") + } + case <-ctx.Done(): + t.Fatal("no barrier") + } + }) + } +} diff --git a/internal/ingest/endpoint_matching_test.go b/internal/ingest/endpoint_matching_test.go new file mode 100644 index 00000000..52d12195 --- /dev/null +++ b/internal/ingest/endpoint_matching_test.go @@ -0,0 +1,51 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/hex" + "reflect" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/meshcore-go/meshcore-go" +) + +type endpointRoutingDB struct { + *stubDB + endpoints []string + paths []string +} + +func (s *endpointRoutingDB) ResolveEndpointHashes(_ context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + for _, hash := range hashes { + s.endpoints = append(s.endpoints, iata+":"+hex.EncodeToString(hash)) + } + return nil, nil +} +func (s *endpointRoutingDB) ResolvePathHashes(_ context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + for _, hash := range hashes { + s.paths = append(s.paths, iata+":"+hex.EncodeToString(hash)) + } + return nil, nil +} + +func TestHandlePacketSeparatesEndpointAndRelayMatching(t *testing.T) { + for _, kind := range []uint8{meshcore.PayloadTypeReq, meshcore.PayloadTypeResponse, meshcore.PayloadTypeTxtMsg, meshcore.PayloadTypePath} { + w, base := newTestWorker() + base.observationInserted = true + db := &endpointRoutingDB{stubDB: base} + w.db = db + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, kind, 0), + PathLength: 1, Path: []byte{0xcc}, Payload: append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...)} + w.handlePacket(context.Background(), "YVR", "0102", packetEnvelope(t, packet)) + if !reflect.DeepEqual(db.endpoints, []string{"YVR:aa", "YVR:bb"}) { + t.Errorf("payload %d endpoint dispatch: %v", kind, db.endpoints) + } + if !reflect.DeepEqual(db.paths, []string{"YVR:cc"}) { + t.Errorf("payload %d relay dispatch: %v", kind, db.paths) + } + } +} diff --git a/internal/ingest/endpoint_work_test.go b/internal/ingest/endpoint_work_test.go new file mode 100644 index 00000000..ae124335 --- /dev/null +++ b/internal/ingest/endpoint_work_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/meshcore-go/meshcore-go" +) + +type endpointLookupDB struct { + *stubDB + calls [][][]byte +} + +func (d *endpointLookupDB) ResolveEndpointHashes(_ context.Context, _ string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + d.calls = append(d.calls, hashes) + return nil, nil +} +func endpointTestPacket() *meshcore.Packet { + return &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeTxtMsg, 0), Payload: append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...)} +} +func TestEndpointLookupsOnlyForLiveHearings(t *testing.T) { + r := newRepeatHarness(t, true) + d := &endpointLookupDB{stubDB: r.db} + r.w.db = d + packet := endpointTestPacket() + hear := func(inserted bool, path byte) { + d.observationInserted = inserted + packet.Path = []byte{path} + packet.PathLength = 1 + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(t, packet)) + } + hear(true, 0x11) + first := len(d.calls) + if first == 0 { + t.Fatal("first hearing missing endpoint lookup") + } + hear(false, 0x11) + if len(d.calls) != first { + t.Fatal("suppressed copy resolved endpoints") + } + hear(false, 0x22) + if len(d.calls) != 2*first { + t.Fatal("new path missing endpoint lookup") + } + hear(false, 0x22) + if len(d.calls) != 2*first { + t.Fatal("suppressed repeat resolved endpoints") + } +} diff --git a/internal/ingest/foreign_nodes_test.go b/internal/ingest/foreign_nodes_test.go new file mode 100644 index 00000000..85f4feef --- /dev/null +++ b/internal/ingest/foreign_nodes_test.go @@ -0,0 +1,81 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/binary" + "encoding/json" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/borders" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/meshcore-go/meshcore-go" + "github.com/paulmach/orb" +) + +func TestForeignNodeUpdates(t *testing.T) { + local, err := borders.New([]orb.Polygon{{{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}}}}) + if err != nil { + t.Fatal(err) + } + w, _ := newTestWorker() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "foreign", hub.Scope{Events: []hub.EventType{hub.EventNodeUpdate, hub.EventObserverStatus}}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + waitForSummarySubscriber(t, ctx, w.hub, client) + for _, tc := range []struct { + name string + enabled, position bool + role byte + lat, lng int32 + present bool + want any + }{ + {"disabled", false, true, 2, 25000000, 25000000, false, nil}, + {"inside", true, true, 2, 15000000, 15000000, true, false}, + {"outside", true, true, 2, 25000000, 25000000, true, true}, + {"omitted retains old position", true, false, 2, 0, 0, false, nil}, + {"explicit reset clears flag", true, true, 2, 0, 0, true, nil}, + {"invalid position clears flag", true, true, 2, 91000000, 25000000, true, nil}, + {"role change clears flag", true, false, meshcore.AdvertTypeChat, 0, 0, true, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + w.cfg.LocalBorders = nil + if tc.enabled { + w.cfg.LocalBorders = borders.NewLive(local) + } + data := []byte{tc.role} + if tc.position { + data[0] |= meshcore.AdvertLatLonMask + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lat)) + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lng)) + } + w.handlePayloadTypeSideEffects(ctx, buildAdvertPacketWithData(t, data, false), "AAA", []byte{1}, RadioSettings{}, nil, nil, nil, 0) + for { + select { + case event := <-client.Send: + if event.Type != hub.EventNodeUpdate { + continue + } + var payload map[string]any + if err := json.Unmarshal(event.Payload, &payload); err != nil { + t.Fatal(err) + } + got, present := payload["possiblyForeign"] + if present != tc.present || got != tc.want { + t.Fatalf("flag = %v (present=%v); want %v (present=%v)", got, present, tc.want, tc.present) + } + return + case <-ctx.Done(): + t.Fatal("node update not received") + } + } + }) + } +} diff --git a/internal/ingest/frame_length_test.go b/internal/ingest/frame_length_test.go new file mode 100644 index 00000000..c2afb511 --- /dev/null +++ b/internal/ingest/frame_length_test.go @@ -0,0 +1,145 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/lora" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +// frameCaptureDB records what handlePacket stored so a test can rebuild the +// frame from the persisted columns and compare it to the bytes off the wire. +type frameCaptureDB struct { + *stubDB + radio RadioSettings + packets []UpsertPacketParams + observed []InsertObservationParams +} + +func (s *frameCaptureDB) UpsertPacket(_ context.Context, p UpsertPacketParams) (bool, error) { + s.packets = append(s.packets, p) + return true, nil +} + +func (s *frameCaptureDB) InsertObservation(_ context.Context, o InsertObservationParams) (bool, int64, error) { + s.observed = append(s.observed, o) + return true, 1, nil +} + +func (s *frameCaptureDB) GetObserverRadio(context.Context, uuid.UUID) (RadioSettings, error) { + return s.radio, nil +} + +// grpTxtPayload is a stand-in encrypted payload; its bytes never need decoding here. +func grpTxtPayload(t *testing.T) []byte { + t.Helper() + return buildGrpTxtPacket(t, 0x1a, make([]byte, 16)).Payload +} + +// pathBytes returns n distinct hashes of hashSize bytes each. +func pathBytes(hashCount, hashSize int) []byte { + path := make([]byte, 0, hashCount*hashSize) + for i := range hashCount { + for j := range hashSize { + path = append(path, byte(0x10+i*hashSize+j)) + } + } + return path +} + +// pathLengthByte packs hash size and hop count the way MeshCore does. +func pathLengthByte(hashCount, hashSize int) uint8 { + return uint8(hashSize-1)<<6 | uint8(hashCount) +} + +// TestFrameLengthReconstruction proves lora.FrameLength rebuilds the on-air byte +// count from the stored columns alone, across the path/transport-code shapes. +func TestFrameLengthReconstruction(t *testing.T) { + for _, tc := range []struct { + name string + packet *meshcore.Packet + wantTransportCodes bool + }{ + { + name: "flood with 1-byte hash path", + packet: &meshcore.Packet{ + Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeGrpTxt, 0), + PathLength: pathLengthByte(2, 1), + Path: pathBytes(2, 1), + Payload: grpTxtPayload(t), + }, + }, + { + name: "transport flood with transport codes", + packet: &meshcore.Packet{ + Header: meshcore.MakeHeader(meshcore.RouteTypeTransportFlood, meshcore.PayloadTypeGrpTxt, 0), + PathLength: pathLengthByte(1, 1), + Path: pathBytes(1, 1), + Payload: grpTxtPayload(t), + TransportCode1: 0xbeef, + TransportCode2: 0xf00d, + }, + wantTransportCodes: true, + }, + { + name: "flood with 2-byte hash path", + packet: &meshcore.Packet{ + Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeGrpTxt, 0), + PathLength: pathLengthByte(3, 2), + Path: pathBytes(3, 2), + Payload: grpTxtPayload(t), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + raw, err := tc.packet.ToBytes() + if err != nil { + t.Fatalf("packet to bytes: %v", err) + } + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base, radio: RadioSettings{FreqMHz: 910.525, SF: 7, BWKHz: 62.5, CR: 5}} + w.db = db + + w.handlePacket(context.Background(), "YOW", "0102", packetEnvelope(t, tc.packet)) + + if len(db.packets) != 1 || len(db.observed) != 1 { + t.Fatalf("stored %d packets and %d observations, want 1 of each", len(db.packets), len(db.observed)) + } + up, obs := db.packets[0], db.observed[0] + // Pin the captured shape: without this a lost transport-code capture would + // quietly turn this case into the plain-flood one and still pass. + if gotCodes := len(up.TransportCodes) == 4; gotCodes != tc.wantTransportCodes { + t.Fatalf("captured %d transport-code bytes, want transport codes = %v", len(up.TransportCodes), tc.wantTransportCodes) + } + got := lora.FrameLength(len(up.TransportCodes) == 4, len(obs.PathBytes), len(up.RawPayload)) + if got != len(raw) { + t.Errorf("FrameLength = %d, want %d (wire bytes)", got, len(raw)) + } + if obs.AirtimeMs == nil { + t.Error("AirtimeMs is nil, want a cost for SF7/62.5/CR5") + } + }) + } +} + +// TestObservationAirtimeUnknownRadio covers the observer that never reported its +// radio: the columns are zero, so there is no airtime to store. +func TestObservationAirtimeUnknownRadio(t *testing.T) { + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} // zero RadioSettings + w.db = db + + w.handlePacket(context.Background(), "YOW", "0102", packetEnvelope(t, buildGrpTxtPacket(t, 0x1a, make([]byte, 16)))) + + if len(db.observed) != 1 { + t.Fatalf("stored %d observations, want 1", len(db.observed)) + } + if got := db.observed[0].AirtimeMs; got != nil { + t.Errorf("AirtimeMs = %v, want nil when the radio settings are unknown", *got) + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index ff907a33..6c3abb72 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -21,22 +21,38 @@ // 1. Parse topic → extract publisher pubkey // 2. Upsert observers row (status_metadata, last_status_at, observer_type, etc.) // 3. Fan out observerStatus event to hub +// +// Pipeline per incoming /neighbors message (sent every 12-368h if mqtt.neighbors +// is enabled on the observer): +// 1. Parse topic → extract IATA + publisher pubkey +// 2. Record the observer's own "self" region scope (always known, unconditional write) +// 3. For each reported zero-hop neighbor: resolve both sides via GetNodeByPubkey +// (skip the entry if either the observer or the neighbor has no node row yet -- +// i.e. hasn't advertised) and upsert a node_neighbors edge. Absence from the +// report means nothing (10KB message cap can truncate a highly-connected node's +// list), so this never deletes existing edges. A neighbor's region_scope is only +// written when status == "responded"; "timeout" (OTA scope queries are flaky) +// leaves any previously known scope untouched. package ingest import ( "context" + "crypto/rand" "encoding/binary" + "encoding/hex" "encoding/json" "fmt" - "log" + "log/slog" "strconv" "strings" + "sync/atomic" "time" mqtt "github.com/eclipse/paho.mqtt.golang" "github.com/google/uuid" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/borders" "github.com/MeshCore-Beacon/beacon-server/internal/hub" "github.com/MeshCore-Beacon/beacon-server/internal/keystore" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" @@ -44,6 +60,8 @@ import ( // Config holds the connection parameters for one broker. type Config struct { + // LocalBorders is the same live classifier used by node API reads. + LocalBorders *borders.Live // BrokerName is a short human-readable label ("mqtt1", "mqtt2") used in // log messages and stored in packet_observations.source_broker. BrokerName string @@ -70,10 +88,10 @@ type Config struct { type DB interface { // UpsertObserver upserts the observers row keyed on pubkey, and returns // the Observer ID, Display Name and an error if any. - UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) + UpsertObserver(ctx context.Context, pubkey []byte, iata string) (uuid.UUID, string, error) // UpsertObserverBroker records that this observer was seen on brokerName. - UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error + UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error // UpsertIATA auto-creates an iata_codes row if it doesn't exist yet. UpsertIATA(ctx context.Context, iata string) error @@ -84,9 +102,9 @@ type DB interface { // SetPacketDecrypted marks a packet as decrypted in the DB SetPacketDecrypted(ctx context.Context, hash []byte) error - // InsertObservation inserts a packet_observations row. - // Returns (inserted, error); inserted=false means ON CONFLICT DO NOTHING fired. - InsertObservation(ctx context.Context, o InsertObservationParams) (bool, error) + // InsertObservation inserts a packet_observations row and returns the packet's + // observation count; inserted=false means ON CONFLICT DO NOTHING fired. + InsertObservation(ctx context.Context, o InsertObservationParams) (inserted bool, observationCount int64, err error) // SetNodeCapability flips supports_multibyte_paths or supports_multibyte_traces // for a node, never downgrading an existing TRUE. @@ -115,14 +133,14 @@ type DB interface { // UpsertNodeShortID upserts a node_short_ids row for path resolution. UpsertNodeShortID(ctx context.Context, nodeID uuid.UUID, iata string, prefix4 []byte) error - // InsertChannelMessage stores a decrypted group text message. Returns insert success and an error. - InsertChannelMessage(ctx context.Context, m InsertChannelMessageParams) (bool, error) + // InsertChannelMessage stores a decrypted group text message. Returns the inserted message, nil for a duplicate, and an error. + InsertChannelMessage(ctx context.Context, m InsertChannelMessageParams) (*InsertedChannelMessage, error) // UpdateObserverStatus updates the observer row from a /status message. Returns the OberserID // and any error. UpdateObserverStatus(ctx context.Context, p UpdateObserverStatusParams) (uuid.UUID, error) - // GetObserverLastIATA returns the IATA from the most recent observation for the given observer. + // GetObserverLastIATA returns the IATA of the observer's most recent packet, or "" if none. GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) // InsertObserverTelemetry stores a telemetry snapshot for an observer. @@ -141,6 +159,9 @@ type DB interface { // ResolvePathHashes returns a list of node UUIDs for the given path hash prefixes and IATA. ResolvePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) + // ResolveEndpointHashes matches one-byte logical endpoints, including companions. + ResolveEndpointHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) + // UpsertChannel upserts a channel row by (hash, keyFingerprint) and returns its integer ID. // Pass nil keyFingerprint to record a hash-only row when the key is unknown. UpsertChannel(ctx context.Context, channelHash []byte, keyFingerprint []byte, name string, hashtag string) (int, error) @@ -155,14 +176,14 @@ type DB interface { // used by BackfillChannelMessages to retry them against the current keystore at boot. ListUndecryptedGroupTextPackets(ctx context.Context) ([]UndecryptedPacket, error) + // ListUndecryptedGroupTextPacketsByHash limits that scan to channels that just gained a key. + ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]UndecryptedPacket, error) + // UpsertChannelIATA upserts a channel_iatas row. UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error - // UpsertTraceIATA upserts a trace_iatas row. - UpsertTraceIATA(ctx context.Context, traceTag []byte, iata string, heardAt time.Time) error - - // GetPacketObservationCount returns the number of rows for the packet observations - GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) + // RecordTrace records one hearing of a TRACE packet in trace_iatas and trace_tags. + RecordTrace(ctx context.Context, h TraceHearing) error // GetTransportScopeByName returns the ID of a transport scope by its normalized name. GetTransportScopeByName(ctx context.Context, name string) (int32, error) @@ -178,7 +199,13 @@ type DB interface { // UpsertNodeNeighbor records or updates a neighbor relationship between two nodes. // nodeID is the advertising node, neighborID is the first-hop forwarder. // snr is optional (nil when no signal reading is available, the common case). - UpsertNodeNeighbor(ctx context.Context, nodeID, neighborID uuid.UUID, iata string, snr *float32) error + // regionScope is optional (nil when there's no fresh OTA-queried scope to + // record for this neighbor, e.g. a /neighbors report entry with a failed query). + UpsertNodeNeighbor(ctx context.Context, nodeID, neighborID uuid.UUID, iata string, snr *float32, regionScope *string) error + + // UpdateObserverRegionScope records the observer's own OTA-reported region + // scope, from the "self" field of a /neighbors report. + UpdateObserverRegionScope(ctx context.Context, observerID uuid.UUID, regionScope string) error } // ChannelKeyStore is a read-only view of the channel keys loaded from config. @@ -195,19 +222,22 @@ type ScopeStore interface { // Worker holds the dependencies for one broker's ingest loop. type Worker struct { + log *slog.Logger cfg Config db DB hub *hub.Hub keys ChannelKeyStore scopes ScopeStore - client mqtt.Client + client atomic.Pointer[mqtt.Client] // read by /brokers while Start sets it + capabilities capabilityCache onNodeUpsert func(ctx context.Context, nodeID uuid.UUID) onObserverUpsert func(ctx context.Context, observerID uuid.UUID) } // New creates an ingest Worker. Call Start() to connect and begin processing. func New(cfg Config, db DB, h *hub.Hub, keys ChannelKeyStore, scopes ScopeStore) *Worker { - return &Worker{cfg: cfg, db: db, hub: h, keys: keys, scopes: scopes} + return &Worker{cfg: cfg, db: db, hub: h, keys: keys, scopes: scopes, + log: slog.Default().With("component", "ingest", "broker", cfg.BrokerName)} } // Start connects to the broker and blocks until ctx is cancelled. It @@ -216,9 +246,33 @@ func New(cfg Config, db DB, h *hub.Hub, keys ChannelKeyStore, scopes ScopeStore) // // Intended usage: go worker.Start(ctx) func (w *Worker) Start(ctx context.Context) { + if w.cfg.URL == "" { + return + } + queue := newMessageQueue(8, 2048, 32<<20, func(ctx context.Context, msg mqtt.Message) { + w.handleMessageContext(ctx, msg) + }) + reportDrops := func() { + if n := queue.takeDropped(); n > 0 { + w.log.Error("ingest queue full, messages dropped", "count", n) + } + } + defer func() { + drainCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := queue.close(drainCtx); err != nil { + w.log.Warn("ingest shutdown timed out", "error", err) + } + reportDrops() + w.log.Info("stopped") + }() + + // Isolate workers across deployments; Paho reuses this ID on reconnect. + // Keep it alphanumeric and within MQTT 3.1's 23-character client ID limit. + // Auto-ack: sessions are clean, so deferring acks only throttled the broker's inflight window. opts := mqtt.NewClientOptions(). AddBroker(w.cfg.URL). - SetClientID(fmt.Sprintf("beacon-%s", w.cfg.BrokerName)). + SetClientID(rand.Text()[:23]). SetUsername(w.cfg.Username). SetPassword(w.cfg.Password). SetAutoReconnect(true). @@ -230,22 +284,35 @@ func (w *Worker) Start(ctx context.Context) { SetConnectRetry(true). SetConnectRetryInterval(5 * time.Second). SetOnConnectHandler(func(c mqtt.Client) { - log.Printf("ingest[%s]: connected to %s", w.cfg.BrokerName, w.cfg.URL) - w.subscribe(c) + w.log.Info("connected") + w.subscribe(c, queue) }). SetConnectionLostHandler(func(_ mqtt.Client, err error) { - log.Printf("ingest[%s]: connection lost, will reconnect: %v", w.cfg.BrokerName, err) + w.log.Warn("connection lost, will reconnect", "error", err) }) - w.client = mqtt.NewClient(opts) - if tok := w.client.Connect(); tok.Wait() && tok.Error() != nil { - log.Printf("ingest[%s]: initial connect failed: %v", w.cfg.BrokerName, tok.Error()) - // paho will retry; we fall through and wait for ctx + client := mqtt.NewClient(opts) + w.client.Store(&client) + + token := client.Connect() + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + connected := token.Done() + for { + select { + case <-connected: + if err := token.Error(); err != nil { + w.log.Error("initial connect failed", "error", err) + } + connected = nil + case <-ticker.C: + reportDrops() + case <-ctx.Done(): + client.Disconnect(500) + return + } } - <-ctx.Done() - w.client.Disconnect(500) - log.Printf("ingest[%s]: stopped", w.cfg.BrokerName) } func (w *Worker) BrokerName() string { @@ -253,10 +320,11 @@ func (w *Worker) BrokerName() string { } func (w *Worker) IsConnected() bool { - if w.client == nil { + c := w.client.Load() + if c == nil { return false } - return w.client.IsConnected() + return (*c).IsConnected() } func (w *Worker) SetCacheInvalidators(onNode, onObserver func(ctx context.Context, id uuid.UUID)) { @@ -265,22 +333,50 @@ func (w *Worker) SetCacheInvalidators(onNode, onObserver func(ctx context.Contex } // subscribe registers the wildcard topic handler after (re)connect. -func (w *Worker) subscribe(client mqtt.Client) { +func (w *Worker) subscribe(client mqtt.Client, queue *messageQueue) { // meshcore/{IATA}/{pubkey}/packets // meshcore/{IATA}/{pubkey}/status // We do NOT subscribe to /internal (Role 2 access). tok := client.Subscribe("meshcore/#", 1, func(_ mqtt.Client, msg mqtt.Message) { - w.handleMessage(msg) + queue.enqueue(msg) }) if tok.Wait() && tok.Error() != nil { - log.Printf("ingest[%s]: subscribe error: %v", w.cfg.BrokerName, tok.Error()) + w.log.Error("subscribe error", "error", tok.Error()) } } +// isValidIATA reports whether s is 3 uppercase ASCII letters, matching the +// iata_codes.iata CHAR(3) column. MQTT topic segments are attacker/observer +// controlled and must be validated before touching the DB. +func isValidIATA(s string) bool { + if len(s) != 3 { + return false + } + for i := 0; i < 3; i++ { + if s[i] < 'A' || s[i] > 'Z' { + return false + } + } + return true +} + +// isValidPubkeyHex reports whether s is a 32-byte public key in hex. +func isValidPubkeyHex(s string) bool { + if len(s) != 64 { + return false + } + _, err := hex.DecodeString(s) + return err == nil +} + // handleMessage dispatches incoming MQTT messages by subtopic. // Each message is processed with a 30s timeout to prevent slow DB calls -// from blocking the MQTT receive goroutine indefinitely. +// from holding a processing worker indefinitely. func (w *Worker) handleMessage(msg mqtt.Message) { + w.handleMessageContext(context.Background(), msg) +} + +func (w *Worker) handleMessageContext(parent context.Context, msg mqtt.Message) { // Topic shape: meshcore/{IATA}/{pubkey}/{subtopic} parts := strings.SplitN(msg.Topic(), "/", 4) if len(parts) != 4 || parts[0] != "meshcore" { @@ -288,15 +384,28 @@ func (w *Worker) handleMessage(msg mqtt.Message) { } iata, pubkeyHex, subtopic := parts[1], parts[2], parts[3] + // iata_codes.iata is CHAR(3); anything else would fail the DB insert + // downstream, so reject malformed topic segments here instead. + if !isValidIATA(iata) { + w.log.Debug("dropped packet with malformed IATA", "iata", iata, "topic", msg.Topic()) + return + } + if !isValidPubkeyHex(pubkeyHex) { + w.log.Debug("dropped message with malformed observer pubkey", "pubkey", pubkeyHex, "topic", msg.Topic()) + return + } + // Drop packets from IATAs outside the configured geographic filter. if w.cfg.AllowedIATAs != nil { if _, ok := w.cfg.AllowedIATAs[iata]; !ok { - log.Printf("ingest[%s]: dropped packet from %s (not in allowed IATAs)", w.cfg.BrokerName, iata) + if w.log.Enabled(context.Background(), slog.LevelDebug) { + w.log.Debug(fmt.Sprintf("dropped packet from %s (not in allowed IATAs)", iata)) + } return } } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(parent, 30*time.Second) defer cancel() switch subtopic { @@ -304,6 +413,8 @@ func (w *Worker) handleMessage(msg mqtt.Message) { w.handlePacket(ctx, iata, pubkeyHex, msg.Payload()) case "status": w.handleStatus(ctx, pubkeyHex, msg.Payload()) + case "neighbors": + w.handleNeighbors(ctx, iata, pubkeyHex, msg.Payload()) // "internal" is intentionally not handled (Role 2 access) } } @@ -311,7 +422,7 @@ func (w *Worker) handleMessage(msg mqtt.Message) { func (w *Worker) broadcast(eventType hub.EventType, iata string, payloadType uint8, channelHash string, payload any) { b, err := json.Marshal(payload) if err != nil { - log.Printf("ingest[%s]: failed to marshal %s event: %v", w.cfg.BrokerName, eventType, err) + w.log.Error(fmt.Sprintf("failed to marshal %s event", eventType), "error", err) return } w.hub.Broadcast(hub.Event{ @@ -323,32 +434,50 @@ func (w *Worker) broadcast(eventType hub.EventType, iata string, payloadType uin }) } -// broadcastPacketObservation marshals evt twice: once as-is (the default +// broadcastPacketObservation marshals evt once as-is (the default // payload every packetObservation subscriber gets) and once with // resolvedPath populated (delivered only to connections that opted in via // the "configure" WS message; see hub.Client.ResolvePath). resolvedPath is // passed in rather than computed here because the caller already has the // path-hash resolution results in hand from other per-packet work (known // route detection, capability detection) — this adds no extra DB calls. -func (w *Worker) broadcastPacketObservation(iata string, payloadType uint8, evt packetObservationEvent, resolvedPath []api.ResolvedHop) { +// Repeats go through the hub's drop-first path. +func (w *Worker) broadcastPacketObservation(iata string, payloadType uint8, evt packetObservationEvent, resolvedPath []api.ResolvedHop, observerKey string, repeat bool) { base, err := json.Marshal(evt) if err != nil { - log.Printf("ingest[%s]: failed to marshal packetObservation event: %v", w.cfg.BrokerName, err) + w.log.Error("failed to marshal packetObservation event", "error", err) return } + out := hub.Event{ + Type: hub.EventPacketObservation, + Payload: base, + IATA: iata, + PayloadType: payloadType, + RouteType: evt.Packet.RouteType, + ObserverID: evt.Observation.ObserverID, + } evt.Observation.ResolvedPath = resolvedPath - resolved, err := json.Marshal(evt) - if err != nil { - log.Printf("ingest[%s]: failed to marshal packetObservation event (resolved variant): %v", w.cfg.BrokerName, err) - resolved = nil // fall back to base-only; not fatal + if out.PayloadResolved, err = json.Marshal(evt); err != nil { + w.log.Error("failed to marshal packetObservation event (resolved variant)", "error", err) + out.PayloadResolved = nil } - w.hub.Broadcast(hub.Event{ - Type: hub.EventPacketObservation, - Payload: base, - PayloadResolved: resolved, - IATA: iata, - PayloadType: payloadType, - }) + if w.hub.ObserverKeyWanted() { + evt.Observation.ObserverPublicKey = observerKey + if out.PayloadResolvedWithKey, err = json.Marshal(evt); err != nil { + w.log.Error("failed to marshal packetObservation event (resolved key variant)", "error", err) + out.PayloadResolvedWithKey = nil + } + evt.Observation.ResolvedPath = nil + if out.PayloadWithKey, err = json.Marshal(evt); err != nil { + w.log.Error("failed to marshal packetObservation event (key variant)", "error", err) + out.PayloadWithKey = nil + } + } + if repeat { + w.hub.BroadcastRepeat(out) + return + } + w.hub.Broadcast(out) } // parseNumber handles RSSI and SNR fields that different observer types send as diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index b01b9cc3..cffa4654 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "encoding/json" "errors" + "log/slog" "testing" "time" @@ -175,13 +176,20 @@ type stubDB struct { setCapabilityCalls []setCapabilityCall upsertNodeCalls int + upsertNodeParams UpsertNodeParams upsertChannelCalls int upsertChannelHashOnlyCalls int upsertChannelIATACalls int - upsertTraceIATACalls int + traceHearings []TraceHearing + packetIsNew bool observationInserted bool + channelMessages []InsertChannelMessageParams + observationCount int64 insertChannelMessageResult bool // configurable return for InsertChannelMessage; default false undecryptedPackets []UndecryptedPacket + undecryptedByHash []UndecryptedPacket + backfillHashes [][]byte + dbHook func() // runs in UpsertChannelIATA, the first DB call after InsertObservation } type setCapabilityCall struct { @@ -196,21 +204,24 @@ func (s *stubDB) SetNodeCapability(_ context.Context, nodeID uuid.UUID, paths, t } // no-op implementations for remaining DB interface methods -func (s *stubDB) UpsertObserver(_ context.Context, _ []byte) (uuid.UUID, string, error) { +func (s *stubDB) UpsertObserver(_ context.Context, _ []byte, _ string) (uuid.UUID, string, error) { return uuid.Nil, "", nil } -func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string) error { return nil } -func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } +func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, _ bool) error { + return nil +} +func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } func (s *stubDB) UpsertPacket(_ context.Context, _ UpsertPacketParams) (bool, error) { - return false, nil + return s.packetIsNew, nil } func (s *stubDB) SetPacketDecrypted(_ context.Context, _ []byte) error { return nil } -func (s *stubDB) InsertObservation(_ context.Context, _ InsertObservationParams) (bool, error) { - return s.observationInserted, nil +func (s *stubDB) InsertObservation(_ context.Context, _ InsertObservationParams) (bool, int64, error) { + return s.observationInserted, s.observationCount, nil } func (s *stubDB) SetNodeDefaultScope(_ context.Context, _ uuid.UUID, _ int32) error { return nil } -func (s *stubDB) UpsertNode(_ context.Context, _ UpsertNodeParams, _ RadioSettings) (uuid.UUID, error) { +func (s *stubDB) UpsertNode(_ context.Context, params UpsertNodeParams, _ RadioSettings) (uuid.UUID, error) { s.upsertNodeCalls++ + s.upsertNodeParams = params return uuid.Nil, nil } func (s *stubDB) UpsertNodeIATA(_ context.Context, _ uuid.UUID, _ string) error { return nil } @@ -226,8 +237,12 @@ func (s *stubDB) GetNodesByIDs(_ context.Context, _ []uuid.UUID) (map[uuid.UUID] return nil, nil } -func (s *stubDB) InsertChannelMessage(_ context.Context, _ InsertChannelMessageParams) (bool, error) { - return s.insertChannelMessageResult, nil +func (s *stubDB) InsertChannelMessage(_ context.Context, m InsertChannelMessageParams) (*InsertedChannelMessage, error) { + s.channelMessages = append(s.channelMessages, m) + if !s.insertChannelMessageResult { + return nil, nil + } + return &InsertedChannelMessage{ID: 1, ScopeStatus: "unavailable"}, nil } func (s *stubDB) UpdateObserverStatus(_ context.Context, _ UpdateObserverStatusParams) (uuid.UUID, error) { @@ -252,6 +267,10 @@ func (s *stubDB) ResolvePathHashes(_ context.Context, _ string, _ [][]byte) (map return nil, nil } +func (s *stubDB) ResolveEndpointHashes(_ context.Context, _ string, _ [][]byte) (map[string][]api.ResolvedPathEntry, error) { + return nil, nil +} + func (s *stubDB) UpsertChannel(_ context.Context, _ []byte, _ []byte, _, _ string) (int, error) { s.upsertChannelCalls++ return 0, nil @@ -266,33 +285,42 @@ func (s *stubDB) ListUndecryptedGroupTextPackets(_ context.Context) ([]Undecrypt return s.undecryptedPackets, nil } +func (s *stubDB) ListUndecryptedGroupTextPacketsByHash(_ context.Context, hashes [][]byte) ([]UndecryptedPacket, error) { + s.backfillHashes = hashes + return s.undecryptedByHash, nil +} + func (s *stubDB) UpsertChannelIATA(_ context.Context, _ []byte, _ string, _ time.Time) error { + if s.dbHook != nil { + s.dbHook() + } s.upsertChannelIATACalls++ return nil } -func (s *stubDB) UpsertTraceIATA(_ context.Context, _ []byte, _ string, _ time.Time) error { - s.upsertTraceIATACalls++ +func (s *stubDB) RecordTrace(_ context.Context, h TraceHearing) error { + s.traceHearings = append(s.traceHearings, h) return nil } -func (s *stubDB) GetPacketObservationCount(_ context.Context, _ []byte) (int64, error) { - return 0, nil -} - func (s *stubDB) GetTransportScopeByName(_ context.Context, _ string) (int32, error) { return 0, nil } func (s *stubDB) UpsertObserverScope(_ context.Context, _ uuid.UUID, _ int32) error { return nil } func (s *stubDB) UpsertKnownRoute(_ context.Context, _ []uuid.UUID, _ [][]byte, _ string, _ int32) error { return nil } -func (s *stubDB) UpsertNodeNeighbor(_ context.Context, _, _ uuid.UUID, _ string, _ *float32) error { +func (s *stubDB) UpsertNodeNeighbor(_ context.Context, _, _ uuid.UUID, _ string, _ *float32, _ *string) error { + return nil +} + +func (s *stubDB) UpdateObserverRegionScope(_ context.Context, _ uuid.UUID, _ string) error { return nil } func newTestWorker() (*Worker, *stubDB) { db := &stubDB{} w := &Worker{ + log: slog.Default().With("component", "ingest", "broker", "test"), cfg: Config{BrokerName: "test"}, db: db, hub: hub.New(), diff --git a/internal/ingest/logging_test.go b/internal/ingest/logging_test.go new file mode 100644 index 00000000..3f9d343a --- /dev/null +++ b/internal/ingest/logging_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "bytes" + "encoding/json" + "log/slog" + "strconv" + "strings" + "testing" +) + +type malformedTopicMessage struct{ topic string } + +func (m malformedTopicMessage) Duplicate() bool { return false } +func (m malformedTopicMessage) Qos() byte { return 0 } +func (m malformedTopicMessage) Retained() bool { return false } +func (m malformedTopicMessage) Topic() string { return m.topic } +func (m malformedTopicMessage) MessageID() uint16 { return 0 } +func (m malformedTopicMessage) Payload() []byte { return nil } +func (m malformedTopicMessage) Ack() {} + +func TestMalformedIATALogNamesSource(t *testing.T) { + for _, format := range []string{"json", "text"} { + for _, iata := range []string{"XX", "XX\r\nforged=true"} { + t.Run(format+"/"+strconv.Quote(iata), func(t *testing.T) { + var output bytes.Buffer + opts := &slog.HandlerOptions{Level: slog.LevelDebug} + var handler slog.Handler = slog.NewJSONHandler(&output, opts) + if format == "text" { + handler = slog.NewTextHandler(&output, opts) + } + previous := slog.Default() + slog.SetDefault(slog.New(handler)) + t.Cleanup(func() { slog.SetDefault(previous) }) + worker, _ := newTestWorker() + topic := "meshcore/" + iata + "/0011223344556677/packets" + worker.handleMessage(malformedTopicMessage{topic: topic}) + if bytes.Count(output.Bytes(), []byte("\n")) != 1 || bytes.ContainsRune(output.Bytes(), '\r') { + t.Fatalf("log did not remain one escaped record: %q", output.String()) + } + if format == "json" { + var record map[string]any + if err := json.Unmarshal(output.Bytes(), &record); err != nil { + t.Fatal(err) + } + if record["iata"] != iata || record["topic"] != topic || record["msg"] != "dropped packet with malformed IATA" { + t.Fatalf("log lost its source: %v", record) + } + } else { + expectedIATA, expectedTopic := iata, topic + if strings.ContainsAny(iata, "\r\n") { + expectedIATA, expectedTopic = strconv.Quote(iata), strconv.Quote(topic) + } + if !strings.Contains(output.String(), "iata="+expectedIATA) || !strings.Contains(output.String(), "topic="+expectedTopic) { + t.Fatalf("log lost its source: %q", output.String()) + } + } + }) + } + } +} diff --git a/internal/ingest/mqtt_client_test.go b/internal/ingest/mqtt_client_test.go new file mode 100644 index 00000000..096dd89e --- /dev/null +++ b/internal/ingest/mqtt_client_test.go @@ -0,0 +1,30 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" +) + +// Run with -race: /brokers reads the client while Start creates it. +func TestIsConnectedWhileStarting(t *testing.T) { + w := New(Config{BrokerName: "test", URL: "tcp://127.0.0.1:1"}, &stubDB{}, hub.New(), &stubKeys{}, &stubScopes{}) + ctx, cancel := context.WithCancel(context.Background()) + stopped := make(chan struct{}) + go func() { w.Start(ctx); close(stopped) }() + deadline := time.Now().Add(100 * time.Millisecond) + for time.Now().Before(deadline) { + w.IsConnected() + } + cancel() + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("worker did not stop") + } +} diff --git a/internal/ingest/mqtt_test.go b/internal/ingest/mqtt_test.go new file mode 100644 index 00000000..6cde148a --- /dev/null +++ b/internal/ingest/mqtt_test.go @@ -0,0 +1,141 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "net" + "regexp" + "sync" + "testing" + "time" + + "github.com/eclipse/paho.mqtt.golang/packets" +) + +type mqttConnection struct { + id string + conn net.Conn +} + +// A local protocol fixture observes the actual CONNECT packets sent by Start. +// It neither publishes nor connects to a real broker. +func mqttTestBroker(t *testing.T) (string, <-chan mqttConnection) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + connected := make(chan mqttConnection, 4) + var clients sync.WaitGroup + stopped := make(chan struct{}) + go func() { + defer close(stopped) + for { + conn, err := listener.Accept() + if err != nil { + return + } + clients.Add(1) + go func() { + defer clients.Done() + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) + first, err := packets.ReadPacket(conn) + if err != nil { + return + } + connect, ok := first.(*packets.ConnectPacket) + if !ok { + return + } + if err := packets.NewControlPacket(packets.Connack).Write(conn); err != nil { + return + } + announced := false + for { + packet, err := packets.ReadPacket(conn) + if err != nil { + return + } + switch packet := packet.(type) { + case *packets.SubscribePacket: + ack := packets.NewControlPacket(packets.Suback).(*packets.SubackPacket) + ack.MessageID = packet.MessageID + ack.ReturnCodes = []byte{1} + if err := ack.Write(conn); err != nil { + return + } + if !announced { + connected <- mqttConnection{connect.ClientIdentifier, conn} + announced = true + } + case *packets.PingreqPacket: + if err := packets.NewControlPacket(packets.Pingresp).Write(conn); err != nil { + return + } + case *packets.DisconnectPacket: + return + } + } + }() + } + }() + t.Cleanup(func() { _ = listener.Close(); <-stopped; clients.Wait() }) + return "tcp://" + listener.Addr().String(), connected +} + +func startMQTTTestWorker(t *testing.T, address string) { + t.Helper() + w, _ := newTestWorker() + w.cfg.URL = address + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); w.Start(ctx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Error("worker did not stop") + } + }) +} + +func awaitMQTTConnection(t *testing.T, connected <-chan mqttConnection) mqttConnection { + t.Helper() + select { + case c := <-connected: + return c + case <-time.After(5 * time.Second): + t.Fatal("MQTT CONNECT not received") + return mqttConnection{} + } +} + +func TestWorkersUseDistinctMQTTClientIDs(t *testing.T) { + address, connected := mqttTestBroker(t) + startMQTTTestWorker(t, address) + first := awaitMQTTConnection(t, connected) + startMQTTTestWorker(t, address) + second := awaitMQTTConnection(t, connected) + if first.id == "" || second.id == "" || first.id == second.id { + t.Fatalf("workers with the same broker name must use distinct client IDs: %q and %q", first.id, second.id) + } + valid := regexp.MustCompile(`^[A-Za-z0-9]{1,23}$`) + if !valid.MatchString(first.id) || !valid.MatchString(second.id) { + t.Fatal("client ID must be 1..23 alphanumeric characters") + } +} + +func TestReconnectKeepsMQTTClientID(t *testing.T) { + address, connected := mqttTestBroker(t) + startMQTTTestWorker(t, address) + first := awaitMQTTConnection(t, connected) + _ = first.conn.Close() + second := awaitMQTTConnection(t, connected) + if first.id != second.id { + t.Fatalf("reconnect changed client ID: %q to %q", first.id, second.id) + } +} diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go new file mode 100644 index 00000000..3972c652 --- /dev/null +++ b/internal/ingest/neighbors.go @@ -0,0 +1,105 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" +) + +// neighborReportEntry is one entry in the "neighbors" array of a /neighbors report. +type neighborReportEntry struct { + PubKey string `json:"pubkey"` + SNR float32 `json:"snr"` + HeardSecsAgo int64 `json:"heard_secs_ago"` + Scopes string `json:"scopes"` + Status string `json:"status"` // "responded" or "timeout" +} + +// neighborReport is the JSON payload for an observer /neighbors message. +type neighborReport struct { + Self struct { + Scopes string `json:"scopes"` + } `json:"self"` + Neighbors []neighborReportEntry `json:"neighbors"` +} + +// handleNeighbors processes a /neighbors message: the observer's own OTA-configured +// region scope, plus a snapshot of its zero-hop neighbors and (best-effort) their +// OTA-queried region scopes. See the package doc comment for the full pipeline. +func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, raw []byte) { + var report neighborReport + if err := json.Unmarshal(raw, &report); err != nil { + w.log.Warn(fmt.Sprintf("malformed neighbors envelope from %s", pubkeyHex), "error", err) + return + } + + pubkey, err := hex.DecodeString(pubkeyHex) + if err != nil { + w.log.Warn(fmt.Sprintf("invalid pubkey hex in neighbors from %s", pubkeyHex), "error", err) + return + } + + observerID, _, err := w.db.UpsertObserver(ctx, pubkey, "") + if err != nil { + w.log.Error(fmt.Sprintf("db: upsert observer failed in neighbors from %s", pubkeyHex), "error", err) + return + } + if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName, false); err != nil { + w.log.Error(fmt.Sprintf("db: upsert observer broker failed in neighbors from %s", pubkeyHex), "error", err) + } + + // self.scopes is always known (it's the observer's own config, not an OTA + // query), so this is an unconditional write -- unlike the per-neighbor + // scopes below, there's no "query failed" case to protect against here. + if err := w.db.UpdateObserverRegionScope(ctx, observerID, cleanText(report.Self.Scopes)); err != nil { + w.log.Error(fmt.Sprintf("db: update observer region scope failed for %s", pubkeyHex), "error", err) + } + + // The observer's own node row (as opposed to its observers row above) is + // what node_neighbors edges hang off of. As elsewhere in this package + // (see GetNodeByPubkey's doc comment), an observer that hasn't advertised + // yet has no node row, in which case we simply can't record edges for it + // this round -- not an error, just nothing to attach the neighbors to. + observerNodeID, err := w.db.GetNodeByPubkey(ctx, pubkey) + if err != nil { + return + } + + for _, n := range report.Neighbors { + neighborPubkey, err := hex.DecodeString(n.PubKey) + if err != nil { + w.log.Warn(fmt.Sprintf("invalid neighbor pubkey hex %q from %s", n.PubKey, pubkeyHex), "error", err) + continue + } + neighborNodeID, err := w.db.GetNodeByPubkey(ctx, neighborPubkey) + if err != nil { + // Neighbor hasn't advertised yet -- nothing to link to. Its absence + // here doesn't mean it's gone; it'll show up once it advertises. + continue + } + if neighborNodeID == observerNodeID { + continue // don't record a node as its own neighbor + } + + snr := n.SNR + + // OTA region scope queries are unreliable (per the firmware author, even + // the mobile app sees this) -- only "responded" is a trustworthy read. + // A "timeout" must not be taken as "the neighbor cleared its scope", so + // we pass nil and let UpsertNodeNeighbor's COALESCE preserve whatever + // was last known. + var regionScope *string + if n.Status == "responded" { + scope := cleanText(n.Scopes) + regionScope = &scope + } + + if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, neighborNodeID, iata, &snr, regionScope); err != nil { + w.log.Error(fmt.Sprintf("db: upsert neighbor failed for %s -> %s", pubkeyHex, n.PubKey), "error", err) + } + } +} diff --git a/internal/ingest/observer_key_test.go b/internal/ingest/observer_key_test.go new file mode 100644 index 00000000..c2021212 --- /dev/null +++ b/internal/ingest/observer_key_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" +) + +func TestBroadcastPacketObservation_ObserverKeyOptIn(t *testing.T) { + const key = "ab01020304050607080910111213141516171819202122232425262728293031" + w, _ := newTestWorker() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "all", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + waitForSummarySubscriber(t, ctx, w.hub, client) + + broadcast := func() hub.Event { + t.Helper() + w.broadcastPacketObservation("YVR", 4, packetObservationEvent{}, []api.ResolvedHop{{}}, key, false) + for { + select { + case evt := <-client.Send: + if evt.Type == hub.EventPacketObservation { + return evt + } + case <-ctx.Done(): + t.Fatal("packetObservation not delivered") + } + } + } + observation := func(raw json.RawMessage) map[string]json.RawMessage { + t.Helper() + var evt struct { + Observation map[string]json.RawMessage `json:"observation"` + } + if err := json.Unmarshal(raw, &evt); err != nil { + t.Fatal(err) + } + return evt.Observation + } + + evt := broadcast() + if evt.PayloadWithKey != nil || evt.PayloadResolvedWithKey != nil { + t.Fatal("key variants built with no opted-in client") + } + if strings.Contains(string(evt.Payload)+string(evt.PayloadResolved), "observerPublicKey") { + t.Fatal("default payloads must not carry observerPublicKey") + } + + // client stays plain so its Payload is still the base variant. + keyed := w.hub.NewClient() + defer w.hub.Remove(keyed) + w.hub.Configure(keyed, hub.ClientOptions{IncludeObserverKey: true}) + for !w.hub.ObserverKeyWanted() { + time.Sleep(time.Millisecond) + } + evt = broadcast() + if _, ok := observation(evt.Payload)["observerPublicKey"]; ok { + t.Fatal("base payload must not carry observerPublicKey") + } + withKey := observation(evt.PayloadWithKey) + if string(withKey["observerPublicKey"]) != `"`+key+`"` || string(withKey["resolvedPath"]) != "null" { + t.Fatalf("key variant = %v", withKey) + } + both := observation(evt.PayloadResolvedWithKey) + if string(both["observerPublicKey"]) != `"`+key+`"` || string(both["resolvedPath"]) == "null" { + t.Fatalf("resolved key variant = %v", both) + } +} diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index eb90db1b..e8754925 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -11,11 +11,13 @@ import ( "encoding/hex" "encoding/json" "fmt" - "log" + "slices" "strings" "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/lora" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" "github.com/google/uuid" "github.com/meshcore-go/meshcore-go" ) @@ -36,6 +38,14 @@ type UpsertPacketParams struct { TraceTag []byte } +// TraceHearing is one hearing of a TRACE packet. The packet's own contribution to its tag +// (count, type, best path) is recorded when the packet is stored. +type TraceHearing struct { + TraceTag []byte + IATA string + HeardAt time.Time +} + // InsertObservationParams mirrors the columns written on packet_observations insert. type InsertObservationParams struct { PacketHash []byte @@ -55,6 +65,7 @@ type InsertObservationParams struct { CodingRate int16 SourceBroker string PayloadType int16 + AirtimeMs *float32 // nil when the observer never reported costable radio settings } // RadioSettings holds the radio configuration for an observer, populated from @@ -78,17 +89,22 @@ type packetObservationEvent struct { IsFirstObservation bool `json:"isFirstObservation"` ObservationCount int64 `json:"observationCount"` Scope *string `json:"scope,omitempty"` + Summary *string `json:"summary,omitempty"` // advert name, or "ACK …"/"TRACE …"/"PING …" for those types + // Only set on later hearings sent to includeRepeats clients; those carry observationCount 0. + IsRepeat bool `json:"isRepeat,omitempty"` } `json:"packet"` Observation struct { - ObserverID string `json:"observerId"` - ObserverName string `json:"observerName"` - IATA string `json:"iata"` - HeardAt int64 `json:"heardAt"` - RSSI int16 `json:"rssi"` - SNR float32 `json:"snr"` - SourceBroker string `json:"sourceBroker"` - PathBytes string `json:"pathBytes"` - PathLength struct { + ObserverID string `json:"observerId"` + ObserverName string `json:"observerName"` + // Only set for clients that sent includeObserverKey. + ObserverPublicKey string `json:"observerPublicKey,omitempty"` + IATA string `json:"iata"` + HeardAt int64 `json:"heardAt"` + RSSI int16 `json:"rssi"` + SNR float32 `json:"snr"` + SourceBroker string `json:"sourceBroker"` + PathBytes string `json:"pathBytes"` + PathLength struct { Raw string `json:"raw"` HashSize uint8 `json:"hashSize"` HopCount uint8 `json:"hopCount"` @@ -286,46 +302,46 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } err := json.Unmarshal(raw, &envelope) if err != nil { - log.Printf("ingest[%s]: malformed packet envelope from %s/%s", w.cfg.BrokerName, iata, pubkeyHex) + w.log.Warn(fmt.Sprintf("malformed packet envelope from %s/%s", iata, pubkeyHex)) return } if envelope.Raw == "" { if envelope.Len == "0" || envelope.Len == "" { return // observer keepalive with no packet data } - log.Printf("ingest[%s]: malformed packet envelope from %s/%s", w.cfg.BrokerName, iata, pubkeyHex) + w.log.Warn(fmt.Sprintf("malformed packet envelope from %s/%s", iata, pubkeyHex)) return } hexBytes, err := hex.DecodeString(strings.ReplaceAll(envelope.Raw, " ", "")) if err != nil { - log.Printf("ingest[%s]: invalid hex from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("invalid hex from %s/%s", iata, pubkeyHex), "error", err) return } packet, err := meshcore.PacketFromBytes(hexBytes) if err != nil { - log.Printf("ingest[%s]: error decoding packet from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error decoding packet from %s/%s", iata, pubkeyHex), "error", err) return } pubkeyBytes, err := hex.DecodeString(pubkeyHex) if err != nil { - log.Printf("ingest[%s]: invalid pubkey hex from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("invalid pubkey hex from %s/%s", iata, pubkeyHex), "error", err) return } - id, observerName, err := w.db.UpsertObserver(ctx, pubkeyBytes) + id, observerName, err := w.db.UpsertObserver(ctx, pubkeyBytes, iata) if err != nil { - log.Printf("ingest[%s]: db: upsert observer failed with packet from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert observer failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } - err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName) + err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, true) if err != nil { - log.Printf("ingest[%s]: db: update observer broker failed with packet from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: update observer broker failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } err = w.db.UpsertIATA(ctx, iata) if err != nil { - log.Printf("ingest[%s]: db: upsert IATA failed with packet from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert IATA failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } packetHash := packet.PacketHash() @@ -340,6 +356,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ var channelHash []byte originPubkey := []byte(nil) var parsedPayload json.RawMessage + var summary *string var traceTag []byte // For PayloadTypeTrace, packet.Path holds one SNR byte per hop (not hashes -- see // below), so the "physical route" hashes for resolvedPath/known-route purposes come @@ -410,8 +427,11 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ var name *string if hasName { - n := strings.ToValidUTF8(appData.Name, "\uFFFD") + n := cleanText(appData.Name) name = &n + if n != "" { + summary = name + } } deviceRole := int(flags & 0x0F) @@ -557,6 +577,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ SNRValues: snrValues, } parsedPayload, _ = json.Marshal(pt) + summary = new(traceType + " " + pt.TraceTag) // Each consecutive hop pair becomes a node_neighbors edge: // hop[i]'s measured SNR of receiving from hop[i-1]. hop[0]'s @@ -575,8 +596,8 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ currEntries := resolved[hashes[i]] if len(prevEntries) == 1 && len(currEntries) == 1 { snr := snrValues[i] - if err := w.db.UpsertNodeNeighbor(ctx, currEntries[0].NodeID, prevEntries[0].NodeID, iata, &snr); err != nil { - log.Printf("ingest[%s]: failed to upsert trace neighbor: %v", w.cfg.BrokerName, err) + if err := w.db.UpsertNodeNeighbor(ctx, currEntries[0].NodeID, prevEntries[0].NodeID, iata, &snr, nil); err != nil { + w.log.Error("failed to upsert trace neighbor", "error", err) } } } @@ -593,6 +614,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ Checksum: hex.EncodeToString(uint32ToBytes(ack.CRC())), } parsedPayload, _ = json.Marshal(pa) + summary = new("ACK " + pa.Checksum) } case meshcore.PayloadTypeMultiPart: @@ -655,8 +677,8 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ // don't insert our own node as a neighbor if oErr == nil && rErr == nil && observerNodeID != responderNodeID { rxSNR := float32(parseNumber(envelope.SNR)) - if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, responderNodeID, iata, &rxSNR); err != nil { - log.Printf("ingest[%s]: failed to upsert observer-discover neighbor: %v", w.cfg.BrokerName, err) + if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, responderNodeID, iata, &rxSNR, nil); err != nil { + w.log.Error("failed to upsert observer-discover neighbor", "error", err) } } } @@ -682,20 +704,13 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ var matchedScope *string if packet.RouteType() == meshcore.RouteTypeTransportFlood || packet.RouteType() == meshcore.RouteTypeTransportDirect { - for _, entry := range w.scopes.Entries() { - code := computeTransportCode(entry.TransportKey, packet.PayloadType(), packet.Payload) - if code == packet.TransportCode1 { - s := entry.Name - matchedScope = &s - break - } - } + matchedScope = matchTransportScope(w.scopes.Entries(), iata, packet.PayloadType(), packet.Payload, packet.TransportCode1) } var scopeID *int32 if matchedScope != nil { id, err := w.db.GetTransportScopeByName(ctx, *matchedScope) if err != nil { - log.Printf("ingest[%s]: failed to get scope ID for %s: %v", w.cfg.BrokerName, *matchedScope, err) + w.log.Error(fmt.Sprintf("failed to get scope ID for %s", *matchedScope), "error", err) } else { scopeID = &id } @@ -720,39 +735,33 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } isNew, err := w.db.UpsertPacket(ctx, pParams) if err != nil { - log.Printf("ingest[%s]: db: upsert packet failed from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert packet failed from %s/%s", iata, pubkeyHex), "error", err) return } - // Try parsing with timezone offset first - heardAt, err := time.Parse("2006-01-02T15:04:05.000000-07:00", envelope.Timestamp) - if err != nil { - heardAt, err = time.Parse("2006-01-02T15:04:05.000000", envelope.Timestamp) - } - if err != nil { - heardAt, err = time.Parse("2006-01-02T15:04:05.000000Z", envelope.Timestamp) - } + heardAt, err := parseHeardAt(envelope.Timestamp) if err != nil { - heardAt, err = time.Parse("2006-01-02T15:04:05", envelope.Timestamp) - } - if err != nil { - heardAt, err = time.Parse("2006-01-02T15:04:05Z", envelope.Timestamp) - } - if err != nil { - log.Printf("ingest[%s]: failed to parse timestamp %q: %v", w.cfg.BrokerName, envelope.Timestamp, err) + w.log.Debug(fmt.Sprintf("failed to parse timestamp %q", envelope.Timestamp), "error", err) heardAt = time.Now().UTC() } else { // clamp to server time if offset is suspicious (> 30 min drift) now := time.Now().UTC() diff := heardAt.UTC().Sub(now) if diff > 30*time.Minute || diff < -30*time.Minute { - log.Printf("ingest[%s]: clamping suspicious timestamp %s (diff %v) for pubkey %s", w.cfg.BrokerName, envelope.Timestamp, diff, pubkeyHex[:8]) + w.log.Debug(fmt.Sprintf("clamping suspicious timestamp %s (diff %v) for pubkey %s", envelope.Timestamp, diff, pubkeyHex)) heardAt = now } } radio, err := w.db.GetObserverRadio(ctx, id) if err != nil { - log.Printf("ingest[%s]: db: get observer radio failed for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: get observer radio failed for %s", pubkeyHex), "error", err) + } + // Airtime is costed from the frame as received; zero radio columns mean the + // observer never reported its settings, so there is nothing to cost. + var airtimeMs *float32 + if ms, ok := lora.TimeOnAirMs(len(hexBytes), int(radio.SF), float64(radio.BWKHz), int(radio.CR)); ok { + cost := float32(ms) + airtimeMs = &cost } oParams := InsertObservationParams{ PacketHash: packetHash[:], @@ -772,30 +781,36 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ CodingRate: radio.CR, SourceBroker: w.cfg.BrokerName, PayloadType: int16(packet.PayloadType()), + AirtimeMs: airtimeMs, } - inserted, err := w.db.InsertObservation(ctx, oParams) + inserted, observationCount, err := w.db.InsertObservation(ctx, oParams) if err != nil { - log.Printf("ingest[%s]: db: insert observation failed from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: insert observation failed from %s/%s", iata, pubkeyHex), "error", err) return } + // Record the path before any more DB work so a broker copy racing through it can't be + // streamed as a repeat of this hearing; TRACE reuses Path for per-hop SNR. + pathNew := packet.PayloadType() != meshcore.PayloadTypeTrace && w.hub.MarkSent(packetHash[:], id[:], packet.Path) + // A duplicate is streamed, never stored, to includeRepeats clients when its path is new. + repeat := !inserted && pathNew && w.hub.RepeatsWanted() if scopeID != nil && inserted { if err := w.db.UpsertObserverScope(ctx, id, *scopeID); err != nil { - log.Printf("ingest[%s]: failed to upsert observer scope for %s: %v", w.cfg.BrokerName, id, err) + w.log.Error(fmt.Sprintf("failed to upsert observer scope for %s", id), "error", err) } } // Runs on duplicate observations too; the upsert only writes when the row is >1h stale. if channelHash != nil { if err := w.db.UpsertChannelIATA(ctx, channelHash, iata, heardAt); err != nil { - log.Printf("ingest[%s]: db: upsert channel IATA failed from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert channel IATA failed from %s/%s", iata, pubkeyHex), "error", err) } } - // Runs on duplicate observations too; the upsert only writes when the row is >1h stale. + // Runs on duplicate observations too, so last_heard_at tracks every hearing. if traceTag != nil { - if err := w.db.UpsertTraceIATA(ctx, traceTag, iata, heardAt); err != nil { - log.Printf("ingest[%s]: db: upsert trace IATA failed from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + if err := w.db.RecordTrace(ctx, TraceHearing{TraceTag: traceTag, IATA: iata, HeardAt: heardAt}); err != nil { + w.log.Error(fmt.Sprintf("db: record trace failed from %s/%s", iata, pubkeyHex), "error", err) } } @@ -809,7 +824,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } resolved, err := w.db.ResolvePathHashes(ctx, iata, hashes) if err != nil { - log.Printf("ingest[%s]: path resolution failed: %v", w.cfg.BrokerName, err) + w.log.Error("path resolution failed", "error", err) } var resolvedIDs []uuid.UUID for _, entries := range resolved { @@ -833,44 +848,48 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } if allHigh && len(nodeIDs) > 1 { if err := w.db.UpsertKnownRoute(ctx, nodeIDs, hashPrefixes, iata, int32(len(nodeIDs))); err != nil { - log.Printf("ingest[%s]: failed to upsert known route: %v", w.cfg.BrokerName, err) + w.log.Error("failed to upsert known route", "error", err) } } } w.runCapabilityDetection(ctx, packet.PayloadType(), packet.PathHashSize(), resolvedIDs) - var resolvedSource, resolvedDestination *api.ResolvedHop - if packet.PayloadType() == meshcore.PayloadTypeAdvert && originPubkey != nil { - // Exact match: ADVERT carries the sender's real identity pubkey, not a - // short ambiguous hash prefix like the other resolvable payload types. - if nodeID, err := w.db.GetNodeByPubkey(ctx, originPubkey); err == nil { - if nodes, err := w.db.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { - hop := api.ResolveExactNode(nodes[nodeID]) + if inserted || repeat { + if inserted { + w.handlePayloadTypeSideEffects(ctx, packet, iata, packetHash[:], radio, scopeID, matchedScope, pubkeyBytes, float32(parseNumber(envelope.SNR))) + } + // Resolve after advert updates; suppressed copies need no endpoint lookup. + var resolvedSource, resolvedDestination *api.ResolvedHop + if packet.PayloadType() == meshcore.PayloadTypeAdvert && originPubkey != nil { + // Exact match: ADVERT carries the sender's real identity pubkey, not a + // short ambiguous hash prefix like the other resolvable payload types. + if nodeID, err := w.db.GetNodeByPubkey(ctx, originPubkey); err == nil { + if nodes, err := w.db.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { + hop := api.ResolveExactNode(nodes[nodeID]) + resolvedSource = &hop + } + } + } else if len(sourceHashByte) == 1 { + if r, err := w.db.ResolveEndpointHashes(ctx, iata, [][]byte{sourceHashByte}); err == nil { + hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] resolvedSource = &hop } } - } else if len(sourceHashByte) == 1 { - if r, err := w.db.ResolvePathHashes(ctx, iata, [][]byte{sourceHashByte}); err == nil { - hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] - resolvedSource = &hop - } - } - if len(destHashByte) == 1 { - if r, err := w.db.ResolvePathHashes(ctx, iata, [][]byte{destHashByte}); err == nil { - hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] - resolvedDestination = &hop + if len(destHashByte) == 1 { + if r, err := w.db.ResolveEndpointHashes(ctx, iata, [][]byte{destHashByte}); err == nil { + hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] + resolvedDestination = &hop + } } - } - - if inserted { - w.handlePayloadTypeSideEffects(ctx, packet, iata, packetHash[:], radio, scopeID, matchedScope, pubkeyBytes, float32(parseNumber(envelope.SNR))) evt := packetObservationEvent{} evt.PacketHash = hex.EncodeToString(packetHash[:]) evt.Packet.PayloadType = packet.PayloadType() evt.Packet.PayloadTypeName = packet.PayloadTypeString() evt.Packet.RouteType = packet.RouteType() evt.Packet.RouteTypeName = api.RouteTypeName(int16(packet.RouteType())) - evt.Packet.IsFirstObservation = isNew + evt.Packet.IsFirstObservation = isNew && !repeat + evt.Packet.IsRepeat = repeat + evt.Packet.Summary = summary evt.Observation.ObserverID = id.String() evt.Observation.ObserverName = observerName evt.Observation.IATA = iata @@ -883,22 +902,45 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ evt.Observation.PathLength.HashSize = packet.PathHashSize() evt.Observation.PathLength.HopCount = packet.PathHashCount() evt.Observation.PropagationTimeMs = 0 // not yet calculated - count, err := w.db.GetPacketObservationCount(ctx, packetHash[:]) - if err != nil { - log.Printf("ingest[%s]: failed to get observation count: %v", w.cfg.BrokerName, err) - count = 0 + if !repeat { + evt.Packet.ObservationCount = observationCount } - evt.Packet.ObservationCount = count if matchedScope != nil { evt.Packet.Scope = matchedScope } resolvedPath := api.BuildResolvedPath(hashes, resolved) evt.Observation.ResolvedSource = resolvedSource evt.Observation.ResolvedDestination = resolvedDestination - w.broadcastPacketObservation(iata, packet.PayloadType(), evt, resolvedPath) + w.broadcastPacketObservation(iata, packet.PayloadType(), evt, resolvedPath, hex.EncodeToString(pubkeyBytes), repeat) } } +// matchTransportScope preserves manual precedence; imported collisions remain unresolved. +func matchTransportScope(entries []scopestore.Entry, iata string, payloadType uint8, payload []byte, code uint16) *string { + var matched *string + ambiguous := false + for _, entry := range entries { + if entry.IATAs != nil && !slices.Contains(entry.IATAs, iata) { + continue + } + if computeTransportCode(entry.TransportKey, payloadType, payload) != code { + continue + } + name := entry.Name + if entry.IATAs == nil { + return &name + } + if matched != nil && *matched != name { + ambiguous = true + } + matched = &name + } + if ambiguous { + return nil + } + return matched +} + // computeTransportCode derives transport_code_1 from a transport key and packet payload. // code = HMAC-SHA256(key, payload_type_byte || payload)[0:2] as little-endian uint16. // Coerces reserved values 0x0000 → 0x0001 and 0xFFFF → 0xFFFE per §2.4. @@ -916,3 +958,25 @@ func computeTransportCode(key []byte, payloadType uint8, payload []byte) uint16 } return code } + +// heardAtLayouts lists offset forms first; timezone-less observers are read as UTC. +// Fractional seconds are optional in every layout. +var heardAtLayouts = []string{ + time.RFC3339Nano, + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02T15:04:05.999999999Z0700", + "2006-01-02 15:04:05.999999999Z0700", + "2006-01-02T15:04:05", + "2006-01-02 15:04:05", +} + +func parseHeardAt(s string) (time.Time, error) { + var err error + for _, layout := range heardAtLayouts { + var t time.Time + if t, err = time.Parse(layout, s); err == nil { + return t, nil + } + } + return time.Time{}, err +} diff --git a/internal/ingest/packet_endpoints_test.go b/internal/ingest/packet_endpoints_test.go new file mode 100644 index 00000000..ef37c9d0 --- /dev/null +++ b/internal/ingest/packet_endpoints_test.go @@ -0,0 +1,99 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +type endpointCaptureDB struct { + *stubDB + node api.ResolvedNode + observed int +} + +func (s *endpointCaptureDB) GetNodeByPubkey(context.Context, []byte) (uuid.UUID, error) { + return s.node.ID, nil +} +func (s *endpointCaptureDB) UpsertNode(context.Context, UpsertNodeParams, RadioSettings) (uuid.UUID, error) { + return s.node.ID, nil +} +func (s *endpointCaptureDB) GetNodesByIDs(context.Context, []uuid.UUID) (map[uuid.UUID]*api.ResolvedNode, error) { + return map[uuid.UUID]*api.ResolvedNode{s.node.ID: &s.node}, nil +} +func (s *endpointCaptureDB) ResolveEndpointHashes(_ context.Context, _ string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + if len(hashes) == 1 && hashes[0][0] == 0xaa { + return map[string][]api.ResolvedPathEntry{"aa": {{NodeID: s.node.ID, Name: s.node.Name, PublicKey: []byte{0xaa}}}}, nil + } + return nil, nil +} +func (s *endpointCaptureDB) InsertObservation(context.Context, InsertObservationParams) (bool, int64, error) { + s.observed++ + return true, 1, nil +} + +// Stored rows resolve endpoints at read time, but live events still carry them. +func TestHandlePacketBroadcastsEndpoints(t *testing.T) { + name := "Companion" + for _, kind := range []string{"advert", "direct message"} { + t.Run(kind, func(t *testing.T) { + w, base := newTestWorker() + db := &endpointCaptureDB{stubDB: base, node: api.ResolvedNode{ID: uuid.New(), Name: &name, PublicKey: "aa"}} + w.db = db + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + w.hub.AddScope(client, "endpoints", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + defer w.hub.Remove(client) + waitForSummarySubscriber(t, ctx, w.hub, client) + + packet := buildAdvertPacket(t, false) + wantSource := api.ResolveExactNode(&db.node) + var wantDestination *api.ResolvedHop + if kind == "direct message" { + packet = &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeTxtMsg, 0), Payload: append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...)} + wantSource = api.BuildResolvedPath([][]byte{{0xaa}}, map[string][]api.ResolvedPathEntry{"aa": {{NodeID: db.node.ID, Name: &name, PublicKey: []byte{0xaa}}}})[0] + d := api.BuildResolvedPath([][]byte{{0xbb}}, nil)[0] + wantDestination = &d + } + w.handlePacket(ctx, "YYZ", "0102", packetEnvelope(t, packet)) + if db.observed != 1 { + t.Fatalf("observation written %d times", db.observed) + } + for { + select { + case event := <-client.Send: + if event.Type != hub.EventPacketObservation { + continue + } + var got struct { + Observation struct { + ResolvedSource *api.ResolvedHop `json:"resolvedSource"` + ResolvedDestination *api.ResolvedHop `json:"resolvedDestination"` + } `json:"observation"` + } + if err := json.Unmarshal(event.Payload, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Observation.ResolvedSource, &wantSource) || !reflect.DeepEqual(got.Observation.ResolvedDestination, wantDestination) { + t.Fatalf("live endpoints: got %+v / %+v", got.Observation.ResolvedSource, got.Observation.ResolvedDestination) + } + return + case <-ctx.Done(): + t.Fatal("packet observation event missing") + } + } + }) + } +} diff --git a/internal/ingest/packet_summary_test.go b/internal/ingest/packet_summary_test.go new file mode 100644 index 00000000..7f17aba2 --- /dev/null +++ b/internal/ingest/packet_summary_test.go @@ -0,0 +1,213 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/meshcore-go/meshcore-go" +) + +// The hub has no subscription acknowledgement. Wait for a probe to make it +// through before ingesting the test packet, rather than assuming a short sleep. +func waitForSummarySubscriber(t *testing.T, ctx context.Context, h *hub.Hub, client *hub.Client) { + t.Helper() + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + for { + h.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + select { + case <-client.Send: + // A slow hub can leave more probes queued. Drain through a final + // marker so the next test read receives its packet, not a probe. + marker := json.RawMessage(`"` + rand.Text() + `"`) + h.Broadcast(hub.Event{Type: hub.EventObserverStatus, Payload: marker}) + for { + select { + case event := <-client.Send: + if string(event.Payload) == string(marker) { + return + } + case <-ctx.Done(): + t.Fatal("hub readiness probes did not drain") + } + } + case <-ticker.C: + case <-ctx.Done(): + t.Fatal("hub subscription did not become ready") + } + } +} + +func TestAdvertSummaryLive(t *testing.T) { + for _, tc := range []struct { + name string + hasName, tamper bool + want string + }{ + {"MD00-Repeater", true, false, "MD00-Repeater"}, + {"Relay 📡", true, false, "Relay 📡"}, + {"R\xffP", true, false, "R\uFFFDP"}, + {"", true, false, ""}, + {"", false, false, ""}, + {"unverified", true, true, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Run("wire", func(t *testing.T) { + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} + w.db = db + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + w.hub.AddScope(client, "summary", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + + waitForSummarySubscriber(t, ctx, w.hub, client) + defer w.hub.Remove(client) + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + identity, err := meshcore.NewIdentityFromBytes(pub) + if err != nil { + t.Fatal(err) + } + flags := byte(meshcore.AdvertTypeRepeater) + if tc.hasName { + flags |= 0x80 + } + advert := &meshcore.Advert{PublicKey: identity, Timestamp: 12345, RawAppData: append([]byte{flags}, []byte(tc.name)...)} + advert.Sign(priv) + if tc.tamper { + advert.RawAppData[0] ^= 1 + } + payload, err := advert.ToBytes() + if err != nil { + t.Fatal(err) + } + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeAdvert, 0), Payload: payload} + w.handlePacket(ctx, "YVR", "0102", packetEnvelope(t, packet)) + + for { + select { + case event := <-client.Send: + if event.Type != hub.EventPacketObservation { + continue + } + for _, raw := range []json.RawMessage{event.Payload, event.PayloadResolved} { + var got struct { + Packet struct { + Summary *string `json:"summary"` + } `json:"packet"` + } + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + if tc.want == "" { + if got.Packet.Summary != nil { + t.Fatalf("unexpected summary %q", *got.Packet.Summary) + } + } else if got.Packet.Summary == nil || *got.Packet.Summary != tc.want { + t.Fatalf("live summary = %v, want %q", got.Packet.Summary, tc.want) + } + } + return + case <-ctx.Done(): + t.Fatal("packet observation event missing") + } + } + }) + }) + } +} + +func TestPacketReferenceSummaryLive(t *testing.T) { + trace := func(hashes []byte) []byte { + t.Helper() + payload, err := (&meshcore.Trace{Tag: 0xdeadbeef, AuthCode: 0xfeedface, PathHashes: hashes}).ToBytes() + if err != nil { + t.Fatal(err) + } + return payload + } + for _, tc := range []struct { + name string + kind byte + payload []byte + want string + field string + }{ + {"ack", meshcore.PayloadTypeAck, []byte{1, 2, 3, 4}, "ACK 01020304", "checksum"}, + {"extended ack", meshcore.PayloadTypeAck, []byte{1, 2, 3, 4, 5, 6}, "ACK 01020304", "checksum"}, + {"zero ack", meshcore.PayloadTypeAck, []byte{0, 0, 0, 0}, "ACK 00000000", "checksum"}, + {"short ack", meshcore.PayloadTypeAck, []byte{1, 2, 3}, "", ""}, + {"trace", meshcore.PayloadTypeTrace, trace([]byte{1, 2}), "TRACE efbeadde", "traceTag"}, + {"empty trace path", meshcore.PayloadTypeTrace, trace(nil), "TRACE efbeadde", "traceTag"}, + {"ping", meshcore.PayloadTypeTrace, trace([]byte{1}), "PING efbeadde", "traceTag"}, + {"short trace", meshcore.PayloadTypeTrace, trace(nil)[:8], "", ""}, + {"unsupported", meshcore.PayloadTypeReq, []byte{1, 2, 3, 4}, "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} + w.db = db + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + w.hub.AddScope(client, "summary", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + defer w.hub.Remove(client) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, tc.kind, 0), Payload: tc.payload} + w.handlePacket(ctx, "YVR", "0102", packetEnvelope(t, packet)) + if len(db.packets) != 1 { + t.Fatal("packet was not stored") + } + if tc.want != "" { + var saved map[string]any + if err := json.Unmarshal(db.packets[0].ParsedPayload, &saved); err != nil { + t.Fatal(err) + } + if saved["type"].(string)+" "+saved[tc.field].(string) != tc.want { + t.Fatal("stored reference differs from the expected wire value") + } + } + for { + select { + case event := <-client.Send: + if event.Type != hub.EventPacketObservation { + continue + } + for _, raw := range []json.RawMessage{event.Payload, event.PayloadResolved} { + var got struct { + Packet struct { + Summary *string `json:"summary"` + } `json:"packet"` + } + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + if tc.want == "" { + if got.Packet.Summary != nil { + t.Fatalf("unexpected summary %q", *got.Packet.Summary) + } + } else if got.Packet.Summary == nil || *got.Packet.Summary != tc.want { + t.Fatalf("live summary = %v, want %q", got.Packet.Summary, tc.want) + } + } + return + case <-ctx.Done(): + t.Fatal("packet observation event missing") + } + } + }) + } +} diff --git a/internal/ingest/queue.go b/internal/ingest/queue.go new file mode 100644 index 00000000..6a537c53 --- /dev/null +++ b/internal/ingest/queue.go @@ -0,0 +1,120 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "log/slog" + "runtime/debug" + "strings" + "sync" + + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +type messageQueue struct { + mu sync.Mutex + lanes []chan mqtt.Message + bytes, maxBytes int + dropped uint64 + closed bool + done chan struct{} + cancel context.CancelFunc +} + +func newMessageQueue(workers, capacity, maxBytes int, handle func(context.Context, mqtt.Message)) *messageQueue { + ctx, cancel := context.WithCancel(context.Background()) + q := &messageQueue{lanes: make([]chan mqtt.Message, workers), maxBytes: maxBytes, done: make(chan struct{}), cancel: cancel} + var wg sync.WaitGroup + for i := range q.lanes { + lane := make(chan mqtt.Message, capacity) + q.lanes[i] = lane + wg.Go(func() { + for m := range lane { + if ctx.Err() != nil { + return + } + handleSafely(ctx, m, handle) + q.mu.Lock() + q.bytes -= len(m.Topic()) + len(m.Payload()) + q.mu.Unlock() + } + }) + } + go func() { wg.Wait(); close(q.done) }() + return q +} + +// handleSafely keeps one bad message from killing the lane, and with it the process. +func handleSafely(ctx context.Context, m mqtt.Message, handle func(context.Context, mqtt.Message)) { + defer func() { + if r := recover(); r != nil { + slog.Error("ingest handler panicked", "component", "ingest", "topic", m.Topic(), "panic", r, "stack", string(debug.Stack())) + } + }() + handle(ctx, m) +} + +func (q *messageQueue) enqueue(m mqtt.Message) bool { + // Keep status and packets from each observer in arrival order. + parts := strings.SplitN(m.Topic(), "/", 4) + key := m.Topic() + if len(parts) == 4 { + key = parts[2] + } + hash := uint32(2166136261) + for i := range len(key) { + c := key[i] + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + hash = (hash ^ uint32(c)) * 16777619 + } + size := len(m.Topic()) + len(m.Payload()) + q.mu.Lock() + defer q.mu.Unlock() + if q.closed { + return false + } + if size > q.maxBytes-q.bytes { + q.dropped++ + return false + } + select { + case q.lanes[int(hash%uint32(len(q.lanes)))] <- m: + q.bytes += size + return true + default: + q.dropped++ + return false + } +} + +func (q *messageQueue) takeDropped() uint64 { + q.mu.Lock() + defer q.mu.Unlock() + n := q.dropped + q.dropped = 0 + return n +} + +func (q *messageQueue) close(ctx context.Context) error { + q.mu.Lock() + if !q.closed { + q.closed = true + for _, lane := range q.lanes { + close(lane) + } + } + q.mu.Unlock() + select { + case <-q.done: + q.cancel() + return nil + case <-ctx.Done(): + q.cancel() + <-q.done + return ctx.Err() + } +} diff --git a/internal/ingest/queue_test.go b/internal/ingest/queue_test.go new file mode 100644 index 00000000..6b43fe49 --- /dev/null +++ b/internal/ingest/queue_test.go @@ -0,0 +1,183 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "fmt" + "reflect" + "sync" + "testing" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +type queuedTestMessage struct { + malformedTopicMessage + payload []byte +} + +func (m queuedTestMessage) Payload() []byte { return m.payload } +func queueMessage(observer, kind string, n int) mqtt.Message { + return queuedTestMessage{malformedTopicMessage{"meshcore/SEA/" + observer + "/" + kind}, []byte(fmt.Sprint(n))} +} + +func TestQueueKeepsObserverOrderWhileDatabaseIsBusy(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + var mu sync.Mutex + var got []string + q := newMessageQueue(2, 8, 4096, func(ctx context.Context, m mqtt.Message) { + if string(m.Payload()) == "0" { + close(entered) + select { + case <-release: + case <-ctx.Done(): + return + } + } + mu.Lock() + got = append(got, string(m.Payload())) + mu.Unlock() + }) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = q.close(ctx) + }) + if !q.enqueue(queueMessage("aa", "status", 0)) { + t.Fatal("first message rejected") + } + <-entered + for n := 1; n <= 3; n++ { + if !q.enqueue(queueMessage("AA", "packets", n)) { + t.Fatal("callback blocked or message rejected") + } + } + close(release) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := q.close(ctx); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, []string{"0", "1", "2", "3"}) { + t.Fatalf("observer order: %v", got) + } + if q.enqueue(queueMessage("aa", "packets", 4)) { + t.Fatal("accepted after shutdown") + } +} + +func TestQueueRunsOtherObserversAndReportsOverflow(t *testing.T) { + entered, other := make(chan struct{}), make(chan struct{}) + q := newMessageQueue(2, 1, 4096, func(ctx context.Context, m mqtt.Message) { + if m.Topic() == "meshcore/SEA/a/packets" { + close(entered) + <-ctx.Done() + } else { + close(other) + } + }) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = q.close(ctx) + }) + q.enqueue(queueMessage("a", "packets", 0)) + <-entered + if !q.enqueue(queueMessage("b", "packets", 0)) { + t.Fatal("other observer rejected") + } + select { + case <-other: + case <-time.After(time.Second): + t.Fatal("other observer blocked") + } + if !q.enqueue(queueMessage("a", "packets", 1)) { + t.Fatal("buffered message rejected") + } + if q.enqueue(queueMessage("a", "packets", 2)) { + t.Fatal("accepted beyond queue capacity") + } + if q.takeDropped() != 1 { + t.Fatal("overflow was not counted") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if q.close(ctx) == nil { + t.Fatal("expected shutdown deadline") + } +} + +func TestQueueBoundsPayloadMemory(t *testing.T) { + q := newMessageQueue(1, 8, 64, func(context.Context, mqtt.Message) {}) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = q.close(ctx) + }) + m := queuedTestMessage{malformedTopicMessage{"meshcore/SEA/a/packets"}, make([]byte, 65)} + if q.enqueue(m) { + t.Fatal("accepted oversized payload") + } + if q.takeDropped() != 1 { + t.Fatal("memory overflow was not counted") + } +} + +type subscribeCapture struct { + mqtt.Client + callback mqtt.MessageHandler +} + +func (c *subscribeCapture) Subscribe(_ string, _ byte, callback mqtt.MessageHandler) mqtt.Token { + c.callback = callback + return completedSubscribe{} +} + +type completedSubscribe struct{ mqtt.Token } + +func (completedSubscribe) Wait() bool { return true } +func (completedSubscribe) Error() error { return nil } + +func TestSubscribeCountsRejectedMessage(t *testing.T) { + w, _ := newTestWorker() + q := newMessageQueue(1, 1, 1, func(context.Context, mqtt.Message) { t.Error("rejected message processed") }) + defer q.close(context.Background()) + client := &subscribeCapture{} + w.subscribe(client, q) + client.callback(client, queueMessage("aa", "packets", 0)) + if q.takeDropped() != 1 { + t.Fatal("rejected message was not counted") + } +} + +func TestQueueShutdownWaitsForCanceledHandler(t *testing.T) { + entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + q := newMessageQueue(1, 1, 4096, func(ctx context.Context, _ mqtt.Message) { + close(entered) + <-ctx.Done() + close(canceled) + <-release + }) + q.enqueue(queueMessage("aa", "packets", 0)) + <-entered + ctx, cancel := context.WithCancel(context.Background()) + cancel() + stopped := make(chan error, 1) + go func() { stopped <- q.close(ctx) }() + <-canceled + select { + case <-stopped: + close(release) + t.Fatal("shutdown returned while handler was still running") + case <-time.After(20 * time.Millisecond): + } + close(release) + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("shutdown did not finish") + } +} diff --git a/internal/ingest/repeats_test.go b/internal/ingest/repeats_test.go new file mode 100644 index 00000000..2ad85a98 --- /dev/null +++ b/internal/ingest/repeats_test.go @@ -0,0 +1,198 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/google/uuid" +) + +type repeatHarness struct { + t *testing.T + ctx context.Context + w *Worker + db *stubDB + client *hub.Client +} + +func newRepeatHarness(t *testing.T, includeRepeats bool) *repeatHarness { + w, db := newTestWorker() + db.observationCount = 3 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + go w.hub.Run() + client := w.hub.NewClient() + t.Cleanup(func() { w.hub.Remove(client) }) + w.hub.AddScope(client, "all", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + if includeRepeats { + w.hub.Configure(client, hub.ClientOptions{IncludeRepeats: true}) + for !w.hub.RepeatsWanted() { + time.Sleep(time.Millisecond) + } + } + waitForSummarySubscriber(t, ctx, w.hub, client) + return &repeatHarness{t: t, ctx: ctx, w: w, db: db, client: client} +} + +// hear feeds one hearing of the same group text over path and returns the packetObservation +// payloads it produced. +func (r *repeatHarness) hear(inserted bool, path ...byte) []json.RawMessage { + r.t.Helper() + packet := buildGrpTxtPacket(r.t, 0x1a, make([]byte, 16)) + packet.Path = path + packet.PathLength = byte(len(path)) + r.db.observationInserted = inserted + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(r.t, packet)) + + // Hub delivery is FIFO, so everything handlePacket sent arrives before this marker. + r.w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + var got []json.RawMessage + for { + select { + case evt := <-r.client.Send: + if evt.Type == hub.EventObserverStatus { + return got + } + got = append(got, evt.Payload) + case <-r.ctx.Done(): + r.t.Fatal("marker event not delivered") + } + } +} + +func packetFields(t *testing.T, raw json.RawMessage) map[string]json.RawMessage { + t.Helper() + var evt struct { + Packet map[string]json.RawMessage `json:"packet"` + } + if err := json.Unmarshal(raw, &evt); err != nil { + t.Fatal(err) + } + return evt.Packet +} + +func TestHandlePacket_Repeats(t *testing.T) { + r := newRepeatHarness(t, true) + + first := r.hear(true, 0x11) + if len(first) != 1 || strings.Contains(string(first[0]), "isRepeat") { + t.Fatalf("first hearing: %s", first) + } + if got := string(packetFields(t, first[0])["observationCount"]); got != "3" { + t.Fatalf("first hearing observationCount = %s, want the count InsertObservation returned", got) + } + if got := r.hear(false, 0x11); len(got) != 0 { + t.Fatalf("broker copy of the first hearing sent: %s", got) + } + + repeat := r.hear(false, 0x22, 0x11) + if len(repeat) != 1 { + t.Fatalf("new path sent %d events", len(repeat)) + } + fields := packetFields(t, repeat[0]) + if string(fields["isRepeat"]) != "true" || string(fields["isFirstObservation"]) != "false" || string(fields["observationCount"]) != "0" { + t.Fatalf("repeat packet fields: %v", fields) + } + if got := r.hear(false, 0x22, 0x11); len(got) != 0 { + t.Fatalf("same path sent twice: %s", got) + } +} + +func TestHandlePacket_RepeatsOffByDefault(t *testing.T) { + r := newRepeatHarness(t, false) + if got := r.hear(true, 0x11); len(got) != 1 || strings.Contains(string(got[0]), "isRepeat") { + t.Fatalf("first hearing: %s", got) + } + if got := r.hear(false, 0x22, 0x11); len(got) != 0 { + t.Fatalf("repeat sent with nobody opted in: %s", got) + } +} + +// A broker copy racing through the inserted hearing's DB work must find its path already +// recorded, otherwise it would be streamed as a repeat of the first hearing. +func TestHandlePacket_PathMarkedRightAfterInsert(t *testing.T) { + r := newRepeatHarness(t, true) + packet := buildGrpTxtPacket(t, 0x1a, make([]byte, 16)) + packet.Path = []byte{0x11} + packet.PathLength = 1 + hash := packet.PacketHash() + var repeat bool + r.db.dbHook = func() { + // stubDB.UpsertObserver returns uuid.Nil, so that is the observer id ingest marks with + repeat = r.w.hub.MarkSent(hash[:], uuid.Nil[:], []byte{0x22}) + } + r.db.observationInserted = true + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(r.t, packet)) + if !repeat { + t.Fatal("path was not marked right after the insert") + } +} + +// A broker copy can lose the insert race yet mark its path first; it is still the first hearing. +func TestHandlePacket_BrokerCopyMarkedFirstIsNotARepeat(t *testing.T) { + r := newRepeatHarness(t, true) + if got := r.hear(false, 0x11); len(got) != 0 { + t.Fatalf("broker copy streamed as a repeat: %s", got) + } + if got := r.hear(true, 0x11); len(got) != 1 || strings.Contains(string(got[0]), "isRepeat") { + t.Fatalf("inserted hearing: %s", got) + } + if got := r.hear(false, 0x22, 0x11); len(got) != 1 { + t.Fatalf("later path sent %d events", len(got)) + } +} + +func TestHandlePacket_PathsRecordedWithoutSubscribers(t *testing.T) { + r := newRepeatHarness(t, false) + if got := r.hear(true, 0x11); len(got) != 1 { + t.Fatalf("first hearing: %s", got) + } + r.w.hub.Configure(r.client, hub.ClientOptions{IncludeRepeats: true}) + for !r.w.hub.RepeatsWanted() { + if r.ctx.Err() != nil { + t.Fatal("repeats opt-in never registered") + } + time.Sleep(time.Millisecond) + } + if got := r.hear(false, 0x11); len(got) != 0 { + t.Fatalf("same path streamed as repeat after late opt-in: %s", got) + } +} + +func TestHandlePacket_TraceNeverRepeats(t *testing.T) { + r := newRepeatHarness(t, true) + packet := buildTracePacket(t) + packet.Path = []byte{0x05, 0x06} + packet.PathLength = 2 + r.db.observationInserted = true + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(r.t, packet)) + r.db.observationInserted = false + packet.Path = append([]byte{}, packet.Path...) + packet.Path[0] ^= 0xff // a different per-hop SNR byte, same route + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(r.t, packet)) + r.w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + repeats := 0 + for { + select { + case evt := <-r.client.Send: + if evt.Type == hub.EventObserverStatus { + if repeats != 0 { + t.Fatalf("TRACE copy streamed as %d repeat(s)", repeats) + } + return + } + if strings.Contains(string(evt.Payload), "isRepeat") { + repeats++ + } + case <-r.ctx.Done(): + t.Fatal("marker event not delivered") + } + } +} diff --git a/internal/ingest/sanitize_test.go b/internal/ingest/sanitize_test.go new file mode 100644 index 00000000..dc269866 --- /dev/null +++ b/internal/ingest/sanitize_test.go @@ -0,0 +1,102 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +type statusCaptureDB struct { + *stubDB + status []UpdateObserverStatusParams +} + +func (s *statusCaptureDB) UpdateObserverStatus(_ context.Context, p UpdateObserverStatusParams) (uuid.UUID, error) { + s.status = append(s.status, p) + return uuid.Nil, nil +} + +func TestHandleStatus_RejectsInsaneRadio(t *testing.T) { + for _, tc := range []struct { + radio string + freq, bw float32 + }{ + {"910.5,62.5,7,5", 910.5, 62.5}, + {"910.5,NaN,7,5", 910.5, 0}, + {"Inf,62.5,7,5", 0, 62.5}, + {"910.5,1e-40,7,5", 910.5, 0}, + {"-910.5,-62.5,7,5", 0, 0}, + {"1e9,1e9,7,5", 0, 0}, + } { + t.Run(tc.radio, func(t *testing.T) { + w, base := newTestWorker() + db := &statusCaptureDB{stubDB: base} + w.db = db + w.handleStatus(context.Background(), strings.Repeat("01", 32), []byte(`{"radio":"`+tc.radio+`"}`)) + if len(db.status) != 1 { + t.Fatal("status not stored") + } + if got := db.status[0]; got.RadioFreqMHz != tc.freq || got.RadioBWKHz != tc.bw { + t.Fatalf("radio stored as %v MHz / %v kHz, want %v / %v", got.RadioFreqMHz, got.RadioBWKHz, tc.freq, tc.bw) + } + }) + } +} + +func TestHandlePacket_AdvertNameNULsStripped(t *testing.T) { + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} + w.db = db + data := append([]byte{meshcore.AdvertTypeRepeater | meshcore.AdvertNameMask}, "Base\x00Camp\x00\x00"...) + w.handlePacket(context.Background(), "YOW", "0102", packetEnvelope(t, buildAdvertPacketWithData(t, data, false))) + if len(db.packets) != 1 || bytes.Contains(db.packets[0].ParsedPayload, []byte(`\u0000`)) { + t.Fatalf("parsed payload kept NULs: %s", db.packets[0].ParsedPayload) + } + if base.upsertNodeCalls != 1 || base.upsertNodeParams.Name != "BaseCamp" { + t.Fatalf("node name stored as %q", base.upsertNodeParams.Name) + } +} + +type neighborCaptureDB struct { + *stubDB + selfScope string + scopes []string +} + +func (s *neighborCaptureDB) GetNodeByPubkey(_ context.Context, pubkey []byte) (uuid.UUID, error) { + if len(pubkey) == 0 { + return uuid.Nil, errors.New("not found") + } + return uuid.UUID{pubkey[0]}, nil +} + +func (s *neighborCaptureDB) UpdateObserverRegionScope(_ context.Context, _ uuid.UUID, scope string) error { + s.selfScope = scope + return nil +} + +func (s *neighborCaptureDB) UpsertNodeNeighbor(_ context.Context, _, _ uuid.UUID, _ string, _ *float32, scope *string) error { + if scope != nil { + s.scopes = append(s.scopes, *scope) + } + return nil +} + +func TestHandleNeighbors_ScopeNULsStripped(t *testing.T) { + w, base := newTestWorker() + db := &neighborCaptureDB{stubDB: base} + w.db = db + report := `{"self":{"scopes":"#ott\u0000"},"neighbors":[{"pubkey":"` + strings.Repeat("02", 32) + `","status":"responded","scopes":"#y\u0000ow"}]}` + w.handleNeighbors(context.Background(), "YOW", strings.Repeat("01", 32), []byte(report)) + if db.selfScope != "#ott" || len(db.scopes) != 1 || db.scopes[0] != "#yow" { + t.Fatalf("scopes stored as %q %q", db.selfScope, db.scopes) + } +} diff --git a/internal/ingest/scope_import_test.go b/internal/ingest/scope_import_test.go new file mode 100644 index 00000000..4474f135 --- /dev/null +++ b/internal/ingest/scope_import_test.go @@ -0,0 +1,113 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "fmt" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/meshcore-go/meshcore-go" +) + +func TestTransportScopeRegionalAndAmbiguousMatches(t *testing.T) { + entry := scopestore.FromName("yow") + entry.IATAs = []string{"YOW"} + payload := []byte{0xde, 0xad, 0xbe, 0xef} + // Fixed HMAC-SHA256 fixture: SHA256("#yow")[:16], payload type 4. + if got := matchTransportScope([]scopestore.Entry{entry}, "YOW", 4, payload, 21698); got == nil || *got != "#yow" { + t.Fatal(got) + } + if got := matchTransportScope([]scopestore.Entry{entry}, "YVR", 4, payload, 21698); got != nil { + t.Fatal("imported key escaped source region") + } + entry.IATAs = nil + if got := matchTransportScope([]scopestore.Entry{entry}, "YVR", 4, payload, 21698); got == nil { + t.Fatal("manual key was region-restricted") + } + // Distinct keys with the same short code for this payload must remain unknown. + a, b := scopestore.FromName("collision-320"), scopestore.FromName("collision-322") + a.IATAs, b.IATAs = []string{"YOW"}, []string{"YOW"} + for _, entries := range [][]scopestore.Entry{{a, b}, {b, a}} { + if computeTransportCode(entries[0].TransportKey, 4, payload) != 37018 { + t.Fatal("collision fixture changed") + } + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got != nil { + t.Fatal("ambiguous short code resolved", *got) + } + } +} + +type importCaptureDB struct{ *frameCaptureDB } + +func (s *importCaptureDB) GetTransportScopeByName(context.Context, string) (int32, error) { + return 123, nil +} + +func TestImportedScopeFlowsThroughPacketIngest(t *testing.T) { + for _, iata := range []string{"YOW", "YVR"} { + t.Run(iata, func(t *testing.T) { + w, base := newTestWorker() + capture := &importCaptureDB{&frameCaptureDB{stubDB: base}} + w.db = capture + key := scopestore.FromName("yow") + key.IATAs = []string{"YOW"} + store := scopestore.New() + store.Load([]scopestore.Entry{key}) + w.scopes = store + packet := buildGrpTxtPacket(t, 0x1a, make([]byte, 16)) + packet.Header = meshcore.MakeHeader(meshcore.RouteTypeTransportFlood, meshcore.PayloadTypeGrpTxt, 0) + packet.TransportCode1 = computeTransportCode(key.TransportKey, packet.PayloadType(), packet.Payload) + w.handlePacket(context.Background(), iata, "0102", packetEnvelope(t, packet)) + if len(capture.packets) != 1 { + t.Fatal("packet not stored") + } + got := capture.packets[0].ScopeID + if iata == "YOW" && (got == nil || *got != 123) { + t.Fatal("scope not persisted", got) + } + if iata == "YVR" && got != nil { + t.Fatal("scope assigned outside source region", got) + } + }) + } +} + +func BenchmarkTransportScopeCatalogue(b *testing.B) { + for _, count := range []int{0, 64, 1024} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + entries := []scopestore.Entry{scopestore.FromName("manual")} + for n := 0; n < count; n++ { + entry := scopestore.FromName(fmt.Sprintf("scope-%d", n)) + entry.IATAs = []string{fmt.Sprintf("R%02d", n/64)} + entries = append(entries, entry) + } + store := scopestore.New() + store.Load(entries) + payload := make([]byte, 128) + b.ReportAllocs() + for b.Loop() { + matchTransportScope(store.Entries(), "R00", 4, payload, 1234) + } + }) + } +} + +func TestManualScopeWinsTransportCodeCollision(t *testing.T) { + payload := []byte{0xde, 0xad, 0xbe, 0xef} + manual, imported := scopestore.FromName("collision-320"), scopestore.FromName("collision-322") + imported.IATAs = []string{"YOW"} + for _, entries := range [][]scopestore.Entry{{manual, imported}, {imported, manual}, {imported, scopestore.Entry{Name: "ambiguous", TransportKey: imported.TransportKey, IATAs: []string{"YOW"}}, manual}} { + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got == nil || *got != manual.Name { + t.Fatalf("manual match lost to imported collision: %v", got) + } + } + imported.IATAs = nil + for _, entries := range [][]scopestore.Entry{{manual, imported}, {imported, manual}} { + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got == nil || *got != entries[0].Name { + t.Fatalf("manual-only first-match behavior changed: %v", got) + } + } +} diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 9a78ab19..3299027f 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -7,8 +7,6 @@ import ( "context" "encoding/hex" "fmt" - "log" - "strings" "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" @@ -23,6 +21,8 @@ type UpsertNodeParams struct { NodeType uint8 // 1=companion, 2=repeater, 3=room server Latitude *float64 Longitude *float64 + // ClearLocation drops the stored position; set by an explicit 0/0 advert. + ClearLocation bool // AdvertTimestamp is the device's self-reported wall-clock time (epoch seconds) from the // signed advert body. Used to derive clock drift for repeaters/room servers; see // api.Node.ClockDriftSeconds. @@ -36,16 +36,27 @@ type InsertChannelMessageParams struct { SenderName string Content string SentAt time.Time + Historical bool // backfilled: its hours may already be rolled, so queue them for a re-roll +} + +// InsertedChannelMessage is returned only for a new message, using its stored packet evidence. +type InsertedChannelMessage struct { + ID int64 + Scope *string + ScopeStatus api.ChannelScopeStatus } // channelMessageEvent is the JSON payload for a channelMessage WS event. type channelMessageEvent struct { - ChannelID int `json:"channelId"` - ChannelHash string `json:"channelHash"` // hex-encoded single byte - PacketHash string `json:"packetHash"` // hex-encoded - SenderName string `json:"senderName"` - Content string `json:"content"` - SentAt int64 `json:"sentAt"` // epoch ms + ID int64 `json:"id"` + ChannelID int `json:"channelId"` + ChannelHash string `json:"channelHash"` // hex-encoded single byte + PacketHash string `json:"packetHash"` // hex-encoded + SenderName string `json:"senderName"` + Content string `json:"content"` + SentAt int64 `json:"sentAt"` // epoch ms + Scope *string `json:"scope"` + ScopeStatus api.ChannelScopeStatus `json:"scopeStatus" enums:"matched,unscoped,unknown,unavailable"` } // nodeUpdateEvent is the JSON payload for a nodeUpdate WS event. @@ -56,12 +67,15 @@ type nodeUpdateEvent struct { NodeType uint8 `json:"nodeType"` NodeTypeName string `json:"nodeTypeName"` IATA string `json:"iata"` - Lat *float64 `json:"lat,omitempty"` - Lng *float64 `json:"lng,omitempty"` + Lat **float64 `json:"lat,omitempty"` // Omitted keeps the client's position; explicit null clears it. + Lng **float64 `json:"lng,omitempty"` IsObserver bool `json:"isObserver"` IATAs []api.NodeIATA `json:"iatas"` DefaultScope *string `json:"defaultScope,omitempty"` Radio *string `json:"radio,omitempty"` + // Omission retains the previous classification (no advertised position). + // The outer pointer allows explicit null to clear an unknown/reset position. + PossiblyForeign **bool `json:"possiblyForeign,omitempty"` } // handlePayloadTypeSideEffects runs payload-type-specific processing after a @@ -72,15 +86,18 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc if packet.PayloadType() == meshcore.PayloadTypeAdvert { advert, err := meshcore.AdvertFromBytes(packet.Payload) if err != nil { - log.Printf("ingest[%s]: error decoding advert payload: %v", w.cfg.BrokerName, err) + w.log.Warn("error decoding advert payload", "error", err) return } if !advert.Verify() { - log.Printf("ingest[%s]: dropped advert with invalid signature from pubkey %s", w.cfg.BrokerName, hex.EncodeToString(advert.PublicKey.PublicKeyBytes())) + w.log.Warn(fmt.Sprintf("dropped advert with invalid signature from pubkey %s", hex.EncodeToString(advert.PublicKey.PublicKeyBytes()))) return } var lat, lon *float64 - if advert.AppData().Lat != 0 || advert.AppData().Lon != 0 { + present := advert.Flags()&meshcore.AdvertLatLonMask != 0 + // Presence decides: omitted keeps the stored position, an explicit 0/0 clears it. + cleared := present && advert.AppData().Lat == 0 && advert.AppData().Lon == 0 + if present && !cleared { la := float64(advert.AppData().Lat) / 1e6 lo := float64(advert.AppData().Lon) / 1e6 lat = &la @@ -88,10 +105,11 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc } params := UpsertNodeParams{ PublicKey: advert.PublicKey.PublicKeyBytes(), - Name: strings.ToValidUTF8(advert.AppData().Name, "\uFFFD"), + Name: cleanText(advert.AppData().Name), NodeType: advert.Type(), Latitude: lat, Longitude: lon, + ClearLocation: cleared, AdvertTimestamp: advert.Timestamp, } var nodeRadio RadioSettings @@ -100,7 +118,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc } nodeID, err := w.db.UpsertNode(ctx, params, nodeRadio) if err != nil { - log.Printf("ingest[%s]: db: upsert node failed: %v", w.cfg.BrokerName, err) + w.log.Error("db: upsert node failed", "error", err) return } // invalidate cache for this node @@ -116,8 +134,8 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc if err == nil { key := hex.EncodeToString(firstHop[0]) if entries := resolved[key]; len(entries) == 1 { - if err := w.db.UpsertNodeNeighbor(ctx, nodeID, entries[0].NodeID, iata, nil); err != nil { - log.Printf("ingest[%s]: failed to upsert node neighbor: %v", w.cfg.BrokerName, err) + if err := w.db.UpsertNodeNeighbor(ctx, nodeID, entries[0].NodeID, iata, nil, nil); err != nil { + w.log.Error("failed to upsert node neighbor", "error", err) } } } @@ -131,22 +149,22 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc observerNodeID, oErr := w.db.GetNodeByPubkey(ctx, observerPubkey) if oErr == nil && observerNodeID != nodeID { snr := rxSNR - if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, nodeID, iata, &snr); err != nil { - log.Printf("ingest[%s]: failed to upsert observer-advert neighbor: %v", w.cfg.BrokerName, err) + if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, nodeID, iata, &snr, nil); err != nil { + w.log.Error("failed to upsert observer-advert neighbor", "error", err) } } } if err := w.db.UpsertNodeIATA(ctx, nodeID, iata); err != nil { - log.Printf("ingest[%s]: db: upsert node IATA failed: %v", w.cfg.BrokerName, err) + w.log.Error("db: upsert node IATA failed", "error", err) } if scopeID != nil && (packet.RouteType() == meshcore.RouteTypeTransportFlood || packet.RouteType() == meshcore.RouteTypeTransportDirect) { if err := w.db.SetNodeDefaultScope(ctx, nodeID, *scopeID); err != nil { - log.Printf("ingest[%s]: failed to set default scope for node %s: %v", w.cfg.BrokerName, hex.EncodeToString(advert.PublicKey.PublicKeyBytes()), err) + w.log.Error(fmt.Sprintf("failed to set default scope for node %s", hex.EncodeToString(advert.PublicKey.PublicKeyBytes())), "error", err) } } prefix4 := advert.PublicKey.PublicKeyBytes()[:4] if err := w.db.UpsertNodeShortID(ctx, nodeID, iata, prefix4); err != nil { - log.Printf("ingest[%s]: failed to upsert node short ID for %s: %v", w.cfg.BrokerName, hex.EncodeToString(prefix4), err) + w.log.Error(fmt.Sprintf("failed to upsert node short ID for %s", hex.EncodeToString(prefix4)), "error", err) } pubkeyHex := hex.EncodeToString(advert.PublicKey.PublicKeyBytes()) isObserver := w.db.IsObserverByPubkey(ctx, advert.PublicKey.PublicKeyBytes()) @@ -162,31 +180,36 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc evt := nodeUpdateEvent{ NodeID: nodeID.String(), PublicKey: pubkeyHex, - Name: advert.AppData().Name, + Name: cleanText(advert.AppData().Name), NodeType: advert.Type(), NodeTypeName: api.NodeTypeName(int16(advert.Type())), IATA: iata, - Lat: lat, - Lng: lon, IsObserver: isObserver, IATAs: []api.NodeIATA{{IATA: iata, LastHeard: time.Now().UnixMilli()}}, DefaultScope: defaultScope, Radio: radioStr, } + if present { + evt.Lat, evt.Lng = &lat, &lon + } + if w.cfg.LocalBorders != nil && (present || advert.Type() != meshcore.AdvertTypeRepeater) { + foreign := w.cfg.LocalBorders.PossiblyForeign(int16(advert.Type()), lat, lon) + evt.PossiblyForeign = &foreign + } w.broadcast(hub.EventNodeUpdate, iata, meshcore.PayloadTypeAdvert, "", evt) return } if packet.PayloadType() == meshcore.PayloadTypeGrpTxt { grpTxt, err := meshcore.GroupTextFromBytes(packet.Payload) if err != nil { - log.Printf("ingest[%s]: error decoding group text payload: %v", w.cfg.BrokerName, err) + w.log.Warn("error decoding group text payload", "error", err) return } channelHashBytes := []byte{grpTxt.ChannelHash} - result, err := DecryptGroupText(ctx, w.db, w.keys, packetHash, packet.Payload) + result, err := DecryptGroupText(ctx, w.db, w.keys, packetHash, packet.Payload, false) if err != nil { - log.Printf("ingest[%s]: decrypt group text failed: %v", w.cfg.BrokerName, err) + w.log.Error("decrypt group text failed", "error", err) return } if result == nil { @@ -198,13 +221,16 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc return } - if result.NewMessage { + if result.Message != nil { evt := channelMessageEvent{ + ID: result.Message.ID, + Scope: result.Message.Scope, + ScopeStatus: result.Message.ScopeStatus, ChannelID: result.ChannelID, ChannelHash: hex.EncodeToString(channelHashBytes), PacketHash: hex.EncodeToString(packetHash), - SenderName: strings.ReplaceAll(strings.ToValidUTF8(result.Payload.Sender, "\uFFFD"), "\x00", ""), - Content: strings.ReplaceAll(strings.ToValidUTF8(result.Payload.Text, "\uFFFD"), "\x00", ""), + SenderName: cleanText(result.Payload.Sender), + Content: cleanText(result.Payload.Text), SentAt: time.Unix(int64(result.Payload.Timestamp), 0).UnixMilli(), } w.broadcast(hub.EventChannelMessage, iata, 0, fmt.Sprintf("%02x", grpTxt.ChannelHash), evt) diff --git a/internal/ingest/side_effects_test.go b/internal/ingest/side_effects_test.go index 25bdb2d3..d6e6fd53 100644 --- a/internal/ingest/side_effects_test.go +++ b/internal/ingest/side_effects_test.go @@ -6,11 +6,14 @@ package ingest import ( "context" "crypto/ed25519" + "encoding/binary" "encoding/hex" "encoding/json" + "strings" "testing" "time" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" "github.com/MeshCore-Beacon/beacon-server/internal/keystore" "github.com/meshcore-go/meshcore-go" ) @@ -31,6 +34,10 @@ func (k *mapKeys) GetKey(hash []byte) []keystore.Entry { // buildAdvertPacket signs (or, if tamper is true, signs then mutates) an // advert payload and wraps it in a minimal Packet with no path (zero-hop). func buildAdvertPacket(t *testing.T, tamper bool) *meshcore.Packet { + return buildAdvertPacketWithData(t, []byte{meshcore.AdvertTypeRepeater}, tamper) +} + +func buildAdvertPacketWithData(t *testing.T, data []byte, tamper bool) *meshcore.Packet { t.Helper() pub, priv, err := ed25519.GenerateKey(nil) if err != nil { @@ -43,7 +50,7 @@ func buildAdvertPacket(t *testing.T, tamper bool) *meshcore.Packet { advert := &meshcore.Advert{ PublicKey: id, Timestamp: 12345, - RawAppData: []byte{meshcore.AdvertTypeRepeater}, // flags byte only, no optional fields + RawAppData: data, } advert.Sign(priv) if tamper { @@ -61,6 +68,109 @@ func buildAdvertPacket(t *testing.T, tamper bool) *meshcore.Packet { } } +func TestAdvertLocationPresence(t *testing.T) { + for _, tc := range []struct { + name string + present, tamper, clear bool + lat, lon int32 + }{ + {"ordinary location", true, false, false, 45000000, -75000000}, + {"explicit reset", true, false, true, 0, 0}, + {"zero latitude", true, false, false, 0, -75000000}, + {"zero longitude", true, false, false, 45000000, 0}, + {"no location", false, false, false, 0, 0}, + {"tampered reset", true, true, false, 0, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + data := []byte{meshcore.AdvertTypeRepeater} + if tc.present { + // Preserve the wire presence bit even for zero coordinates; the + // library's AppData encoder omits location when both values are zero. + data[0] |= meshcore.AdvertLatLonMask + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lat)) + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lon)) + } + worker, store := newTestWorker() + packet := buildAdvertPacketWithData(t, data, tc.tamper) + worker.handlePayloadTypeSideEffects(context.Background(), packet, "YOW", []byte{1}, RadioSettings{}, nil, nil, nil, 0) + if tc.tamper { + if store.upsertNodeCalls != 0 { + t.Fatal("invalid signature updated the node") + } + return + } + if store.upsertNodeCalls != 1 { + t.Fatal("signed advert did not update the node") + } + got := store.upsertNodeParams + if got.ClearLocation != tc.clear { + t.Fatalf("ClearLocation = %v, want %v", got.ClearLocation, tc.clear) + } + if !tc.present || tc.clear { + if got.Latitude != nil || got.Longitude != nil { + t.Fatal("absent location became an update") + } + return + } + if got.Latitude == nil || got.Longitude == nil || *got.Latitude != float64(tc.lat)/1e6 || *got.Longitude != float64(tc.lon)/1e6 { + t.Fatal("advertised coordinates did not reach the node update") + } + }) + } +} + +func TestAdvertClearEmitsNullLocation(t *testing.T) { + w, _ := newTestWorker() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "clear", hub.Scope{Events: []hub.EventType{hub.EventNodeUpdate, hub.EventObserverStatus}}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + waitForSummarySubscriber(t, ctx, w.hub, client) + for _, tc := range []struct { + name string + present, want bool + lat, lon int32 + wantLat string + wantLng string + }{ + {"explicit reset sends null", true, true, 0, 0, "null", "null"}, + {"omission sends nothing", false, false, 0, 0, "", ""}, + {"real location sends coordinates", true, true, 45000000, -75000000, "45", "-75"}, + } { + t.Run(tc.name, func(t *testing.T) { + data := []byte{meshcore.AdvertTypeRepeater} + if tc.present { + data[0] |= meshcore.AdvertLatLonMask + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lat)) + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lon)) + } + w.handlePayloadTypeSideEffects(ctx, buildAdvertPacketWithData(t, data, false), "AAA", []byte{1}, RadioSettings{}, nil, nil, nil, 0) + for { + select { + case event := <-client.Send: + if event.Type != hub.EventNodeUpdate { + continue + } + payload := string(event.Payload) + for field, want := range map[string]string{`"lat"`: tc.wantLat, `"lng"`: tc.wantLng} { + if tc.want && !strings.Contains(payload, field+":"+want+",") { + t.Fatalf("%s != %s: %s", field, want, payload) + } + if !tc.want && strings.Contains(payload, field) { + t.Fatalf("%s present on omission: %s", field, payload) + } + } + return + case <-ctx.Done(): + t.Fatal("node update not received") + } + } + }) + } +} + func TestHandlePayloadTypeSideEffects_Advert_ValidSignature_UpsertsNode(t *testing.T) { w, db := newTestWorker() packet := buildAdvertPacket(t, false) @@ -187,14 +297,14 @@ func TestHandlePacket_Advert_SkipsChannelIATA(t *testing.T) { if db.upsertChannelIATACalls != 0 { t.Errorf("expected UpsertChannelIATA NOT to be called for a non-channel packet, got %d calls", db.upsertChannelIATACalls) } - if db.upsertTraceIATACalls != 0 { - t.Errorf("expected UpsertTraceIATA NOT to be called for a non-trace packet, got %d calls", db.upsertTraceIATACalls) + if len(db.traceHearings) != 0 { + t.Errorf("expected RecordTrace NOT to be called for a non-trace packet, got %d calls", len(db.traceHearings)) } } func buildTracePacket(t *testing.T) *meshcore.Packet { t.Helper() - payload, err := (&meshcore.Trace{Tag: 0xdeadbeef, AuthCode: 1}).ToBytes() + payload, err := (&meshcore.Trace{Tag: 0xdeadbeef, AuthCode: 1, PathHashes: []byte{0xaa, 0xbb}}).ToBytes() if err != nil { t.Fatalf("trace to bytes: %v", err) } @@ -204,14 +314,22 @@ func buildTracePacket(t *testing.T) *meshcore.Packet { } } -func TestHandlePacket_Trace_UpsertsTraceIATA(t *testing.T) { +func TestHandlePacket_Trace_RecordsTrace(t *testing.T) { w, db := newTestWorker() db.observationInserted = true + db.packetIsNew = true envelope := packetEnvelope(t, buildTracePacket(t)) + w.handlePacket(context.Background(), "YOW", "0102", envelope) + db.packetIsNew, db.observationInserted = false, false w.handlePacket(context.Background(), "YOW", "0102", envelope) - if db.upsertTraceIATACalls != 1 { - t.Errorf("expected UpsertTraceIATA to be called once for a stored trace, got %d", db.upsertTraceIATACalls) + if len(db.traceHearings) != 2 { + t.Fatalf("expected RecordTrace for every hearing, got %d", len(db.traceHearings)) + } + for _, h := range db.traceHearings { + if h.IATA != "YOW" || h.HeardAt.IsZero() || hex.EncodeToString(h.TraceTag) != "efbeadde" { + t.Errorf("unexpected hearing %+v", h) + } } } diff --git a/internal/ingest/status.go b/internal/ingest/status.go index fb540f3c..9d3222eb 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -4,16 +4,18 @@ package ingest import ( + "bytes" "context" "encoding/hex" "encoding/json" "fmt" - "log" + "slices" "strconv" "strings" "time" "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/MeshCore-Beacon/beacon-server/internal/lora" ) // UpdateObserverStatusParams carries the fields parsed from a /status message. @@ -50,46 +52,142 @@ type statusEvent struct { LastStatusAt int64 `json:"lastStatusAt"` } +type statusStats struct { + UptimeSeconds int64 `json:"uptime_secs"` + BatteryMV int `json:"battery_mv"` + NoiseFloor float32 `json:"noise_floor"` + QueueLen int `json:"queue_len"` + DebugFlags int `json:"debug_flags"` + TxAirSecs float64 `json:"tx_air_secs"` + RxAirSecs float64 `json:"rx_air_secs"` + RecvErrors int `json:"recv_errors"` +} + +// usable reports whether the stats carry real radio readings. A running observer +// never reports zero uptime, and a live radio never reads a 0 dB noise floor, so +// that with zero airtime marks a status sent before the radio stats were filled in. +func (s statusStats) usable() bool { + if s.UptimeSeconds == 0 { + return false + } + return s.NoiseFloor != 0 || s.TxAirSecs != 0 || s.RxAirSecs != 0 +} + +// LoRa radios span 137 MHz sub-GHz to 2.5 GHz SX128x. +const ( + minRadioFreqMHz = 100 + maxRadioFreqMHz = 3000 +) + +// parseRadioFloat rejects NaN, Inf and anything outside [lo, hi]; observers send free text and +// one bad value would poison every airtime the observer is later costed with. +func parseRadioFloat(s string, lo, hi float64) (float64, error) { + f, err := strconv.ParseFloat(s, 32) + if err != nil { + return 0, err + } + if !(f >= lo && f <= hi) { + return 0, fmt.Errorf("radio value %q outside %g..%g", s, lo, hi) + } + return f, nil +} + +// stripNULs removes NUL bytes, which Postgres text columns reject, and returns the +// names of the fields that had any. +func stripNULs(fields map[string]*string) []string { + var dirty []string + for name, s := range fields { + if strings.Contains(*s, "\x00") { + *s = strings.ReplaceAll(*s, "\x00", "") + dirty = append(dirty, name) + } + } + slices.Sort(dirty) + return dirty +} + +// cleanText makes over-the-air text storable: invalid UTF-8 replaced, NULs dropped. +func cleanText(s string) string { + return strings.ReplaceAll(strings.ToValidUTF8(s, "�"), "\x00", "") +} + +// stripJSONNULs drops \u0000 from a JSON document's strings, which jsonb rejects. +// Payloads without one are returned untouched. +func stripJSONNULs(raw []byte) ([]byte, bool) { + if !bytes.Contains(raw, []byte(`\u0000`)) { + return raw, false + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return raw, false + } + out, err := json.Marshal(dropNULs(v)) + if err != nil { + return raw, false + } + return out, true +} + +func dropNULs(v any) any { + switch v := v.(type) { + case string: + return strings.ReplaceAll(v, "\x00", "") + case []any: + for i := range v { + v[i] = dropNULs(v[i]) + } + case map[string]any: + out := make(map[string]any, len(v)) + for k, e := range v { + out[strings.ReplaceAll(k, "\x00", "")] = dropNULs(e) + } + return out + } + return v +} + // handleStatus processes a /status message and fans out an observerStatus event. func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) { var envelope struct { - ObserverType string `json:"source"` - SoftwareVersion string `json:"client_version"` - HardwareModel string `json:"model"` - FirmwareVersion string `json:"firmware_version"` - DisplayName string `json:"origin"` - RadioString string `json:"radio"` - Stats struct { - UptimeSeconds int64 `json:"uptime_secs"` - BatteryMV int `json:"battery_mv"` - NoiseFloor float32 `json:"noise_floor"` - QueueLen int `json:"queue_len"` - DebugFlags int `json:"debug_flags"` - TxAirSecs float64 `json:"tx_air_secs"` - RxAirSecs float64 `json:"rx_air_secs"` - RecvErrors int `json:"recv_errors"` - } `json:"stats"` + ObserverType string `json:"source"` + SoftwareVersion string `json:"client_version"` + HardwareModel string `json:"model"` + FirmwareVersion string `json:"firmware_version"` + DisplayName string `json:"origin"` + RadioString string `json:"radio"` + Stats statusStats `json:"stats"` } if err := json.Unmarshal(raw, &envelope); err != nil { - log.Printf("ingest[%s]: malformed status envelope from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("malformed status envelope from %s", pubkeyHex), "error", err) return } pubkey, err := hex.DecodeString(pubkeyHex) if err != nil { - log.Printf("ingest[%s]: invalid pubkey hex in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("invalid pubkey hex in status from %s", pubkeyHex), "error", err) return } - id, _, err := w.db.UpsertObserver(ctx, pubkey) + dirty := stripNULs(map[string]*string{ + "source": &envelope.ObserverType, "client_version": &envelope.SoftwareVersion, + "model": &envelope.HardwareModel, "firmware_version": &envelope.FirmwareVersion, + "origin": &envelope.DisplayName, "radio": &envelope.RadioString, + }) + raw, rawDirty := stripJSONNULs(raw) + if len(dirty) > 0 || rawDirty { + w.log.Warn(fmt.Sprintf("stripped NUL bytes from status from %s", pubkeyHex), "fields", dirty, "metadata", rawDirty) + } + id, _, err := w.db.UpsertObserver(ctx, pubkey, "") if err != nil { - log.Printf("ingest[%s]: db: upsert observer failed in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert observer failed in status from %s", pubkeyHex), "error", err) return } // invalidate cache for observer details if w.onObserverUpsert != nil { w.onObserverUpsert(ctx, id) } - if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName); err != nil { - log.Printf("ingest[%s]: db: upsert observer broker failed in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, false); err != nil { + w.log.Error(fmt.Sprintf("db: upsert observer broker failed in status from %s", pubkeyHex), "error", err) } params := UpdateObserverStatusParams{ PublicKey: pubkey, @@ -122,29 +220,29 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) radio := strings.Split(strings.TrimSpace(envelope.RadioString), ",") if len(radio) != 4 { - log.Printf("ingest[%s]: missing or malformed radio params in status from %s, skipping radio fields", w.cfg.BrokerName, pubkeyHex) + w.log.Warn(fmt.Sprintf("missing or malformed radio params in status from %s, skipping radio fields", pubkeyHex)) } else { - freq, err := strconv.ParseFloat(radio[0], 32) + freq, err := parseRadioFloat(radio[0], minRadioFreqMHz, maxRadioFreqMHz) if err != nil { - log.Printf("ingest[%s]: error parsing radio freq in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error parsing radio freq in status from %s", pubkeyHex), "error", err) } else { params.RadioFreqMHz = float32(freq) } - bw, err := strconv.ParseFloat(radio[1], 32) + bw, err := parseRadioFloat(radio[1], lora.MinBWKHz, lora.MaxBWKHz) if err != nil { - log.Printf("ingest[%s]: error parsing radio bw in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error parsing radio bw in status from %s", pubkeyHex), "error", err) } else { params.RadioBWKHz = float32(bw) } sf, err := strconv.ParseInt(radio[2], 10, 16) if err != nil { - log.Printf("ingest[%s]: error parsing radio sf in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error parsing radio sf in status from %s", pubkeyHex), "error", err) } else { params.RadioSF = int16(sf) } cr, err := strconv.ParseInt(radio[3], 10, 16) if err != nil { - log.Printf("ingest[%s]: error parsing radio cr in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error parsing radio cr in status from %s", pubkeyHex), "error", err) } else { params.RadioCR = int16(cr) } @@ -152,16 +250,14 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) observerID, err := w.db.UpdateObserverStatus(ctx, params) if err != nil { - log.Printf("ingest[%s]: db: update observer status failed for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: update observer status failed for %s", pubkeyHex), "error", err) return } - // Store a telemetry snapshot at the configured resolution. - // A running observer never reports uptime_secs == 0, so its absence (or a - // missing/renamed "stats" object entirely) means this payload has no usable - // stats — skip the insert rather than writing an all-zero row that would win - // the hourly dedup and pollute the telemetry aggregates. - if envelope.Stats.UptimeSeconds == 0 { - log.Printf("ingest[%s]: status from %s has no usable stats (uptime_secs missing or zero), skipping telemetry insert", w.cfg.BrokerName, pubkeyHex) + // Store a telemetry snapshot at the configured resolution. Partial stats are + // skipped rather than written as zeros that would win the hourly dedup and + // pollute the telemetry aggregates. + if !envelope.Stats.usable() { + w.log.Debug("status has no usable stats; skipping telemetry insert") } else { resolution := w.cfg.TelemetryResolution if resolution == 0 { @@ -180,7 +276,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) envelope.Stats.NoiseFloor, envelope.Stats.UptimeSeconds, &queueLen, &debugFlags, &recvErrors, ); err != nil { - log.Printf("ingest[%s]: db: insert telemetry failed for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: insert telemetry failed for %s", pubkeyHex), "error", err) } } @@ -190,7 +286,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) } scopes, err := w.db.GetObserverScopes(ctx, observerID) if err != nil { - log.Printf("ingest[%s]: failed to get observer scopes for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("failed to get observer scopes for %s", pubkeyHex), "error", err) scopes = []string{} } var radioStr *string @@ -216,8 +312,8 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) } payload, err := json.Marshal(evt) if err != nil { - log.Printf("ingest[%s]: failed to marshal status event payload for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("failed to marshal status event payload for %s", pubkeyHex), "error", err) return } - w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus, Payload: payload, IATA: iata}) + w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus, Payload: payload, IATA: iata, ObserverID: evt.ObserverID}) } diff --git a/internal/ingest/status_test.go b/internal/ingest/status_test.go new file mode 100644 index 00000000..6d8876cc --- /dev/null +++ b/internal/ingest/status_test.go @@ -0,0 +1,74 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "encoding/json" + "testing" +) + +func TestStatusStatsUsable(t *testing.T) { + cases := []struct { + name string + raw string + want bool + }{ + {"full stats", `{"uptime_secs":1093083,"noise_floor":-96,"tx_air_secs":7935,"rx_air_secs":47484,"recv_errors":93689}`, true}, + {"missing stats", `{}`, false}, + {"uptime only", `{"uptime_secs":1096701,"noise_floor":0,"tx_air_secs":0,"rx_air_secs":0,"recv_errors":0}`, false}, + {"uptime and recv errors only", `{"uptime_secs":3834495,"recv_errors":325529}`, false}, + {"fresh boot, no airtime yet", `{"uptime_secs":60,"noise_floor":-110,"tx_air_secs":0,"rx_air_secs":0}`, true}, + {"no noise floor reported", `{"uptime_secs":5000,"tx_air_secs":3,"rx_air_secs":40}`, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var s statusStats + if err := json.Unmarshal([]byte(c.raw), &s); err != nil { + t.Fatal(err) + } + if got := s.usable(); got != c.want { + t.Errorf("usable() = %v, want %v", got, c.want) + } + }) + } +} + +func TestStripNULs(t *testing.T) { + fw := "v1.17.1-d929643\x00\x00\x00" + name := "Canadaverse\x00 RemoteTerm" + model := "Heltec Wireless Paper" + got := stripNULs(map[string]*string{"firmware_version": &fw, "origin": &name, "model": &model}) + if fw != "v1.17.1-d929643" || name != "Canadaverse RemoteTerm" || model != "Heltec Wireless Paper" { + t.Errorf("stripped to %q %q %q", fw, name, model) + } + if len(got) != 2 || got[0] != "firmware_version" || got[1] != "origin" { + t.Errorf("reported fields %v, want [firmware_version origin]", got) + } +} + +func TestStripJSONNULs(t *testing.T) { + clean := []byte(`{"origin":"a","stats":{"uptime_secs":12}}`) + if out, changed := stripJSONNULs(clean); changed || string(out) != string(clean) { + t.Errorf("clean payload rewritten: %s", out) + } + + dirty := []byte(`{"firmware_version":"v1.17\u0000\u0000","extra":{"list":["x\u0000y"],"n":1.50}}`) + out, changed := stripJSONNULs(dirty) + if !changed { + t.Fatal("expected payload to be cleaned") + } + var v struct { + FirmwareVersion string `json:"firmware_version"` + Extra struct { + List []string `json:"list"` + N json.Number `json:"n"` + } `json:"extra"` + } + if err := json.Unmarshal(out, &v); err != nil { + t.Fatal(err) + } + if v.FirmwareVersion != "v1.17" || v.Extra.List[0] != "xy" || v.Extra.N != "1.50" { + t.Errorf("cleaned to %s", out) + } +} diff --git a/internal/ingest/timestamp_test.go b/internal/ingest/timestamp_test.go new file mode 100644 index 00000000..2010f63e --- /dev/null +++ b/internal/ingest/timestamp_test.go @@ -0,0 +1,84 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" +) + +func TestPacketTimestampFormatsAndSkew(t *testing.T) { + want := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Second).Add(123456789 * time.Nanosecond) + capture := func(t *testing.T, timestamp string) time.Time { + t.Helper() + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} + w.db = db + var envelope map[string]string + if err := json.Unmarshal(packetEnvelope(t, buildGrpTxtPacket(t, 0x1a, make([]byte, 16))), &envelope); err != nil { + t.Fatal(err) + } + envelope["timestamp"] = timestamp + body, err := json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + w.handlePacket(context.Background(), "YOW", strings.Repeat("01", 32), body) + if len(db.observed) != 1 { + t.Fatal("timestamp prevented packet storage") + } + return db.observed[0].HeardAt + } + for _, zone := range []*time.Location{time.UTC, time.FixedZone("zero", 0), time.FixedZone("west", -4*60*60), time.FixedZone("east", 19800)} { + for _, precision := range []time.Duration{time.Second, time.Millisecond, time.Microsecond, time.Nanosecond} { + expected := want.Truncate(precision) + stamp := expected.In(zone).Format("2006-01-02T15:04:05.999999999-07:00") + if zone == time.UTC { + stamp = expected.Format(time.RFC3339Nano) + } + t.Run(stamp, func(t *testing.T) { + if got := capture(t, stamp); !got.Equal(expected) { + t.Fatalf("reported time changed: got %s want %s", got, expected) + } + }) + } + } + for _, precision := range []time.Duration{time.Second, time.Microsecond, time.Nanosecond} { + expected := want.Truncate(precision) + for _, layout := range []string{"2006-01-02T15:04:05.999999999", "2006-01-02 15:04:05.999999999"} { + stamp := expected.Format(layout) + t.Run("naive "+stamp, func(t *testing.T) { + if got := capture(t, stamp); !got.Equal(expected) { + t.Fatalf("timezone-less UTC changed: got %s want %s", got, expected) + } + }) + } + } + west := time.FixedZone("west", -4*60*60) + for _, layout := range []string{ + "2006-01-02 15:04:05.999999-07:00", // Python str(datetime) + "2006-01-02T15:04:05.999-0700", + "2006-01-02 15:04:05-0700", + } { + stamp := want.Truncate(time.Microsecond).In(west).Format(layout) + expected, _ := time.Parse(layout, stamp) + t.Run(stamp, func(t *testing.T) { + if got := capture(t, stamp); !got.Equal(expected) { + t.Fatalf("reported time changed: got %s want %s", got, expected) + } + }) + } + for _, stamp := range []string{"", "invalid", "2026-09-29T25:00:00+00:00", time.Now().Add(31 * time.Minute).Format(time.RFC3339), time.Now().Add(-31 * time.Minute).Format(time.RFC3339)} { + t.Run("guard "+stamp, func(t *testing.T) { + before := time.Now().UTC() + got := capture(t, stamp) + if got.Before(before) || got.After(time.Now().UTC()) { + t.Fatalf("invalid/skewed time escaped server-time guard: %s => %s", stamp, got) + } + }) + } +} diff --git a/internal/ingest/topic_test.go b/internal/ingest/topic_test.go new file mode 100644 index 00000000..452c7fe4 --- /dev/null +++ b/internal/ingest/topic_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/google/uuid" +) + +type observerCountDB struct { + *frameCaptureDB + observers int +} + +func (s *observerCountDB) UpsertObserver(context.Context, []byte, string) (uuid.UUID, string, error) { + s.observers++ + return uuid.Nil, "", nil +} + +func TestTopicPubkeyMustBe32Bytes(t *testing.T) { + var envelope map[string]string + if err := json.Unmarshal(packetEnvelope(t, buildGrpTxtPacket(t, 0x1a, make([]byte, 16))), &envelope); err != nil { + t.Fatal(err) + } + envelope["timestamp"] = time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) // takes the clamp log path + body, err := json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"ab", strings.Repeat("ab", 31), strings.Repeat("ab", 33), strings.Repeat("zz", 32)} { + for _, sub := range []string{"packets", "status", "neighbors"} { + t.Run(sub+"/"+key, func(t *testing.T) { + w, base := newTestWorker() + db := &observerCountDB{frameCaptureDB: &frameCaptureDB{stubDB: base}} + w.db = db + w.handleMessage(queuedTestMessage{malformedTopicMessage{"meshcore/YOW/" + key + "/" + sub}, body}) + if db.observers != 0 || len(db.packets) != 0 { + t.Fatalf("malformed pubkey processed: %d observers, %d packets", db.observers, len(db.packets)) + } + }) + } + } + w, base := newTestWorker() + db := &observerCountDB{frameCaptureDB: &frameCaptureDB{stubDB: base}} + w.db = db + w.handleMessage(queuedTestMessage{malformedTopicMessage{"meshcore/YOW/" + strings.Repeat("AB", 32) + "/packets"}, body}) + if len(db.packets) != 1 { + t.Fatal("valid pubkey dropped") + } +} + +func TestQueueSurvivesHandlerPanic(t *testing.T) { + done := make(chan struct{}) + q := newMessageQueue(1, 8, 4096, func(_ context.Context, m mqtt.Message) { + if string(m.Payload()) == "0" { + panic("bad message") + } + close(done) + }) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = q.close(ctx) + }) + q.enqueue(queueMessage("aa", "packets", 0)) + q.enqueue(queueMessage("aa", "packets", 1)) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("lane stopped after a panic") + } +} diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index 92119932..0485442f 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -2,14 +2,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // Package keystore provides channel key lookup for the ingest pipeline. -// Keys are loaded from config at startup and never written at runtime. -// Future: add a DB-backed fallback that checks the channel_keys table. +// Config keys are fixed at startup; MeshMapper imports are swapped in at runtime. package keystore import ( "bytes" "crypto/sha256" "encoding/hex" + "slices" + "sync/atomic" ) // Entry holds a resolved channel key along with its metadata. @@ -23,23 +24,51 @@ type Entry struct { // MapKeyStore maps channel hash hex → list of known key entries. // Multiple entries per hash are supported to handle 1-byte hash collisions. type MapKeyStore struct { - entries map[string][]Entry // channel hash hex → entries + config map[string][]Entry + imported map[string][]Entry + entries atomic.Pointer[map[string][]Entry] // config + imported, read lock-free by ingest } -// NewMapKeyStore creates a keystore from a pre-built map of entries. -// Use BuildFromConfig to construct entries from a config.ChannelKeysConfig. +// NewMapKeyStore creates a keystore from the configured entries. func NewMapKeyStore(entries map[string][]Entry) *MapKeyStore { - store := &MapKeyStore{entries: make(map[string][]Entry)} + store := &MapKeyStore{config: make(map[string][]Entry), imported: map[string][]Entry{}} for hash, list := range entries { - store.entries[hash] = append(store.entries[hash], list...) + store.config[hash] = append(store.config[hash], list...) } + store.entries.Store(&store.config) return store } // GetKey returns all known key entries for the given channel hash byte. // Returns nil if no keys are known for this hash. func (s *MapKeyStore) GetKey(channelHash []byte) []Entry { - return s.entries[hex.EncodeToString(channelHash)] + return (*s.entries.Load())[hex.EncodeToString(channelHash)] +} + +// SetImported replaces the imported keys and returns the channel hashes of keys +// that weren't known before. Config keys win; one writer at a time. +func (s *MapKeyStore) SetImported(entries []Entry) [][]byte { + merged := make(map[string][]Entry, len(s.config)) + for hash, list := range s.config { + merged[hash] = list + } + imported := map[string][]Entry{} + var added [][]byte + for _, e := range entries { + hash := sha256.Sum256(e.Key) + hashHex := hex.EncodeToString(hash[:1]) + if EntryExists(merged[hashHex], e) { + continue + } + merged[hashHex] = append(slices.Clone(merged[hashHex]), e) + imported[hashHex] = append(imported[hashHex], e) + if !EntryExists(s.imported[hashHex], e) { + added = append(added, hash[:1]) + } + } + s.imported = imported + s.entries.Store(&merged) + return added } // DeriveHashtagKey derives the channel secret and hash for a hashtag name. diff --git a/internal/keystore/keystore_test.go b/internal/keystore/keystore_test.go index bed3102a..2060c762 100644 --- a/internal/keystore/keystore_test.go +++ b/internal/keystore/keystore_test.go @@ -153,3 +153,33 @@ func TestEntryExists_Empty(t *testing.T) { t.Error("expected false for empty slice") } } + +func TestSetImported_ConfigWinsAndNewHashesReported(t *testing.T) { + secret, hash, fp := DeriveHashtagKey("meshcore") + hashHex := hex.EncodeToString([]byte{hash}) + configured := Entry{Key: secret, Fingerprint: fp, Hashtag: "meshcore", Name: "Configured"} + store := NewMapKeyStore(map[string][]Entry{hashHex: {configured}}) + + dupSecret, _, dupFP := DeriveHashtagKey("meshcore") + newSecret, newHash, newFP := DeriveHashtagKey("ottawa-mesh") + added := store.SetImported([]Entry{ + {Key: dupSecret, Fingerprint: dupFP, Hashtag: "meshcore", Name: "#meshcore"}, + {Key: newSecret, Fingerprint: newFP, Hashtag: "ottawa-mesh", Name: "#ottawa-mesh"}, + }) + if len(added) != 1 || added[0][0] != newHash { + t.Fatalf("new hashes = %x", added) + } + if got := store.GetKey([]byte{hash}); len(got) != 1 || got[0].Name != "Configured" { + t.Fatalf("config entry replaced: %+v", got) + } + if got := store.GetKey([]byte{newHash}); len(got) != 1 || got[0].Hashtag != "ottawa-mesh" { + t.Fatalf("imported entry missing: %+v", got) + } + if added = store.SetImported([]Entry{{Key: newSecret, Fingerprint: newFP, Hashtag: "ottawa-mesh"}}); len(added) != 0 { + t.Fatal("unchanged import reported new hashes", added) + } + store.SetImported(nil) + if store.GetKey([]byte{newHash}) != nil || len(store.GetKey([]byte{hash})) != 1 { + t.Fatal("clearing imports touched config keys or kept the import") + } +} diff --git a/internal/logging/logging.go b/internal/logging/logging.go new file mode 100644 index 00000000..e7e738cb --- /dev/null +++ b/internal/logging/logging.go @@ -0,0 +1,47 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package logging + +import ( + "fmt" + "io" + "log/slog" + "os" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +// New resolves environment overrides, validates options and writes to out. +// Empty settings default to info/text; callers install the logger before starting workers. +func New(out io.Writer, cfg config.LogConfig) (*slog.Logger, error) { + if value := os.Getenv("LOG_LEVEL"); value != "" { + cfg.Level = value + } + if value := os.Getenv("LOG_FORMAT"); value != "" { + cfg.Format = value + } + var level slog.Level + switch strings.ToLower(strings.TrimSpace(cfg.Level)) { + case "", "info": + level = slog.LevelInfo + case "debug": + level = slog.LevelDebug + case "warn": + level = slog.LevelWarn + case "error": + level = slog.LevelError + default: + return nil, fmt.Errorf("log level must be debug, info, warn or error") + } + opts := &slog.HandlerOptions{Level: level} + switch strings.ToLower(strings.TrimSpace(cfg.Format)) { + case "", "text": + return slog.New(slog.NewTextHandler(out, opts)), nil + case "json": + return slog.New(slog.NewJSONHandler(out, opts)), nil + default: + return nil, fmt.Errorf("log format must be text or json") + } +} diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go new file mode 100644 index 00000000..196ddaba --- /dev/null +++ b/internal/logging/logging_test.go @@ -0,0 +1,97 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package logging + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestLevelsAndFormats(t *testing.T) { + for _, format := range []string{"text", "json"} { + for i, level := range []string{"debug", "info", "warn", "error"} { + t.Run(format+"/"+level, func(t *testing.T) { + t.Setenv("LOG_LEVEL", "") + t.Setenv("LOG_FORMAT", "") + var out bytes.Buffer + logger, err := New(&out, config.LogConfig{Level: level, Format: format}) + if err != nil { + t.Fatal(err) + } + logger = logger.With("component", "ingest", "broker", "test") + logger.Debug("debug message") + logger.Info("info message") + logger.Warn("warn message") + logger.Error("error message") + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(lines) != 4-i { + t.Fatalf("got %d records, want %d: %s", len(lines), 4-i, out.String()) + } + for _, line := range lines { + if format == "json" { + var record map[string]any + if err := json.Unmarshal([]byte(line), &record); err != nil { + t.Fatal(err) + } + if record["component"] != "ingest" || record["broker"] != "test" || record["time"] == nil { + t.Fatalf("missing fields: %v", record) + } + } else if !strings.Contains(line, "component=ingest") || !strings.Contains(line, "broker=test") { + t.Fatalf("missing fields: %s", line) + } + } + if !strings.Contains(out.String(), "error message") { + t.Fatal("error suppressed") + } + }) + } + } +} + +func TestDefaultsAndEnvironment(t *testing.T) { + t.Setenv("LOG_LEVEL", "") + t.Setenv("LOG_FORMAT", "") + var out bytes.Buffer + logger, err := New(&out, config.LogConfig{}) + if err != nil { + t.Fatal(err) + } + if logger.Enabled(t.Context(), slog.LevelDebug) || !logger.Enabled(t.Context(), slog.LevelInfo) { + t.Fatal("default must be info") + } + t.Setenv("LOG_LEVEL", " WARN ") + t.Setenv("LOG_FORMAT", "JSON") + logger, err = New(&out, config.LogConfig{Level: "debug", Format: "text"}) + if err != nil { + t.Fatal(err) + } + logger.Info("hidden") + logger.Error("visible") + var record map[string]any + if err = json.Unmarshal(bytes.TrimSpace(out.Bytes()), &record); err != nil { + t.Fatal(err) + } + if record["level"] != "ERROR" || record["msg"] != "visible" { + t.Fatalf("env overrides ignored: %v", record) + } +} + +func TestInvalidOptions(t *testing.T) { + for _, cfg := range []config.LogConfig{{Level: "verbose"}, {Level: "off"}, {Level: "INFO+2"}, {Format: "xml"}} { + t.Setenv("LOG_LEVEL", "") + t.Setenv("LOG_FORMAT", "") + if _, err := New(&bytes.Buffer{}, cfg); err == nil { + t.Fatalf("accepted invalid options: %+v", cfg) + } + } + t.Setenv("LOG_LEVEL", "invalid") + if _, err := New(&bytes.Buffer{}, config.LogConfig{Level: "info"}); err == nil { + t.Fatal("invalid environment override accepted") + } +} diff --git a/internal/lora/airtime.go b/internal/lora/airtime.go new file mode 100644 index 00000000..5abd63c3 --- /dev/null +++ b/internal/lora/airtime.go @@ -0,0 +1,60 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package lora computes LoRa airtime for MeshCore frames, so an observation +// can be costed in milliseconds of channel occupancy. +package lora + +import "math" + +const ( + minSF = 7 + maxSF = 12 + minCR = 5 + maxCR = 8 + + // MinBWKHz and MaxBWKHz bracket every LoRa bandwidth (7.8 kHz sub-GHz to 1625 kHz SX128x). + MinBWKHz = 7 + MaxBWKHz = 1625 +) + +// PreambleSymbols mirrors MeshCore's preambleLengthForSF: 32 symbols at SF<=8, 16 above. +func PreambleSymbols(sf int) int { + if sf <= 8 { + return 32 + } + return 16 +} + +// FrameLength is the on-air byte count of a MeshCore frame rebuilt from its stored parts. +func FrameLength(transportCodes bool, pathLen, payloadLen int) int { + n := 1 + 1 + pathLen + payloadLen // header byte + path length byte + if transportCodes { + n += 4 + } + return n +} + +// TimeOnAirMs is the Semtech LoRa time-on-air (RadioLib getTimeOnAir: CRC on, explicit header, +// LDRO auto at t_sym >= 16 ms). ok is false when any parameter is outside the costable range. +func TimeOnAirMs(frameLen, sf int, bwKHz float64, cr int) (ms float64, ok bool) { + if frameLen <= 0 || sf < minSF || sf > maxSF || !(bwKHz >= MinBWKHz && bwKHz <= MaxBWKHz) || cr < minCR || cr > maxCR { + return 0, false + } + tSym := math.Exp2(float64(sf)) / bwKHz + de := 0 + if tSym >= 16 { + de = 1 + } + n := float64(PreambleSymbols(sf)) + 4.25 + float64(symbols(frameLen, sf, cr, de)) + return n * tSym, true +} + +// symbols is the Semtech payload symbol count with explicit header and CRC on. +func symbols(frameLen, sf, cr, de int) int { + n := math.Ceil(float64(8*frameLen-4*sf+44)/float64(4*(sf-2*de))) * float64(cr) + if n < 0 { + n = 0 + } + return 8 + int(n) +} diff --git a/internal/lora/airtime_test.go b/internal/lora/airtime_test.go new file mode 100644 index 00000000..d67d898a --- /dev/null +++ b/internal/lora/airtime_test.go @@ -0,0 +1,126 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lora + +import ( + "math" + "testing" +) + +func TestPreambleSymbols(t *testing.T) { + cases := []struct { + sf int + want int + }{ + {7, 32}, + {8, 32}, + {9, 16}, + {12, 16}, + } + for _, tc := range cases { + if got := PreambleSymbols(tc.sf); got != tc.want { + t.Errorf("PreambleSymbols(%d) = %d, want %d", tc.sf, got, tc.want) + } + } +} + +func TestFrameLength(t *testing.T) { + cases := []struct { + name string + transportCodes bool + pathLen int + payloadLen int + want int + }{ + {"no transport codes, no path", false, 0, 10, 12}, + {"transport codes with path", true, 3, 10, 19}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := FrameLength(tc.transportCodes, tc.pathLen, tc.payloadLen); got != tc.want { + t.Errorf("FrameLength(%v, %d, %d) = %d, want %d", tc.transportCodes, tc.pathLen, tc.payloadLen, got, tc.want) + } + }) + } +} + +func TestSymbols(t *testing.T) { + cases := []struct { + name string + frameLen int + sf int + cr int + de int + want int + }{ + {"sf7 no ldro", 50, 7, 5, 0, 83}, + {"sf10 ldro", 50, 10, 5, 1, 73}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := symbols(tc.frameLen, tc.sf, tc.cr, tc.de); got != tc.want { + t.Errorf("symbols(%d, %d, %d, %d) = %d, want %d", tc.frameLen, tc.sf, tc.cr, tc.de, got, tc.want) + } + }) + } +} + +func TestTimeOnAirMs(t *testing.T) { + cases := []struct { + name string + frameLen int + sf int + bwKHz float64 + cr int + want float64 + }{ + {"sf7 bw62.5", 50, 7, 62.5, 5, 244.224}, + {"sf10 bw62.5 ldro", 50, 10, 62.5, 5, 1527.808}, + {"sf7 bw250", 50, 7, 250, 5, 61.056}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := TimeOnAirMs(tc.frameLen, tc.sf, tc.bwKHz, tc.cr) + if !ok { + t.Fatalf("TimeOnAirMs(%d, %d, %v, %d) not costable, want ok", tc.frameLen, tc.sf, tc.bwKHz, tc.cr) + } + if math.Abs(got-tc.want) > 0.001 { + t.Errorf("TimeOnAirMs(%d, %d, %v, %d) = %v, want %v", tc.frameLen, tc.sf, tc.bwKHz, tc.cr, got, tc.want) + } + }) + } +} + +func TestTimeOnAirMsNotCostable(t *testing.T) { + cases := []struct { + name string + frameLen int + sf int + bwKHz float64 + cr int + }{ + {"sf zero", 50, 0, 62.5, 5}, + {"sf below range", 50, 6, 62.5, 5}, + {"sf above range", 50, 13, 62.5, 5}, + {"bw zero", 50, 7, 0, 5}, + {"bw NaN", 50, 7, math.NaN(), 5}, + {"bw infinite", 50, 7, math.Inf(1), 5}, + {"bw denormal", 50, 12, 1e-40, 5}, + {"bw above range", 50, 7, 1e9, 5}, + {"cr below range", 50, 7, 62.5, 4}, + {"cr above range", 50, 7, 62.5, 9}, + {"empty frame", 0, 7, 62.5, 5}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := TimeOnAirMs(tc.frameLen, tc.sf, tc.bwKHz, tc.cr) + if ok { + t.Errorf("TimeOnAirMs(%d, %d, %v, %d) = %v, ok; want not costable", tc.frameLen, tc.sf, tc.bwKHz, tc.cr, got) + } + if got != 0 { + t.Errorf("TimeOnAirMs(%d, %d, %v, %d) = %v, want 0 ms when not costable", tc.frameLen, tc.sf, tc.bwKHz, tc.cr, got) + } + }) + } +} diff --git a/internal/meshmapper/channels.go b/internal/meshmapper/channels.go new file mode 100644 index 00000000..33a07e30 --- /dev/null +++ b/internal/meshmapper/channels.go @@ -0,0 +1,314 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" +) + +const ( + MaxChannels = 256 // per region + MaxChannelBody = 64 << 10 + + channelRetry = config.MinChannelsRefresh // get_channels.php allows one request per region per 23.5h, failed or not +) + +type ChannelStore interface { + ListKnownIATAs(ctx context.Context) ([]string, error) + ListChannelCatalogues(ctx context.Context) ([]Catalogue, error) + // SaveChannelCatalogue replaces the IATA's members when the cache carries a new payload. + SaveChannelCatalogue(ctx context.Context, iata, url string, cache Cache, fingerprints [][]byte) error + ClearChannelMembers(ctx context.Context) error +} + +// ChannelKeys receives every imported key and reports the channel hashes it hadn't seen. +type ChannelKeys interface { + SetImported(entries []keystore.Entry) [][]byte +} + +type channelSource struct { + iata, country, url string + cache Cache + entries []keystore.Entry +} + +// Channels is owned by one background task. +type Channels struct { + store ChannelStore + keys ChannelKeys + dir *Directory + channelsURL func(site string) (string, bool) + client *http.Client + interval time.Duration + seen map[string]bool + sources []channelSource + onNewKeys func(hashes [][]byte) +} + +// NewChannels restores saved channel lists into keys without HTTP requests. +// Disabled, it clears MeshMapper's region membership so config scoping alone applies. +func NewChannels(ctx context.Context, cfg config.MeshMapperChannelsConfig, store ChannelStore, dir *Directory, keys ChannelKeys) (*Channels, error) { + c := &Channels{store: store, keys: keys, dir: dir, channelsURL: channelsEndpoint, interval: cfg.Interval(), seen: map[string]bool{}, + client: &http.Client{ + Timeout: requestTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} + if !cfg.Enabled { + if err := store.ClearChannelMembers(ctx); err != nil { + return nil, fmt.Errorf("clear MeshMapper channel members: %w", err) + } + return c, nil + } + saved, err := store.ListChannelCatalogues(ctx) + if err != nil { + return nil, fmt.Errorf("restore channel catalogues: %w", err) + } + latest := map[string]Catalogue{} + for _, cat := range saved { + if old, ok := latest[cat.IATA]; !ok || cat.AttemptedAt.After(old.AttemptedAt) { + latest[cat.IATA] = cat + } + } + iatas, err := store.ListKnownIATAs(ctx) + if err != nil { + return nil, fmt.Errorf("list IATAs for MeshMapper channels: %w", err) + } + c.track(iatas, latest) + c.publish() + return c, nil +} + +// OnNewKeys is told the channel hashes an import added, for backfill. +func (c *Channels) OnNewKeys(fn func(hashes [][]byte)) { c.onNewKeys = fn } + +func (c *Channels) track(iatas []string, saved map[string]Catalogue) { + for _, iata := range iatas { + if c.seen[iata] { + continue + } + c.seen[iata] = true + country := iatadb.CountryFor(iata) + if country == "" { + continue + } + s := channelSource{iata: iata, country: country} + if cat, ok := saved[iata]; ok { + s.url, s.cache = cat.URL, cat.Cache + if len(cat.Payload) > 0 { + var err error + if s.entries, err = decodeChannels(cat.Payload, iata); err != nil { + s.entries, s.cache.Payload, s.cache.ETag = nil, nil, "" + s.cache.LastError = "invalid saved catalogue; awaiting refresh" + } + } + c.log(s, "restored") + } + c.sources = append(c.sources, s) + } +} + +// Refresh makes at most one request. The scheduler serializes calls every 15s. +func (c *Channels) Refresh(ctx context.Context) (err error) { + parent := ctx + defer func() { + if parent.Err() != nil { + err = nil + } + }() + ctx, cancel := context.WithTimeout(ctx, refreshTimeout) + defer cancel() + now := time.Now().UTC() + iatas, err := c.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper channels: %w", err) + } + c.track(iatas, nil) + for n := range c.sources { + s := &c.sources[n] + if now.Before(s.cache.NextAttempt) { + continue + } + zones, fetched, err := c.dir.List(ctx, s.country, now) + if err != nil || fetched { + return err + } + if zones == nil { + continue + } + entry, listed := zones[s.iata] + endpoint, valid := "", false + if listed { + endpoint, valid = c.channelsURL(entry.url) + } + if !valid { + // No request to make; channels already imported stay until a list replaces them. + s.cache.NextAttempt, s.cache.LastError = now.Add(c.interval), "not listed" + if listed { + s.cache.LastError = "invalid site URL" + } + c.log(*s, "skipped") + continue + } + if endpoint != s.url { + s.url, s.cache.ETag = endpoint, "" + } + return c.refresh(ctx, s, now) + } + return nil +} + +func (c *Channels) refresh(ctx context.Context, s *channelSource, now time.Time) error { + update := Cache{AttemptedAt: now, NextAttempt: now.Add(c.interval)} + etag := "" + if len(s.cache.Payload) > 0 { + etag = s.cache.ETag + } + // Recorded first: an abandoned request may still have used the region's call. + attempt := Cache{AttemptedAt: now, NextAttempt: now.Add(min(c.interval, channelRetry)), LastError: "no response"} + s.cache.NextAttempt = attempt.NextAttempt + if err := c.store.SaveChannelCatalogue(ctx, s.iata, s.url, attempt, nil); err != nil { + return fmt.Errorf("persist channel catalogue %s: %w", s.iata, err) + } + s.cache.AttemptedAt, s.cache.LastError = now, attempt.LastError + status, body, header, err := get(ctx, c.client, "Beacon-MeshMapper-Channels/1", s.url, etag, MaxChannelBody) + if err != nil { + return err + } + var entries []keystore.Entry + var retryAt time.Time + switch status { + case http.StatusOK: + if entries, err = decodeChannels(body, s.iata); err != nil { + update.LastError = "invalid response" + } else { + update.Payload = body + } + case http.StatusNotModified: + if len(s.cache.Payload) == 0 { + update.LastError = "304 without cached catalogue" + } + default: + update.LastError = statusProblem(status, header, now, &retryAt) + } + if update.LastError == "" { + etag := header.Get("ETag") + if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") { + update.LastError, update.Payload, entries = "invalid ETag", nil, nil + } else { + if status == http.StatusNotModified && etag == "" { + etag = s.cache.ETag + } + update.ETag, update.CheckedAt = etag, now + } + } + if update.LastError != "" { + update.NextAttempt = now.Add(min(c.interval, channelRetry)) + if retryAt.After(update.NextAttempt) { + update.NextAttempt = retryAt + } + } + var fingerprints [][]byte + for _, e := range entries { + fingerprints = append(fingerprints, e.Fingerprint) + } + if update.Payload != nil && fingerprints == nil { + fingerprints = [][]byte{} + } + if err := c.store.SaveChannelCatalogue(ctx, s.iata, s.url, update, fingerprints); err != nil { + s.cache.NextAttempt = update.NextAttempt // never retry every tick during a DB outage + return fmt.Errorf("persist channel catalogue %s: %w", s.iata, err) + } + if update.Payload != nil { + s.cache.Payload, s.entries = update.Payload, entries + } + if !update.CheckedAt.IsZero() { + s.cache.CheckedAt, s.cache.ETag = update.CheckedAt, update.ETag + } + s.cache.AttemptedAt, s.cache.NextAttempt, s.cache.LastError = now, update.NextAttempt, update.LastError + if update.Payload != nil { + c.publish() + } + c.log(*s, "checked") + return nil +} + +func (c *Channels) publish() { + var all []keystore.Entry + for _, s := range c.sources { + all = append(all, s.entries...) + } + if added := c.keys.SetImported(all); len(added) > 0 && c.onNewKeys != nil { + c.onNewKeys(added) + } +} + +func (c *Channels) log(s channelSource, action string) { + level := slog.LevelInfo + if s.cache.LastError != "" && s.cache.LastError != "not listed" { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, "MeshMapper channels "+action, "component", "meshmapper.channels", "iata", s.iata, "source", s.url, + "channels", len(s.entries), "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError) +} + +func channelsEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_channels.php") } + +// decodeChannels accepts a single region's list only when every key and hash is +// exactly what Beacon derives from the name, so a bad entry can't plant a wrong key. +func decodeChannels(body []byte, iata string) ([]keystore.Entry, error) { + var document struct { + GeneratedAt time.Time `json:"generated_at"` + Region string `json:"region"` + Zones []string `json:"zones"` + Count *int `json:"count"` + Channels *[]struct { + Name string `json:"name"` + Key string `json:"key"` + Hash string `json:"hash"` + Zones []string `json:"zones"` + } `json:"channels"` + } + if len(body) > MaxChannelBody || !utf8.Valid(body) { + return nil, fmt.Errorf("invalid channel list size/encoding") + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, err + } + if document.GeneratedAt.IsZero() || document.Region != iata || len(document.Zones) != 1 || document.Zones[0] != iata || + document.Channels == nil || document.Count == nil || *document.Count != len(*document.Channels) || len(*document.Channels) > MaxChannels { + return nil, fmt.Errorf("invalid regional channel list") + } + entries := make([]keystore.Entry, 0, len(*document.Channels)) + seen := map[string]bool{} + for _, ch := range *document.Channels { + tag, ok := strings.CutPrefix(ch.Name, "#") + if !ok || tag == "" || len(ch.Name) > 64 || strings.ToLower(ch.Name) != ch.Name || seen[ch.Name] || + strings.ContainsFunc(tag, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) || r == '#' }) || + len(ch.Zones) != 1 || ch.Zones[0] != iata { + return nil, fmt.Errorf("invalid channel entry") + } + secret, hash, fingerprint := keystore.DeriveHashtagKey(tag) + key, err := hex.DecodeString(ch.Key) + if err != nil || !bytes.Equal(key, secret) || !strings.EqualFold(ch.Hash, hex.EncodeToString([]byte{hash})) { + return nil, fmt.Errorf("channel %s key does not match its name", ch.Name) + } + seen[ch.Name] = true + entries = append(entries, keystore.Entry{Key: secret, Fingerprint: fingerprint, Hashtag: tag, Name: ch.Name}) + } + return entries, nil +} diff --git a/internal/meshmapper/channels_test.go b/internal/meshmapper/channels_test.go new file mode 100644 index 00000000..29773aa7 --- /dev/null +++ b/internal/meshmapper/channels_test.go @@ -0,0 +1,256 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" +) + +func channelList(iata string, names ...string) []byte { + rows := []map[string]any{} + for _, name := range names { + secret, hash, _ := keystore.DeriveHashtagKey(strings.TrimPrefix(name, "#")) + rows = append(rows, map[string]any{"name": name, "key": hex.EncodeToString(secret), "hash": hex.EncodeToString([]byte{hash}), "zones": []string{iata}}) + } + b, _ := json.Marshal(map[string]any{"generated_at": "2026-09-30T18:00:00Z", "region": iata, "zones": []string{iata}, "count": len(rows), "channels": rows}) + return b +} + +type channelMemoryStore struct { + iatas []string + rows map[sourceKey]Cache + members map[string][][]byte + fail bool +} + +func newChannelMemoryStore(iatas ...string) *channelMemoryStore { + return &channelMemoryStore{iatas: iatas, rows: map[sourceKey]Cache{}, members: map[string][][]byte{}} +} + +func (s *channelMemoryStore) ListKnownIATAs(context.Context) ([]string, error) { + return slices.Clone(s.iatas), nil +} + +func (s *channelMemoryStore) ListChannelCatalogues(context.Context) ([]Catalogue, error) { + var out []Catalogue + for key, c := range s.rows { + out = append(out, Catalogue{IATA: key.iata, URL: key.url, Cache: c}) + } + return out, nil +} + +func (s *channelMemoryStore) SaveChannelCatalogue(_ context.Context, iata, url string, next Cache, fingerprints [][]byte) error { + if s.fail { + return context.DeadlineExceeded + } + if next.Payload != nil { + s.members[iata] = fingerprints + } + old := s.rows[sourceKey{iata, url}] + if next.Payload == nil { + next.Payload = old.Payload + } + if next.CheckedAt.IsZero() { + next.CheckedAt, next.ETag = old.CheckedAt, old.ETag + } + s.rows[sourceKey{iata, url}] = next + return nil +} + +func (s *channelMemoryStore) ClearChannelMembers(context.Context) error { + s.members = map[string][][]byte{} + return nil +} + +type channelSite struct { + *httptest.Server + list, body string + status int + retryAfter string + listCalls, channels int +} + +func newChannelSite(t *testing.T) *channelSite { + f := &channelSite{status: 200, body: string(channelList("YOW", "#ottawa-mesh"))} + f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/get_zones.php": + f.listCalls++ + _, _ = w.Write([]byte(f.list)) + case "/get_channels.php": + f.channels++ + w.Header().Set("ETag", `"c1"`) + w.Header().Set("Retry-After", f.retryAfter) + w.WriteHeader(f.status) + if f.status == 200 { + _, _ = w.Write([]byte(f.body)) + } + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + f.list = `{"country":"CA","zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}]}` + t.Cleanup(f.Close) + return f +} + +type channelHarness struct { + c *Channels + keys *keystore.MapKeyStore + added [][]byte +} + +func newChannelHarness(t *testing.T, f *channelSite, store *channelMemoryStore, lists *zoneListMemory, enabled bool) *channelHarness { + t.Helper() + dir := NewDirectory(lists) + dir.listURL = f.URL + "/get_zones.php" + if err := dir.Restore(context.Background()); err != nil { + t.Fatal(err) + } + h := &channelHarness{keys: keystore.NewMapKeyStore(nil)} + c, err := NewChannels(context.Background(), config.MeshMapperChannelsConfig{Enabled: enabled}, store, dir, h.keys) + if err != nil { + t.Fatal(err) + } + c.channelsURL = func(site string) (string, bool) { return site + "get_channels.php", strings.HasPrefix(site, f.URL) } + c.OnNewKeys(func(hashes [][]byte) { h.added = append(h.added, hashes...) }) + h.c = c + return h +} + +func (h *channelHarness) refresh(t *testing.T, n int) { + t.Helper() + for range n { + if err := h.c.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestChannelsImportKeysAndMembers(t *testing.T) { + f := newChannelSite(t) + store := newChannelMemoryStore("YOW") + h := newChannelHarness(t, f, store, newZoneListMemory(), true) + h.refresh(t, 2) + secret, hash, fingerprint := keystore.DeriveHashtagKey("ottawa-mesh") + got := h.keys.GetKey([]byte{hash}) + if len(got) != 1 || !bytes.Equal(got[0].Key, secret) || got[0].Name != "#ottawa-mesh" || got[0].Hashtag != "ottawa-mesh" { + t.Fatalf("key not imported: %+v", got) + } + if len(store.members["YOW"]) != 1 || !bytes.Equal(store.members["YOW"][0], fingerprint) { + t.Fatal("YOW membership not saved", store.members) + } + if len(h.added) != 1 || h.added[0][0] != hash { + t.Fatal("new key not reported for backfill", h.added) + } + if b := h.c.sources[0].cache; b.NextAttempt.Sub(b.AttemptedAt) != 24*time.Hour { + t.Fatal("refresh interval", b.NextAttempt.Sub(b.AttemptedAt)) + } +} + +func TestChannelsRejectMismatchKeepsLastGood(t *testing.T) { + f := newChannelSite(t) + store := newChannelMemoryStore("YOW") + h := newChannelHarness(t, f, store, newZoneListMemory(), true) + h.refresh(t, 2) + good := string(channelList("YOW", "#ottawa-mesh")) + secret, hash, _ := keystore.DeriveHashtagKey("ottawa-mesh") + for name, body := range map[string]string{ + "planted key": strings.Replace(good, hex.EncodeToString(secret), strings.Repeat("0", 32), 1), + "wrong key": strings.Replace(string(channelList("YOW", "#other")), `"#other"`, `"#planted"`, 1), + "wrong hash": strings.Replace(good, `"hash":"`+hex.EncodeToString([]byte{hash})+`"`, `"hash":"zz"`, 1), + "other region": string(channelList("YYZ", "#ottawa-mesh")), + "bad count": strings.Replace(good, `"count":1`, `"count":2`, 1), + "uppercase": string(channelList("YOW", "#Ottawa")), + "group zones": strings.Replace(good, `"zones":["YOW"]}`, `"zones":["YOW","YYZ"]}`, 1), + "missing count": strings.Replace(good, `"count":1,`, ``, 1), + } { + f.body = body + h.c.sources[0].cache.NextAttempt = time.Time{} + h.refresh(t, 1) + s := h.c.sources[0] + if s.cache.LastError != "invalid response" || len(h.keys.GetKey([]byte{hash})) != 1 || len(store.members["YOW"]) != 1 { + t.Fatalf("%s: accepted or dropped last good list: %q", name, s.cache.LastError) + } + if s.cache.NextAttempt.Sub(s.cache.AttemptedAt) != 24*time.Hour { + t.Fatalf("%s: retried before the rate limit", name) + } + } +} + +func TestChannelsRateLimitAndFailuresWait(t *testing.T) { + for _, tc := range []struct { + status int + retryAfter string + want time.Duration + }{{503, "", 24 * time.Hour}, {429, "60", 24 * time.Hour}, {429, "172800", 48 * time.Hour}} { + f := newChannelSite(t) + f.status, f.retryAfter = tc.status, tc.retryAfter + h := newChannelHarness(t, f, newChannelMemoryStore("YOW"), newZoneListMemory(), true) + h.refresh(t, 3) + s := h.c.sources[0] + if f.channels != 1 || s.cache.NextAttempt.Sub(s.cache.AttemptedAt) != tc.want { + t.Fatalf("HTTP %d Retry-After %q: calls=%d wait=%v", tc.status, tc.retryAfter, f.channels, s.cache.NextAttempt.Sub(s.cache.AttemptedAt)) + } + } +} + +func TestChannelsRestoreWithoutRequests(t *testing.T) { + f := newChannelSite(t) + store, lists := newChannelMemoryStore("YOW"), newZoneListMemory() + newChannelHarness(t, f, store, lists, true).refresh(t, 2) + restarted := newChannelHarness(t, f, store, lists, true) + _, hash, _ := keystore.DeriveHashtagKey("ottawa-mesh") + if len(restarted.keys.GetKey([]byte{hash})) != 1 { + t.Fatal("saved channels not restored") + } + restarted.refresh(t, 2) + if f.listCalls != 1 || f.channels != 1 { + t.Fatal("restart made requests", f.listCalls, f.channels) + } +} + +func TestChannelsDisabledClearsMembers(t *testing.T) { + f := newChannelSite(t) + store := newChannelMemoryStore("YOW") + store.members["YOW"] = [][]byte{{1}} + h := newChannelHarness(t, f, store, newZoneListMemory(), false) + if len(store.members) != 0 || len(h.c.sources) != 0 { + t.Fatal("disabled import kept membership or sources") + } +} + +func TestChannelsRequestCutShortStillCounts(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() })) + defer server.Close() + store := newChannelMemoryStore("YOW") + c, err := NewChannels(context.Background(), config.MeshMapperChannelsConfig{Enabled: true}, store, NewDirectory(newZoneListMemory()), keystore.NewMapKeyStore(nil)) + if err != nil { + t.Fatal(err) + } + s := &c.sources[0] + s.url = server.URL + now := time.Now().UTC() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _ = c.refresh(ctx, s, now) + if got := s.cache.NextAttempt.Sub(now); got != channelRetry { + t.Fatal("abandoned request retried inside the rate limit", got) + } + if row := store.rows[sourceKey{"YOW", server.URL}]; row.NextAttempt.Sub(now) != channelRetry { + t.Fatal("abandoned request not persisted", row) + } +} diff --git a/internal/meshmapper/directory.go b/internal/meshmapper/directory.go new file mode 100644 index 00000000..5113f02b --- /dev/null +++ b/internal/meshmapper/directory.go @@ -0,0 +1,269 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/url" + "regexp" + "slices" + "strings" + "sync" + "time" + "unicode" +) + +type zoneEntry struct { + url string + hasBoundary bool + name string // short display name, e.g. "Nanaimo" + lat, lon *float64 // nil when missing or out of range +} + +type zoneGroup struct { + name string + members []string +} + +type zoneList struct { + zones map[string]zoneEntry + groups map[string]zoneGroup + etag string + fetchedAt, nextAttempt time.Time + fetching bool +} + +// ZoneList is one country's saved get_zones.php response and fetch state. +type ZoneList struct { + Country string + Payload json.RawMessage + ETag string + FetchedAt, AttemptedAt, NextAttempt time.Time + LastError string +} + +type DirectoryStore interface { + ListZoneLists(ctx context.Context) ([]ZoneList, error) + SaveZoneList(ctx context.Context, l ZoneList) error +} + +// Directory caches MeshMapper's per-country zone lists, shared by the zones and scopes imports. +type Directory struct { + mu sync.Mutex + store DirectoryStore + client *http.Client + listURL string + lists map[string]*zoneList + version int // bumped whenever a list is replaced +} + +func NewDirectory(store DirectoryStore) *Directory { + return &Directory{store: store, listURL: ZonesURL, lists: map[string]*zoneList{}, client: &http.Client{ + Timeout: requestTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} +} + +// Restore loads saved lists so a restart doesn't spend each country's daily request. +func (d *Directory) Restore(ctx context.Context) error { + saved, err := d.store.ListZoneLists(ctx) + if err != nil { + return fmt.Errorf("restore MeshMapper zone lists: %w", err) + } + d.mu.Lock() + defer d.mu.Unlock() + for _, l := range saved { + list := &zoneList{nextAttempt: l.NextAttempt} + if zones, groups, err := decodeZones(l.Payload, l.Country); err == nil { + list.zones, list.groups, list.etag, list.fetchedAt = zones, groups, l.ETag, l.FetchedAt + d.version++ + } + d.lists[l.Country] = list + } + return nil +} + +// List returns the country's fresh zone list. fetched reports that this call +// spent the caller's one request; nil zones without fetched means back off, +// including while another task's request for the country is in flight. +func (d *Directory) List(ctx context.Context, country string, now time.Time) (zones map[string]zoneEntry, fetched bool, err error) { + d.mu.Lock() + list := d.lists[country] + if list != nil && list.zones != nil && now.Sub(list.fetchedAt) < zoneListFresh { + d.mu.Unlock() + return list.zones, false, nil + } + if list != nil && (list.fetching || now.Before(list.nextAttempt)) { + d.mu.Unlock() + return nil, false, nil + } + if list == nil { + list = &zoneList{} + d.lists[country] = list + } + // Every request counts against the country's 23.5h allowance, failed, abandoned or not. + list.fetching, list.nextAttempt = true, now.Add(zoneListFresh) + cached, etag := list.zones != nil, "" + if cached { + etag = list.etag + } + d.mu.Unlock() + defer func() { + d.mu.Lock() + list.fetching = false + d.mu.Unlock() + }() + return nil, true, d.fetch(ctx, list, country, cached, etag, now) +} + +// fetch runs without d.mu so other tasks never wait out this request. +func (d *Directory) fetch(ctx context.Context, list *zoneList, country string, cached bool, etag string, now time.Time) error { + attempt := ZoneList{Country: country, AttemptedAt: now, NextAttempt: now.Add(zoneListFresh), LastError: "no response"} + if err := d.store.SaveZoneList(ctx, attempt); err != nil { + return fmt.Errorf("persist MeshMapper zone list %s: %w", country, err) + } + endpoint := d.listURL + "?country=" + url.QueryEscape(country) + status, body, header, err := get(ctx, d.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxZoneList) + if err != nil { + return err + } + problem := "" + var payload json.RawMessage + var retryAt time.Time + var zones map[string]zoneEntry + var groups map[string]zoneGroup + switch status { + case http.StatusOK: + var decodeErr error + if zones, groups, decodeErr = decodeZones(body, country); decodeErr != nil { + problem = "invalid response" + } else { + payload = body + } + case http.StatusNotModified: + if !cached { + problem = "304 without cached list" + } + default: + problem = statusProblem(status, header, now, &retryAt) + } + d.mu.Lock() + if payload != nil { + list.zones, list.groups, list.etag = zones, groups, header.Get("ETag") + d.version++ + } + if retryAt.After(list.nextAttempt) { + list.nextAttempt = retryAt + } + saved := ZoneList{Country: country, Payload: payload, AttemptedAt: now, NextAttempt: list.nextAttempt, LastError: problem} + if problem == "" { + list.fetchedAt = now + saved.FetchedAt, saved.ETag = now, list.etag + } + nZones, nGroups := len(list.zones), len(list.groups) + d.mu.Unlock() + level := slog.LevelInfo + if problem != "" { + level = slog.LevelWarn + } + slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country, + "zones", nZones, "groups", nGroups, "next_attempt", saved.NextAttempt, "last_error", problem) + if err := d.store.SaveZoneList(ctx, saved); err != nil { + return fmt.Errorf("persist MeshMapper zone list %s: %w", country, err) + } + return nil +} + +// snapshot merges every loaded list's zones and groups by code. complete is false while +// a country has no list, so a cold start or failed list can't look like groups disappearing. +func (d *Directory) snapshot(countries []string) (merged map[string]zoneGroup, zones map[string]zoneEntry, version int, complete bool) { + d.mu.Lock() + defer d.mu.Unlock() + merged, zones, complete = map[string]zoneGroup{}, map[string]zoneEntry{}, true + for _, country := range countries { + list := d.lists[country] + if list == nil || list.zones == nil { + complete = false + continue + } + for code, z := range list.zones { + zones[code] = z + } + for code, g := range list.groups { + if have, ok := merged[code]; ok { + g.members = append(slices.Clone(have.members), g.members...) + slices.Sort(g.members) + g.members = slices.Compact(g.members) + g.name = have.name + } + merged[code] = g + } + } + return merged, zones, d.version, complete +} + +var ( + groupCode = regexp.MustCompile(`^[A-Z0-9]{2,16}$`) + memberIATA = regexp.MustCompile(`^[A-Z]{3}$`) +) + +func decodeZones(body []byte, country string) (map[string]zoneEntry, map[string]zoneGroup, error) { + var document struct { + Country string `json:"country"` + Zones *[]struct { + Code string `json:"code"` + Name string `json:"name"` + ShortName string `json:"short_name"` + Lat *float64 `json:"lat"` + Lon *float64 `json:"lon"` + URL string `json:"url"` + HasBoundary bool `json:"has_boundary"` + } `json:"zones"` + Groups []struct { + Code string `json:"code"` + Name string `json:"name"` + Members []string `json:"members"` + } `json:"groups"` + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, nil, err + } + if !strings.EqualFold(document.Country, country) || document.Zones == nil { + return nil, nil, fmt.Errorf("invalid zone list") + } + zones := make(map[string]zoneEntry, len(*document.Zones)) + for _, zone := range *document.Zones { + entry := zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary, name: cmp.Or(zone.ShortName, zone.Name)} + if entry.name = strings.TrimSpace(entry.name); len(entry.name) > 128 || strings.ContainsFunc(entry.name, unicode.IsControl) { + entry.name = "" + } + if zone.Lat != nil && zone.Lon != nil && *zone.Lat >= -90 && *zone.Lat <= 90 && *zone.Lon >= -180 && *zone.Lon <= 180 { + entry.lat, entry.lon = zone.Lat, zone.Lon + } + zones[strings.ToUpper(zone.Code)] = entry + } + groups := map[string]zoneGroup{} + for _, g := range document.Groups { + code, name := strings.ToUpper(g.Code), strings.TrimSpace(g.Name) + if !groupCode.MatchString(code) || name == "" || len(name) > 128 || strings.ContainsFunc(name, unicode.IsControl) { + continue // a bad group never blocks the zone list + } + var members []string + for _, m := range g.Members { + if m = strings.ToUpper(m); memberIATA.MatchString(m) { + members = append(members, m) + } + } + slices.Sort(members) + if members = slices.Compact(members); len(members) > 0 { + groups[code] = zoneGroup{name: name, members: members} + } + } + return zones, groups, nil +} diff --git a/internal/meshmapper/groups.go b/internal/meshmapper/groups.go new file mode 100644 index 00000000..9a18154a --- /dev/null +++ b/internal/meshmapper/groups.go @@ -0,0 +1,227 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "cmp" + "context" + "fmt" + "log/slog" + "maps" + "slices" + "strings" +) + +// RegionState is a stored region; imported ones are also what SaveImportedRegion writes. +type RegionState struct { + Slug, Name string + DisplayOrder int + Imported bool + IATAs []string + CenterLat, CenterLng *float64 +} + +// IATADetails is an IATA's display name and approximate location. +type IATADetails struct { + IATA string + Name string + Lat, Lng *float64 +} + +// SetConfiguredIATAs names the IATAs whose config entry sets a name or location; those win. +func (z *Zones) SetConfiguredIATAs(iatas []string) { + z.configured = map[string]bool{} + for _, iata := range iatas { + z.configured[iata] = true + } +} + +// OnIATAsChange is told whenever imported IATA names or locations change. +func (z *Zones) OnIATAsChange(fn func(ctx context.Context)) { z.onIATAs = fn } + +// OnRegionsChange is told whenever the imported regions change, for cache invalidation. +func (z *Zones) OnRegionsChange(fn func(ctx context.Context)) { z.onRegions = fn } + +// pruneGroups drops every imported region when group import is off. +func (z *Zones) pruneGroups(ctx context.Context) error { + pruned, err := z.store.PruneImportedRegions(ctx, nil) + if err != nil { + return fmt.Errorf("prune MeshMapper regions: %w", err) + } + z.regionsRemoved(pruned) + if len(pruned) > 0 { + z.regionsChanged(ctx) + } + return nil +} + +// syncInputs is what syncDirectory last applied; it reruns when any of them changes. +type syncInputs struct { + version, known int + complete bool + heard string +} + +// syncDirectory applies the loaded zone lists to IATA details and, when enabled, +// imported regions. Groups are only pruned once every tracked country has a list. +func (z *Zones) syncDirectory(ctx context.Context, iatas []string) error { + var countries []string + for _, r := range z.regions { + countries = append(countries, r.country) + } + slices.Sort(countries) + groups, zones, version, complete := z.dir.snapshot(slices.Compact(countries)) + var heard []string + if z.importGroups { + var err error + if heard, err = z.store.ListHeardIATAs(ctx); err != nil { + return fmt.Errorf("list heard IATAs for MeshMapper groups: %w", err) + } + slices.Sort(heard) + } + inputs := syncInputs{version: version, known: len(iatas), complete: complete, heard: strings.Join(heard, ",")} + if z.groupsSynced && inputs == z.synced { + return nil + } + if err := z.syncIATADetails(ctx, zones); err != nil { + return err + } + if z.importGroups { + if err := z.syncGroups(ctx, heard, groups, zones, complete); err != nil { + return err + } + } + z.groupsSynced, z.synced = true, inputs + return nil +} + +// syncIATADetails names and locates each listed IATA unless its config entry does. +func (z *Zones) syncIATADetails(ctx context.Context, zones map[string]zoneEntry) error { + stored, err := z.store.ListIATADetails(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper details: %w", err) + } + changed := false + for _, cur := range stored { + zone, ok := zones[cur.IATA] + if !ok || z.configured[cur.IATA] || zone.name == "" && zone.lat == nil { + continue + } + name := cmp.Or(zone.name, cur.Name) + lat, lng := cur.Lat, cur.Lng + if zone.lat != nil { + lat, lng = zone.lat, zone.lon + } + if name == cur.Name && sameFloat(lat, cur.Lat) && sameFloat(lng, cur.Lng) { + continue + } + if err := z.store.UpsertIATADetails(ctx, cur.IATA, name, lat, lng); err != nil { + return fmt.Errorf("save MeshMapper details for %s: %w", cur.IATA, err) + } + changed = true + } + if changed && z.onIATAs != nil { + z.onIATAs(ctx) + } + return nil +} + +// syncGroups imports groups with a member that is heard or configured, never one only +// its own import created. Without every country's list it adds and updates but never prunes. +func (z *Zones) syncGroups(ctx context.Context, heard []string, groups map[string]zoneGroup, zones map[string]zoneEntry, complete bool) error { + inUse := maps.Clone(z.configured) + if inUse == nil { + inUse = map[string]bool{} + } + for _, iata := range heard { + inUse[iata] = true + } + state, err := z.store.ListRegionState(ctx) + if err != nil { + return fmt.Errorf("list regions for MeshMapper groups: %w", err) + } + stored, handOrder := map[string]RegionState{}, 0 + for _, r := range state { + stored[r.Slug] = r + if !r.Imported { + handOrder = max(handOrder, r.DisplayOrder) + for _, iata := range r.IATAs { + inUse[iata] = true + } + } + } + var want []RegionState + for code, g := range groups { + if slices.ContainsFunc(g.members, func(m string) bool { return inUse[m] }) { + lat, lng := center(g.members, zones) + want = append(want, RegionState{Slug: strings.ToLower(code), Name: g.name, Imported: true, IATAs: g.members, CenterLat: lat, CenterLng: lng}) + } + } + slices.SortFunc(want, func(a, b RegionState) int { return cmp.Or(cmp.Compare(a.Name, b.Name), cmp.Compare(a.Slug, b.Slug)) }) + keep, changed := []string{}, false + for n, r := range want { + r.DisplayOrder = handOrder + 1 + n + if cur, ok := stored[r.Slug]; ok && !cur.Imported { + slog.Info("MeshMapper group skipped: a configured region uses its slug", "component", "meshmapper.zones", "slug", r.Slug) + continue + } else if ok && cur.Name == r.Name && cur.DisplayOrder == r.DisplayOrder && slices.Equal(cur.IATAs, r.IATAs) && + sameFloat(cur.CenterLat, r.CenterLat) && sameFloat(cur.CenterLng, r.CenterLng) { + keep = append(keep, r.Slug) + continue + } + saved, err := z.store.SaveImportedRegion(ctx, r) + if err != nil { + return fmt.Errorf("save MeshMapper region %s: %w", r.Slug, err) + } + if !saved { + slog.Info("MeshMapper group skipped: a configured region uses its slug", "component", "meshmapper.zones", "slug", r.Slug) + continue + } + keep, changed = append(keep, r.Slug), true + slog.Info("MeshMapper region imported", "component", "meshmapper.zones", "slug", r.Slug, "name", r.Name, "iatas", r.IATAs) + } + var pruned []string + if complete { + if pruned, err = z.store.PruneImportedRegions(ctx, keep); err != nil { + return fmt.Errorf("prune MeshMapper regions: %w", err) + } + z.regionsRemoved(pruned) + } + if changed || len(pruned) > 0 { + z.regionsChanged(ctx) + } + return nil +} + +// center averages the members' zone locations; nil when none has one. +func center(members []string, zones map[string]zoneEntry) (*float64, *float64) { + var lat, lng float64 + n := 0 + for _, m := range members { + if z := zones[m]; z.lat != nil { + lat, lng, n = lat+*z.lat, lng+*z.lon, n+1 + } + } + if n == 0 { + return nil, nil + } + lat, lng = lat/float64(n), lng/float64(n) + return &lat, &lng +} + +func sameFloat(a, b *float64) bool { + return a == nil && b == nil || a != nil && b != nil && *a == *b +} + +func (z *Zones) regionsRemoved(slugs []string) { + for _, slug := range slugs { + slog.Info("MeshMapper region removed", "component", "meshmapper.zones", "slug", slug) + } +} + +func (z *Zones) regionsChanged(ctx context.Context) { + if z.onRegions != nil { + z.onRegions(ctx) + } +} diff --git a/internal/meshmapper/groups_test.go b/internal/meshmapper/groups_test.go new file mode 100644 index 00000000..fa1d853c --- /dev/null +++ b/internal/meshmapper/groups_test.go @@ -0,0 +1,236 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +const groupedList = `{"country":"CA","zones":[` + + `{"code":"YOW","name":"Ottawa, CA","short_name":"Ottawa","lat":45,"lon":-76,"url":"URL/","has_boundary":true,"group":"ONQC"},` + + `{"code":"YUL","name":"Montreal, CA","lat":46,"lon":-74,"url":"URL/","has_boundary":true,"group":"ONQC"},` + + `{"code":"YQB","name":"Quebec City, CA","url":"URL/","has_boundary":true,"group":"ONQC"}],"groups":[` + + `{"code":"ONQC","name":"Ottawa-Quebec Corridor","members":["YUL","YOW","YQB"],"url":"https://onqc.meshmapper.net/"},` + + `{"code":"GOLM","name":"Great Ontario Lake Mesh","members":["YYZ","YKF"],"url":"https://golm.meshmapper.net/"}]}` + +type groupHarness struct { + z *Zones + f *fakeMeshMapper + store *zoneMemoryStore + invalidated int +} + +func newGroupHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, importGroups bool) *groupHarness { + t.Helper() + if store.iatas == nil { + store.iatas = []string{"YOW"} + } + if store.heard == nil { + store.heard = slices.Clone(store.iatas) + } + if store.rows == nil { + store.rows = map[string]Boundary{} + } + h := &groupHarness{f: f, store: store} + dir := NewDirectory(newZoneListMemory()) + dir.listURL = f.URL + "/get_zones.php" + h.z = NewZones(config.MeshMapperZonesConfig{Enabled: true, ImportGroups: importGroups}, store, dir) + h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", true } + h.z.OnRegionsChange(func(context.Context) { h.invalidated++ }) + if err := h.z.Restore(context.Background()); err != nil { + t.Fatal(err) + } + return h +} + +func groupedFake(t *testing.T) *fakeMeshMapper { + f := newFakeMeshMapper(t) + f.list = strings.ReplaceAll(groupedList, "URL", f.URL) + return f +} + +// refetch makes the country list and YOW due, then runs the fetch and the sync tick. +func (h *groupHarness) refetch(t *testing.T) { + t.Helper() + if list := h.z.dir.lists["CA"]; list != nil { + list.fetchedAt, list.nextAttempt = time.Time{}, time.Time{} + } + for _, r := range h.z.regions { + r.b.NextAttempt = time.Time{} + } + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestGroupImportedWhenAMemberIsKnown(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + got, ok := h.store.regions["onqc"] + if !ok || got.Name != "Ottawa-Quebec Corridor" || !slices.Equal(got.IATAs, []string{"YOW", "YQB", "YUL"}) || got.DisplayOrder != 1 { + t.Fatalf("group not imported with full members: %+v", h.store.regions) + } + if _, ok := h.store.regions["golm"]; ok { + t.Fatal("group with no known member imported") + } + if got.CenterLat == nil || *got.CenterLat != 45.5 || *got.CenterLng != -75 { + t.Fatalf("center should average the located members: %v, %v", got.CenterLat, got.CenterLng) + } + if !slices.Contains(h.store.iatas, "YQB") || !slices.Contains(h.store.iatas, "YUL") { + t.Fatal("members not created as IATAs", h.store.iatas) + } + if h.invalidated != 1 { + t.Fatal("region caches not invalidated", h.invalidated) + } + h.refetch(t) + if h.invalidated != 1 { + t.Fatal("unchanged groups invalidated again", h.invalidated) + } +} + +func TestGroupSlugClashKeepsConfiguredRegion(t *testing.T) { + hand := RegionState{Slug: "onqc", Name: "Mine", DisplayOrder: 4, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{regions: map[string]RegionState{"onqc": hand}} + h := newGroupHarness(t, groupedFake(t), store, true) + h.refetch(t) + if got := store.regions["onqc"]; got.Imported || got.Name != "Mine" || !slices.Equal(got.IATAs, []string{"YOW"}) || h.invalidated != 0 { + t.Fatalf("configured region replaced: %+v", got) + } +} + +func TestGroupDroppedRemovesRegion(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + h.f.list = strings.ReplaceAll(`{"country":"CA","zones":[{"code":"YOW","url":"URL/","has_boundary":true,"group":null}],"groups":[]}`, "URL", h.f.URL) + h.refetch(t) + if len(h.store.regions) != 0 || h.invalidated != 2 { + t.Fatalf("dropped group kept: %+v invalidated=%d", h.store.regions, h.invalidated) + } +} + +func TestGroupImportOffRemovesImportedRegions(t *testing.T) { + hand := RegionState{Slug: "east", Name: "East", IATAs: []string{"YOW"}} + imported := RegionState{Slug: "onqc", Name: "Ottawa-Quebec Corridor", Imported: true, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{regions: map[string]RegionState{"east": hand, "onqc": imported}} + h := newGroupHarness(t, groupedFake(t), store, false) + if _, ok := store.regions["onqc"]; ok || h.invalidated != 1 { + t.Fatal("imported region kept with import_groups off") + } + if _, ok := store.regions["east"]; !ok { + t.Fatal("configured region removed") + } + h.refetch(t) + if len(store.regions) != 1 { + t.Fatal("groups imported with import_groups off", store.regions) + } +} + +func TestGroupFailedFetchKeepsGroups(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + h.f.listStatus = 500 + h.z.groupsSynced = false // force a reconcile against the retained list + h.refetch(t) + if _, ok := h.store.regions["onqc"]; !ok { + t.Fatal("failed fetch dropped imported region") + } +} + +func TestGroupColdStartWaitsForEveryList(t *testing.T) { + old := RegionState{Slug: "old", Name: "Old", Imported: true, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{regions: map[string]RegionState{"old": old}} + f := groupedFake(t) + f.listStatus = 500 + h := newGroupHarness(t, f, store, true) + h.refetch(t) + if _, ok := store.regions["old"]; !ok { + t.Fatal("imported regions pruned before any list loaded") + } +} + +func TestZoneListNamesAndLocatesIATAs(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{iatas: []string{"YOW", "YUL", "YQB", "YYZ"}}, false) + h.z.SetConfiguredIATAs([]string{"YUL"}) + changed := 0 + h.z.OnIATAsChange(func(context.Context) { changed++ }) + h.refetch(t) + yow := h.store.details["YOW"] + if yow.Name != "Ottawa" || yow.Lat == nil || *yow.Lat != 45 || *yow.Lng != -76 { + t.Fatalf("YOW not named and located from short_name: %+v", yow) + } + if yqb := h.store.details["YQB"]; yqb.Name != "Quebec City, CA" || yqb.Lat != nil { + t.Fatalf("YQB should fall back to name and stay unlocated: %+v", yqb) + } + if _, ok := h.store.details["YUL"]; ok { + t.Fatal("configured IATA overwritten") + } + if _, ok := h.store.details["YYZ"]; ok { + t.Fatal("unlisted IATA written") + } + if changed != 1 || h.store.writes != 2 { + t.Fatal("writes/invalidations", h.store.writes, changed) + } + h.z.groupsSynced = false + h.refetch(t) + if h.store.writes != 2 || changed != 1 { + t.Fatal("unchanged details rewritten", h.store.writes, changed) + } +} + +func TestGroupStopsQualifyingOnImportedMembers(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + if _, ok := h.store.regions["onqc"]; !ok { + t.Fatal("group not imported") + } + without := strings.Replace(groupedList, `"members":["YUL","YOW","YQB"]`, `"members":["YUL","YQB"]`, 1) + h.f.list = strings.ReplaceAll(without, "URL", h.f.URL) + h.refetch(t) + if _, ok := h.store.regions["onqc"]; ok { + t.Fatal("group kept only by members its own import created") + } +} + +func TestGroupQualifiesThroughConfiguredRegion(t *testing.T) { + hand := RegionState{Slug: "east", Name: "East", IATAs: []string{"YYZ"}} + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{regions: map[string]RegionState{"east": hand}}, true) + h.refetch(t) + if _, ok := h.store.regions["golm"]; !ok { + t.Fatal("group with a configured member not imported") + } +} + +func TestGroupImportsFromListsItHas(t *testing.T) { + old := RegionState{Slug: "old", Name: "Old", Imported: true, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{iatas: []string{"YOW", "SEA"}, regions: map[string]RegionState{"old": old}} + f := groupedFake(t) + f.otherStatus = 500 + h := newGroupHarness(t, f, store, true) + h.refetch(t) + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if f.otherStatus != 500 || h.z.dir.lists["US"] == nil || h.z.dir.lists["US"].zones != nil { + t.Fatal("US list should have failed") + } + if _, ok := store.regions["onqc"]; !ok { + t.Fatal("one failed country blocked group import") + } + if store.details["YOW"].Name != "Ottawa" { + t.Fatal("one failed country blocked IATA details", store.details) + } + if _, ok := store.regions["old"]; !ok { + t.Fatal("pruned with a country list missing") + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go new file mode 100644 index 00000000..4099aae1 --- /dev/null +++ b/internal/meshmapper/scopes.go @@ -0,0 +1,372 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package meshmapper imports published regional scope catalogues and boundaries, off the ingest path. +package meshmapper + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "slices" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +const ( + MaxScopes = 64 // per source + MaxBody = 64 << 10 + PollInterval = 15 * time.Second // one source per tick, at most four requests/minute + + failureRetry = config.MinScopesRefresh // get_scopes.php allows one request per region per 55m +) + +// Cache stores source provenance and freshness separately from packet evidence. +type Cache struct { + Payload json.RawMessage + ETag string + CheckedAt, AttemptedAt, NextAttempt time.Time + LastError string +} + +// Catalogue is one saved source snapshot. +type Catalogue struct { + IATA, URL string + Cache +} + +type Store interface { + ListKnownIATAs(context.Context) ([]string, error) + ListScopeCatalogues(context.Context) ([]Catalogue, error) + SaveScopeCatalogue(context.Context, string, string, Cache, []scopestore.Entry) error +} + +type source struct { + iata, country, url string + cache Cache + entries []scopestore.Entry + generated time.Time +} + +// Importer is owned by one background task; ScopeStore synchronizes its consumers. +type Importer struct { + store Store + scopes *scopestore.ScopeStore + manual []scopestore.Entry + dir *Directory + scopesURL func(site string) (string, bool) + seen map[string]bool + sources []source + interval time.Duration + client *http.Client + onChange func(context.Context) +} + +// SetCacheInvalidator is wired once at startup, before the background task starts. +func (i *Importer) SetCacheInvalidator(fn func(context.Context)) { i.onChange = fn } + +// New restores validated snapshots before ingestion, without making HTTP requests. +func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, dir *Directory, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) { + i := &Importer{store: store, scopes: scopes, manual: manual, dir: dir, scopesURL: scopesEndpoint, seen: map[string]bool{}, + interval: cfg.Interval(), client: &http.Client{ + Timeout: requestTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} + if !cfg.Enabled { + return i, nil + } + saved, err := store.ListScopeCatalogues(ctx) + if err != nil { + return nil, fmt.Errorf("restore scope catalogues: %w", err) + } + // A site that moved leaves its old row behind; the latest attempt wins. + latest := map[string]Catalogue{} + for _, c := range saved { + if old, ok := latest[c.IATA]; !ok || c.AttemptedAt.After(old.AttemptedAt) { + latest[c.IATA] = c + } + } + iatas, err := store.ListKnownIATAs(ctx) + if err != nil { + return nil, fmt.Errorf("list IATAs for MeshMapper scopes: %w", err) + } + i.track(iatas, latest) + i.publish() + return i, nil +} + +// track adds a source for each IATA not seen before. +func (i *Importer) track(iatas []string, saved map[string]Catalogue) { + for _, iata := range iatas { + if i.seen[iata] { + continue + } + i.seen[iata] = true + country := iatadb.CountryFor(iata) + if country == "" { + continue + } + s := source{iata: iata, country: country} + if c, ok := saved[iata]; ok { + s.url, s.cache = c.URL, c.Cache + if len(c.Payload) > 0 { + var err error + s.entries, s.generated, err = decode(c.Payload, iata) + if err != nil { + s.entries = nil + s.cache.Payload, s.cache.ETag = nil, "" + s.cache.LastError = "invalid saved catalogue; awaiting refresh" + } + } + i.log(s, "restored") + } + i.sources = append(i.sources, s) + } +} + +// Refresh checks only one due source. The scheduler serializes calls every 15s. +func (i *Importer) Refresh(ctx context.Context) (err error) { + parent := ctx + defer func() { + if parent.Err() != nil { + err = nil + } + }() + ctx, cancel := context.WithTimeout(ctx, refreshTimeout) + defer cancel() + now := time.Now().UTC() + iatas, err := i.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper scopes: %w", err) + } + i.track(iatas, nil) + for n := range i.sources { + s := &i.sources[n] + if now.Before(s.cache.NextAttempt) { + continue + } + zones, fetched, err := i.dir.List(ctx, s.country, now) + if err != nil || fetched { + return err + } + if zones == nil { + continue + } + entry, listed := zones[s.iata] + endpoint, valid := "", false + if listed { + endpoint, valid = i.scopesURL(entry.url) + } + if !valid { + // No request to make; names already imported stay until a catalogue replaces them. + s.cache.NextAttempt, s.cache.LastError = now.Add(i.interval), "not listed" + if listed { + s.cache.LastError = "invalid site URL" + } + i.log(*s, "skipped") + continue + } + if endpoint != s.url { + s.url, s.cache.ETag = endpoint, "" + } + return i.refresh(ctx, s, now) + } + return nil +} + +func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error { + update := Cache{AttemptedAt: now, NextAttempt: now.Add(i.interval)} + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil) + if err != nil { + return err + } + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", "Beacon-MeshMapper-Scopes/1") + if len(s.cache.Payload) > 0 && s.cache.ETag != "" { + request.Header.Set("If-None-Match", s.cache.ETag) + } + // Recorded first: an abandoned request may still have used the region's call. + attempt := Cache{AttemptedAt: now, NextAttempt: now.Add(min(i.interval, failureRetry)), LastError: "no response"} + s.cache.NextAttempt = attempt.NextAttempt + if err := i.store.SaveScopeCatalogue(ctx, s.iata, s.url, attempt, nil); err != nil { + return fmt.Errorf("persist scope catalogue %s: %w", s.iata, err) + } + s.cache.AttemptedAt, s.cache.LastError = now, attempt.LastError + response, err := i.client.Do(request) + var entries []scopestore.Entry + var generated time.Time + var retryAfter time.Time + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + update.LastError = "request failed" // do not persist untrusted response bodies or URLs from errors + } else { + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + body, readErr := io.ReadAll(io.LimitReader(response.Body, MaxBody+1)) + if readErr == nil { + entries, generated, readErr = decode(body, s.iata) + } + if readErr != nil { + update.LastError = "invalid response" + } else { + update.Payload = body + } + case http.StatusNotModified: + if len(s.cache.Payload) == 0 || s.cache.ETag == "" { + update.LastError = "304 without cached catalogue" + } + default: + update.LastError = fmt.Sprintf("HTTP %d", response.StatusCode) + if response.StatusCode == http.StatusTooManyRequests || response.StatusCode == http.StatusServiceUnavailable { + // The published contract uses seconds. Ignore invalid/overflowing headers. + if seconds, parseErr := strconv.ParseInt(response.Header.Get("Retry-After"), 10, 64); parseErr == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) { + retryAfter = now.Add(time.Duration(seconds) * time.Second) + } + } + } + if update.LastError == "" { + etag := response.Header.Get("ETag") + if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") { + update.LastError = "invalid ETag" + update.Payload = nil + entries = nil + } else { + update.ETag = etag + if response.StatusCode == http.StatusNotModified && etag == "" { + update.ETag = s.cache.ETag + } + update.CheckedAt = now + } + } + } + if update.LastError != "" { + // Failures retry before the full interval, but never sooner than the rate limit or a Retry-After. + update.NextAttempt = now.Add(min(i.interval, failureRetry)) + if retryAfter.After(update.NextAttempt) { + update.NextAttempt = retryAfter + } + } + if err := i.store.SaveScopeCatalogue(ctx, s.iata, s.url, update, entries); err != nil { + // Avoid a retry every scheduler tick during a DB outage; never publish uncommitted keys. + s.cache.NextAttempt = update.NextAttempt + return fmt.Errorf("persist scope catalogue %s: %w", s.iata, err) + } + if len(update.Payload) > 0 { + s.cache.Payload = update.Payload + s.entries = entries + s.generated = generated + } + if !update.CheckedAt.IsZero() { + s.cache.CheckedAt = update.CheckedAt + s.cache.ETag = update.ETag + } + s.cache.AttemptedAt = now + s.cache.NextAttempt = update.NextAttempt + s.cache.LastError = update.LastError + if len(update.Payload) > 0 { + i.publish() + if i.onChange != nil { + i.onChange(ctx) + } + } + i.log(*s, "checked") + return nil +} + +func (i *Importer) publish() { + byName := make(map[string]scopestore.Entry, len(i.manual)) + for _, e := range i.manual { + byName[e.Name] = e + } + members := make(map[string][]string, len(i.sources)) + for _, s := range i.sources { + for _, candidate := range s.entries { + members[s.iata] = append(members[s.iata], candidate.Name) + entry, exists := byName[candidate.Name] + if exists && entry.IATAs == nil { + continue + } // manual metadata/keys win + if !exists { + entry = candidate + } + entry.IATAs = append(entry.IATAs, s.iata) + byName[entry.Name] = entry + } + } + names := make([]string, 0, len(byName)) + for name := range byName { + names = append(names, name) + } + slices.Sort(names) + entries := make([]scopestore.Entry, 0, len(names)) + for _, name := range names { + entries = append(entries, byName[name]) + } + i.scopes.Load(entries) + i.scopes.SetCatalogueMembers(members) +} + +func (i *Importer) log(s source, action string) { + level := slog.LevelInfo + if s.cache.LastError != "" && s.cache.LastError != "not listed" { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, "MeshMapper scopes "+action, "component", "meshmapper.scopes", "iata", s.iata, "source", s.url, + "names", len(s.entries), "generated_at", s.generated, "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError) +} + +func scopesEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_scopes.php") } + +func decode(body []byte, iata string) ([]scopestore.Entry, time.Time, error) { + var document struct { + GeneratedAt time.Time `json:"generated_at"` + Region string `json:"region"` + Zones []string `json:"zones"` + Repeaters, Scoped *int + Scopes *[]struct { + Name string + Repeaters, Default *int + Monitored, Wardriving *bool + } + } + if len(body) > MaxBody || !utf8.Valid(body) { + return nil, time.Time{}, fmt.Errorf("invalid catalogue size/encoding") + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, time.Time{}, err + } + if document.GeneratedAt.IsZero() || document.Region != iata || len(document.Zones) != 1 || document.Zones[0] != iata || + document.Repeaters == nil || document.Scoped == nil || *document.Scoped < 0 || *document.Repeaters < *document.Scoped || + document.Scopes == nil || len(*document.Scopes) > MaxScopes { + return nil, time.Time{}, fmt.Errorf("invalid regional catalogue") + } + entries := make([]scopestore.Entry, 0, len(*document.Scopes)) + seen := map[string]bool{} + for _, s := range *document.Scopes { + if s.Name == "" || len(s.Name) > 128 || strings.TrimSpace(s.Name) != s.Name || strings.ContainsFunc(s.Name, unicode.IsControl) || + s.Repeaters == nil || s.Default == nil || s.Monitored == nil || s.Wardriving == nil || + *s.Default < 0 || *s.Repeaters < *s.Default || *s.Repeaters > *document.Scoped { + return nil, time.Time{}, fmt.Errorf("invalid scope entry") + } + entry := scopestore.FromName(s.Name) + if len(entry.Name) < 2 || seen[entry.Name] { + return nil, time.Time{}, fmt.Errorf("empty or duplicate scope name") + } + seen[entry.Name] = true + entries = append(entries, entry) + } + return entries, document.GeneratedAt, nil +} diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go new file mode 100644 index 00000000..cd645508 --- /dev/null +++ b/internal/meshmapper/scopes_test.go @@ -0,0 +1,427 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func catalogue(names ...string) []byte { + rows := make([]map[string]any, 0, len(names)) + for _, name := range names { + rows = append(rows, map[string]any{"name": name, "repeaters": 0, "default": 0, "monitored": true, "wardriving": false}) + } + b, _ := json.Marshal(map[string]any{"generated_at": "2026-09-27T16:58:49Z", "region": "YOW", "zones": []string{"YOW"}, "repeaters": 5, "scoped": 3, "scopes": rows}) + return b +} + +type sourceKey struct{ iata, url string } + +type memoryStore struct { + iatas []string + rows map[sourceKey]Cache + fail bool + saved []scopestore.Entry +} + +func newMemoryStore(iatas ...string) *memoryStore { + return &memoryStore{iatas: iatas, rows: map[sourceKey]Cache{}} +} + +func (s *memoryStore) ListKnownIATAs(context.Context) ([]string, error) { + return slices.Clone(s.iatas), nil +} + +func (s *memoryStore) ListScopeCatalogues(context.Context) ([]Catalogue, error) { + var out []Catalogue + for key, c := range s.rows { + out = append(out, Catalogue{IATA: key.iata, URL: key.url, Cache: c}) + } + return out, nil +} + +func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, next Cache, entries []scopestore.Entry) error { + if s.fail { + return errors.New("offline") + } + old := s.rows[sourceKey{iata, url}] + if next.Payload == nil { + next.Payload = old.Payload + } + if next.CheckedAt.IsZero() { + next.CheckedAt = old.CheckedAt + next.ETag = old.ETag + } + s.rows[sourceKey{iata, url}] = next + s.saved = append(s.saved, entries...) + return nil +} + +// newImporter restores an importer for YOW whose source points at url. +func newImporter(t *testing.T, cfg config.MeshMapperScopesConfig, store *memoryStore, url string, scopes *scopestore.ScopeStore, manual ...scopestore.Entry) *Importer { + t.Helper() + imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopes, manual) + if err != nil { + t.Fatal(err) + } + if len(imp.sources) != 1 { + t.Fatal("expected one source", imp.sources) + } + imp.sources[0].url = url + return imp +} + +func names(store *scopestore.ScopeStore) []string { + out := []string{} + for _, e := range store.Entries() { + out = append(out, e.Name) + } + return out +} + +func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { + ctx := context.Background() + status, body, etag, after := 200, catalogue("yow", "manual"), `"v1"`, "" + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls > 1 && r.Header.Get("If-None-Match") != `"v1"` { + t.Errorf("missing conditional refresh: %v", r.Header) + } + w.Header().Set("ETag", etag) + w.Header().Set("Retry-After", after) + w.WriteHeader(status) + if status != 304 { + _, _ = w.Write(body) + } + })) + defer server.Close() + cfg := config.MeshMapperScopesConfig{Enabled: true} + store := newMemoryStore("YOW") + scopes := scopestore.New() + manual := scopestore.FromName("manual") + manual.TransportKey = []byte("manual wins") + imp := newImporter(t, cfg, store, server.URL, scopes, manual) + now := time.Now().UTC() + check := func() { + t.Helper() + now = now.Add(time.Minute) + if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { + t.Fatal(err) + } + } + check() + if got := names(scopes); !reflect.DeepEqual(got, []string{"#manual", "#yow"}) { + t.Fatal(got) + } + if string(scopes.Entries()[0].TransportKey) != "manual wins" || scopes.Entries()[0].IATAs != nil { + t.Fatal("manual key replaced/restricted") + } + if !reflect.DeepEqual(scopes.Entries()[1].IATAs, []string{"YOW"}) { + t.Fatal("import not bound to region") + } + if got := scopes.NamesForIATAs([]string{"YOW"}); !reflect.DeepEqual(got, []string{"#manual", "#yow"}) { + t.Fatal("catalogue membership must include manually overridden names", got) + } + generated := imp.sources[0].generated + status = 304 + check() + checked := imp.sources[0].cache.CheckedAt + if checked != now || imp.sources[0].generated != generated { + t.Fatal("304 lost freshness or changed generation time") + } + for _, failure := range []int{200, 404, 503, 302} { + status, body = failure, []byte(`{"scopes":null}`) + check() + if imp.sources[0].cache.CheckedAt != checked || len(scopes.Entries()) != 2 { + t.Fatal("failed refresh replaced known catalogue") + } + } + status, after = 429, "7200" + check() + if !imp.sources[0].cache.NextAttempt.Equal(now.Add(2 * time.Hour)) { + t.Fatal("Retry-After ignored") + } + restored := scopestore.New() + restart := newImporter(t, cfg, store, server.URL, restored, manual) + if !reflect.DeepEqual(names(restored), names(scopes)) { + t.Fatal("outage restart lost cached names") + } + before := calls + if err := restart.Refresh(ctx); err != nil { + t.Fatal(err) + } + if calls != before { + t.Fatal("restart retried before Retry-After") + } + status, after, body = 200, "", catalogue() + check() + if got := names(scopes); !reflect.DeepEqual(got, []string{"#manual"}) { + t.Fatalf("empty snapshot: %v", got) + } + store.fail = true + body = catalogue("new") + now = now.Add(time.Hour) + if err := imp.refresh(ctx, &imp.sources[0], now); err == nil || len(scopes.Entries()) != 1 { + t.Fatal("uncommitted names became active") + } +} + +func TestDecodeCatalogueBoundaries(t *testing.T) { + valid := string(catalogue("yow", "YOW", "$private")) + bad := []string{ + `null`, `{}`, `[]`, valid + `{}`, + strings.Replace(valid, `"region":"YOW"`, `"region":"PNW"`, 1), + strings.Replace(valid, `"zones":["YOW"]`, `"zones":["YOW","YVR"]`, 1), + strings.Replace(valid, `"scoped":3`, `"scoped":6`, 1), + strings.Replace(valid, `"repeaters":0`, `"repeaters":-1`, 1), + strings.Replace(valid, `"monitored":true,`, ``, 1), + strings.Replace(valid, `"generated_at":"2026-09-27T16:58:49Z"`, `"generated_at":null`, 1), + string(catalogue("yow", "#yow")), string(catalogue("#")), string(catalogue(" yow")), string(catalogue("a\nb")), + strings.Repeat(" ", MaxBody+1), + } + tooMany := []string{} + for n := 0; n <= MaxScopes; n++ { + tooMany = append(tooMany, fmt.Sprint(n)) + } + bad = append(bad, string(catalogue(tooMany...))) + for n, raw := range bad { + if _, _, err := decode([]byte(raw), "YOW"); err == nil { + t.Errorf("invalid case %d accepted", n) + } + } + entries, _, err := decode([]byte(valid), "YOW") + if err != nil || len(entries) != 3 { + t.Fatal(entries, err) + } + if entries[0].Name != "#yow" || entries[1].Name != "#YOW" || entries[2].Name != "$private" { + t.Fatal("case/prefix changed", entries) + } +} + +func TestNoRedirectAndResponseBounds(t *testing.T) { + targetCalls := 0 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { targetCalls++ })) + defer target.Close() + var mode atomic.Value + mode.Store("redirect") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch mode.Load().(string) { + case "redirect": + http.Redirect(w, r, target.URL, 302) + case "oversize": + _, _ = w.Write([]byte(strings.Repeat("x", MaxBody+1))) + case "timeout": + select { + case <-r.Context().Done(): + case <-time.After(time.Second): + } + case "304": + w.WriteHeader(304) + } + })) + defer server.Close() + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) + imp.client.Timeout = 20 * time.Millisecond + for _, value := range []string{"redirect", "oversize", "timeout", "304"} { + mode.Store(value) + if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil { + t.Fatal(err) + } + if imp.sources[0].cache.LastError == "" || len(imp.sources[0].entries) != 0 { + t.Fatal("invalid response accepted", value) + } + } + if targetCalls != 0 { + t.Fatal("followed redirect") + } +} + +func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { + ctx := context.Background() + store := newMemoryStore("YOW", "YVR") + for _, iata := range []string{"YOW", "YVR"} { + store.rows[sourceKey{iata, "old-" + iata}] = Cache{Payload: []byte(`{}`)} + store.rows[sourceKey{iata, iata}] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata)), AttemptedAt: time.Now()} + } + scopes := scopestore.New() + _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, NewDirectory(newZoneListMemory()), scopes, nil) + if err != nil || len(scopes.Entries()) != 1 || !reflect.DeepEqual(scopes.Entries()[0].IATAs, []string{"YOW", "YVR"}) { + t.Fatal("latest saved catalogue per IATA not restored", scopes.Entries(), err) + } + if got := scopes.NamesForIATAs([]string{"YVR"}); !reflect.DeepEqual(got, []string{"#can"}) { + t.Fatal(got) + } + // Disabled mode never calls the store. + disabled := scopestore.New() + if _, err = New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(newZoneListMemory()), disabled, nil); err != nil || len(disabled.Entries()) != 0 { + t.Fatal(err) + } +} + +func TestInvalidSavedCatalogueDoesNotBlockStartupOrRefresh(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-None-Match") != "" { + t.Error("invalid saved catalogue must not accept a 304") + } + _, _ = w.Write(catalogue("yow")) + })) + defer server.Close() + store := newMemoryStore("YOW") + store.rows[sourceKey{"YOW", server.URL}] = Cache{Payload: []byte(`{"invalid":true}`), ETag: `"old"`} + scopes := scopestore.New() + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, store, server.URL, scopes, scopestore.FromName("manual")) + if !reflect.DeepEqual(names(scopes), []string{"#manual"}) { + t.Fatal("invalid optional source blocked manual startup") + } + if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual", "#yow"}) { + t.Fatal("source did not recover on refresh", err, names(scopes)) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + imp.sources[0].cache.NextAttempt = time.Time{} + if err := imp.Refresh(ctx); err != nil { + t.Fatalf("normal shutdown is a task failure: %v", err) + } +} + +func TestRetryAfterNeverBeatsTheRateLimit(t *testing.T) { + for retryAfter, want := range map[string]time.Duration{"1800": time.Hour, "7200": 2 * time.Hour} { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", retryAfter) + w.WriteHeader(http.StatusTooManyRequests) + })) + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) + now := time.Now().UTC() + if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil { + t.Fatal(err) + } + server.Close() + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != want { + t.Fatalf("Retry-After %ss: next attempt in %v, want %v", retryAfter, got, want) + } + } +} + +func TestFailedRefreshRetriesSoon(t *testing.T) { + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) })) + defer server.Close() + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) + now := time.Now().UTC() + if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { + t.Fatal(err) + } + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != time.Hour { + t.Fatalf("next attempt after failure in %v, want 1h", got) + } +} + +// fakeSites serves a CA zone list and YOW's scope catalogue. +type fakeSites struct { + *httptest.Server + list string + listCalls, scopeHits int +} + +func newFakeSites(t *testing.T) *fakeSites { + f := &fakeSites{} + f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/get_zones.php": + f.listCalls++ + _, _ = w.Write([]byte(f.list)) + case "/get_scopes.php": + f.scopeHits++ + _, _ = w.Write(catalogue("yow")) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + f.list = `{"country":"CA","zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}]}` + t.Cleanup(f.Close) + return f +} + +func newDiscoveringImporter(t *testing.T, f *fakeSites, store *memoryStore, scopes *scopestore.ScopeStore) *Importer { + t.Helper() + dir := NewDirectory(newZoneListMemory()) + dir.listURL = f.URL + "/get_zones.php" + imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopes, nil) + if err != nil { + t.Fatal(err) + } + imp.scopesURL = func(site string) (string, bool) { return site + "get_scopes.php", strings.HasPrefix(site, f.URL) } + return imp +} + +func refreshN(t *testing.T, imp *Importer, n int) { + t.Helper() + for range n { + if err := imp.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestScopesDiscoverSourcesFromZoneList(t *testing.T) { + f := newFakeSites(t) + store := newMemoryStore() + scopes := scopestore.New() + imp := newDiscoveringImporter(t, f, store, scopes) + refreshN(t, imp, 2) + if f.listCalls != 0 || f.scopeHits != 0 { + t.Fatal("requested with no IATAs", f.listCalls, f.scopeHits) + } + store.iatas = []string{"YOW", "ZZZ"} + refreshN(t, imp, 3) + if f.listCalls != 1 || f.scopeHits != 1 || !reflect.DeepEqual(names(scopes), []string{"#yow"}) { + t.Fatal("source not discovered", f.listCalls, f.scopeHits, names(scopes)) + } + if _, ok := store.rows[sourceKey{"YOW", f.URL + "/get_scopes.php"}]; !ok { + t.Fatal("catalogue not saved under the discovered URL", store.rows) + } +} + +func TestScopesSkipUnlistedIATAs(t *testing.T) { + f := newFakeSites(t) + f.list = `{"country":"CA","zones":[]}` + imp := newDiscoveringImporter(t, f, newMemoryStore("YOW"), scopestore.New()) + refreshN(t, imp, 3) + s := imp.sources[0] + if f.scopeHits != 0 || s.cache.LastError != "not listed" || s.cache.NextAttempt.Sub(time.Now()) < 59*time.Minute { + t.Fatalf("unlisted IATA requested or retried early: hits=%d %+v", f.scopeHits, s.cache) + } +} + +func TestScopesRequestCutShortStillCounts(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() })) + defer server.Close() + store := newMemoryStore("YOW") + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, store, server.URL, scopestore.New()) + now := time.Now().UTC() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _ = imp.refresh(ctx, &imp.sources[0], now) + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != failureRetry { + t.Fatal("abandoned request retried inside the rate limit", got) + } + if row := store.rows[sourceKey{"YOW", server.URL}]; row.NextAttempt.Sub(now) != failureRetry { + t.Fatal("abandoned request not persisted", row) + } +} diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go new file mode 100644 index 00000000..4bec3897 --- /dev/null +++ b/internal/meshmapper/zones.go @@ -0,0 +1,408 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" +) + +const ( + ZonesURL = "https://meshmapper.net/get_zones.php" + MaxZoneList = 1 << 20 + MaxBoundaryBody = 8 << 20 // outlines are never simplified + + // MeshMapper asks clients to allow 60-120s; a request given up on may still use the day's call. + requestTimeout = 60 * time.Second + refreshTimeout = requestTimeout + 30*time.Second // room for the DB writes around a slow request + + zoneListFresh = config.MinZonesRefresh // get_zones.php allows one request per country per 23.5h + zoneFailureRetry = config.MinZonesRefresh // failed requests count too +) + +var zoneSite = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) + +// Boundary is one IATA's imported MeshMapper outline and its fetch state. +type Boundary struct { + IATA, URL string + Feature json.RawMessage + ETag string + CheckedAt, AttemptedAt, NextAttempt time.Time + LastError string +} + +type ZoneStore interface { + ListKnownIATAs(ctx context.Context) ([]string, error) + // ListHeardIATAs returns IATAs observers report from; group imports create IATAs too. + ListHeardIATAs(ctx context.Context) ([]string, error) + PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) + ListZoneBoundaries(ctx context.Context) ([]Boundary, error) + SaveZoneBoundary(ctx context.Context, b Boundary) error + ListRegionState(ctx context.Context) ([]RegionState, error) + ListIATADetails(ctx context.Context) ([]IATADetails, error) + UpsertIATADetails(ctx context.Context, iata, name string, lat, lng *float64) error + SaveImportedRegion(ctx context.Context, r RegionState) (bool, error) + PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) +} + +type zoneRegion struct { + country string + b Boundary +} + +// Zones is owned by one background task; listeners are wired before Restore. +type Zones struct { + store ZoneStore + enabled bool + seen map[string]bool + interval time.Duration + client *http.Client + dir *Directory + boundsURL func(site string) (string, bool) + regions []*zoneRegion + onChange func(ctx context.Context, iata string) + onUpdate func(imported map[string]json.RawMessage) + + importGroups bool + groupsSynced bool + synced syncInputs + onRegions func(ctx context.Context) + configured map[string]bool + onIATAs func(ctx context.Context) +} + +func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore, dir *Directory) *Zones { + return &Zones{store: store, enabled: cfg.Enabled, importGroups: cfg.Enabled && cfg.ImportGroups, seen: map[string]bool{}, interval: cfg.Interval(), dir: dir, + boundsURL: boundaryEndpoint, client: &http.Client{ + Timeout: requestTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} +} + +// OnChange receives each IATA whose served border changed, for cache invalidation. +func (z *Zones) OnChange(fn func(ctx context.Context, iata string)) { z.onChange = fn } + +// OnUpdate receives every imported boundary whenever the set changes. +func (z *Zones) OnUpdate(fn func(imported map[string]json.RawMessage)) { z.onUpdate = fn } + +// Restore prunes imports for unknown IATAs (all of them when disabled), and +// imported regions when group import is off, then loads saved boundaries, +// without making HTTP requests. +func (z *Zones) Restore(ctx context.Context) error { + if !z.importGroups { + if err := z.pruneGroups(ctx); err != nil { + return err + } + } + iatas, err := z.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) + } + keep := []string{} + if z.enabled { + keep = iatas + } + pruned, err := z.store.PruneZoneBoundaries(ctx, keep) + if err != nil { + return fmt.Errorf("prune MeshMapper boundaries: %w", err) + } + for _, iata := range pruned { + slog.Info("MeshMapper boundary removed", "component", "meshmapper.zones", "iata", iata) + z.changed(ctx, iata) + } + if !z.enabled { + return nil + } + saved, err := z.store.ListZoneBoundaries(ctx) + if err != nil { + return fmt.Errorf("restore MeshMapper boundaries: %w", err) + } + byIATA := map[string]Boundary{} + for _, b := range saved { + byIATA[b.IATA] = b + } + z.track(iatas, byIATA) + z.publish() + return nil +} + +// track adds a region for each IATA not seen before. +func (z *Zones) track(iatas []string, saved map[string]Boundary) { + for _, iata := range iatas { + if z.seen[iata] { + continue + } + z.seen[iata] = true + country := iatadb.CountryFor(iata) + if country == "" { + slog.Warn("MeshMapper boundary skipped: IATA has no known country", "component", "meshmapper.zones", "iata", iata) + continue + } + b, ok := saved[iata] + if !ok { + b = Boundary{IATA: iata} + } + z.regions = append(z.regions, &zoneRegion{country: country, b: b}) + if ok { + z.log(b, "restored") + } + } +} + +// Refresh makes at most one request. The scheduler serializes calls every 15s. +func (z *Zones) Refresh(ctx context.Context) (err error) { + parent := ctx + defer func() { + if parent.Err() != nil { + err = nil + } + }() + ctx, cancel := context.WithTimeout(ctx, refreshTimeout) + defer cancel() + now := time.Now().UTC() + // IATAs are created from traffic, so pick up new ones every tick. + iatas, err := z.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) + } + z.track(iatas, nil) + if err := z.syncDirectory(ctx, iatas); err != nil { + return err + } + for _, r := range z.regions { + if now.Before(r.b.NextAttempt) { + continue + } + zones, fetched, err := z.dir.List(ctx, r.country, now) + if err != nil || fetched { + return err + } + if zones == nil { + continue + } + return z.refresh(ctx, r, zones, now) + } + return nil +} + +func (z *Zones) refresh(ctx context.Context, r *zoneRegion, zones map[string]zoneEntry, now time.Time) error { + update := Boundary{IATA: r.b.IATA, URL: r.b.URL, AttemptedAt: now, NextAttempt: now.Add(z.interval)} + var retryAt time.Time + entry, listed := zones[r.b.IATA] + endpoint, valid := "", false + if listed { + update.URL = entry.url + endpoint, valid = z.boundsURL(entry.url) + } + switch { + case !listed: + update.LastError = "not listed" + case !entry.hasBoundary: + update.LastError = "no boundary" + case !valid: + update.LastError = "invalid site URL" + default: + etag := "" + if r.b.Feature != nil { + etag = r.b.ETag + } + // Recorded first: an abandoned request may still have used the region's call. + attempt := Boundary{IATA: r.b.IATA, URL: update.URL, AttemptedAt: now, NextAttempt: now.Add(min(z.interval, zoneFailureRetry)), LastError: "no response"} + r.b.NextAttempt = attempt.NextAttempt + if err := z.store.SaveZoneBoundary(ctx, attempt); err != nil { + return fmt.Errorf("persist MeshMapper boundary %s: %w", r.b.IATA, err) + } + r.b.URL, r.b.AttemptedAt, r.b.LastError = attempt.URL, now, attempt.LastError + status, body, header, err := get(ctx, z.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxBoundaryBody) + if err != nil { + return err + } + switch status { + case http.StatusOK: + feature, decodeErr := decodeBoundary(body, r.b.IATA) + switch { + case decodeErr != nil: + update.LastError = "invalid response" // includes truncated bodies; retry + case feature == nil: + update.LastError = "no boundary" + default: + update.Feature = feature + } + case http.StatusNotModified: + if r.b.Feature == nil { + update.LastError = "304 without cached boundary" + } + default: + update.LastError = statusProblem(status, header, now, &retryAt) + } + if update.LastError == "" { + etag := header.Get("ETag") + if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") { + update.LastError, update.Feature = "invalid ETag", nil + } else { + if status == http.StatusNotModified && etag == "" { + etag = r.b.ETag + } + update.ETag, update.CheckedAt = etag, now + } + } + if update.LastError != "" && update.LastError != "no boundary" { + update.NextAttempt = now.Add(min(z.interval, zoneFailureRetry)) + if retryAt.After(update.NextAttempt) { + update.NextAttempt = retryAt + } + } + } + if err := z.store.SaveZoneBoundary(ctx, update); err != nil { + r.b.NextAttempt = update.NextAttempt // avoid retrying every tick during a DB outage + return fmt.Errorf("persist MeshMapper boundary %s: %w", r.b.IATA, err) + } + changed := update.Feature != nil && !bytes.Equal(update.Feature, r.b.Feature) + if update.Feature != nil { + r.b.Feature = update.Feature + } + if !update.CheckedAt.IsZero() { + r.b.CheckedAt, r.b.ETag = update.CheckedAt, update.ETag + } + r.b.URL, r.b.AttemptedAt, r.b.NextAttempt, r.b.LastError = update.URL, now, update.NextAttempt, update.LastError + if changed { + z.changed(ctx, r.b.IATA) + z.publish() + } + z.log(r.b, "checked") + return nil +} + +// get returns transport errors only when ctx ended; other failures become status 0. +func get(ctx context.Context, client *http.Client, agent, endpoint, etag string, limit int64) (int, []byte, http.Header, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return 0, nil, nil, err + } + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", agent) + if etag != "" { + request.Header.Set("If-None-Match", etag) + } + response, err := client.Do(request) + if err != nil { + if ctx.Err() != nil { + return 0, nil, nil, ctx.Err() + } + return 0, nil, http.Header{}, nil + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return response.StatusCode, nil, response.Header, nil + } + body, err := io.ReadAll(io.LimitReader(response.Body, limit+1)) + if err != nil || int64(len(body)) > limit { + return response.StatusCode, nil, response.Header, nil + } + return response.StatusCode, body, response.Header, nil +} + +// statusProblem describes a failed response and reads Retry-After on 429/503. +func statusProblem(status int, header http.Header, now time.Time, retryAt *time.Time) string { + if status == 0 { + return "request failed" // don't persist untrusted error text + } + if status == http.StatusTooManyRequests || status == http.StatusServiceUnavailable { + if seconds, err := strconv.ParseInt(header.Get("Retry-After"), 10, 64); err == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) { + *retryAt = now.Add(time.Duration(seconds) * time.Second) + } + } + return fmt.Sprintf("HTTP %d", status) +} + +func (z *Zones) changed(ctx context.Context, iata string) { + if z.onChange != nil { + z.onChange(ctx, iata) + } +} + +func (z *Zones) publish() { + if z.onUpdate == nil { + return + } + imported := map[string]json.RawMessage{} + for _, r := range z.regions { + if r.b.Feature != nil { + imported[r.b.IATA] = r.b.Feature + } + } + z.onUpdate(imported) +} + +func (z *Zones) log(b Boundary, action string) { + level := slog.LevelInfo + if b.LastError != "" && b.LastError != "no boundary" && b.LastError != "not listed" { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, "MeshMapper boundary "+action, "component", "meshmapper.zones", "iata", b.IATA, + "source", b.URL, "imported", b.Feature != nil, "checked_at", b.CheckedAt, "next_attempt", b.NextAttempt, "last_error", b.LastError) +} + +func boundaryEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_geojson.php") } + +// siteEndpoint accepts only a published region site root. +func siteEndpoint(site, file string) (string, bool) { + u, err := url.Parse(site) + if err != nil || u.Scheme != "https" || u.User != nil || !zoneSite.MatchString(u.Host) || + (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return "", false + } + return "https://" + u.Host + "/" + file, true +} + +// decodeBoundary returns nil when MeshMapper has no usable outline for the region. +func decodeBoundary(body []byte, iata string) (json.RawMessage, error) { + var collection struct { + Type string `json:"type"` + Features []json.RawMessage `json:"features"` + } + if err := json.Unmarshal(body, &collection); err != nil { + return nil, err + } + if collection.Type != "FeatureCollection" || len(collection.Features) != 1 { + return nil, fmt.Errorf("expected one region feature") + } + var feature struct { + Geometry json.RawMessage `json:"geometry"` + Properties struct { + Code string `json:"code"` + } `json:"properties"` + } + if err := json.Unmarshal(collection.Features[0], &feature); err != nil { + return nil, err + } + if !strings.EqualFold(feature.Properties.Code, iata) { + return nil, fmt.Errorf("feature is for another region") + } + if len(feature.Geometry) == 0 || string(feature.Geometry) == "null" { + return nil, nil + } + validated, err := config.ValidateBorder(collection.Features[0]) + if err != nil { + return nil, err + } + // Foreign marking must accept anything the map serves. + if _, err := config.BuildLocalBorders(nil, map[string]json.RawMessage{iata: validated}); err != nil { + return nil, err + } + return validated, nil +} diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go new file mode 100644 index 00000000..b4bb35fb --- /dev/null +++ b/internal/meshmapper/zones_test.go @@ -0,0 +1,587 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +const square = `{"type":"Polygon","coordinates":[[[-76,45],[-75,45],[-75,46],[-76,46],[-76,45]]]}` + +func boundaryBody(code, geometry string) string { + return `{"type":"FeatureCollection","generated_at":"2026-09-25T15:00:00Z","features":[{"type":"Feature","geometry":` + geometry + + `,"properties":{"code":"` + code + `","name":"Ottawa, CA","center":[-75.7,45.4],"radius_km":42,"has_boundary":true}}]}` +} + +type zoneMemoryStore struct { + iatas []string + heard []string + rows map[string]Boundary + pruned []string + fail bool + regions map[string]RegionState + lists *zoneListMemory + details map[string]IATADetails + writes int +} + +func (s *zoneMemoryStore) ListIATADetails(context.Context) ([]IATADetails, error) { + var out []IATADetails + for _, iata := range s.iatas { + d := s.details[iata] + d.IATA = iata + out = append(out, d) + } + return out, nil +} + +func (s *zoneMemoryStore) UpsertIATADetails(_ context.Context, iata, name string, lat, lng *float64) error { + if s.details == nil { + s.details = map[string]IATADetails{} + } + s.details[iata] = IATADetails{IATA: iata, Name: name, Lat: lat, Lng: lng} + s.writes++ + return nil +} + +func (s *zoneMemoryStore) ListRegionState(context.Context) ([]RegionState, error) { + var out []RegionState + for _, r := range s.regions { + out = append(out, r) + } + return out, nil +} + +func (s *zoneMemoryStore) SaveImportedRegion(_ context.Context, r RegionState) (bool, error) { + if s.regions == nil { + s.regions = map[string]RegionState{} + } + if cur, ok := s.regions[r.Slug]; ok && !cur.Imported { + return false, nil + } + r.Imported = true + s.regions[r.Slug] = r + for _, m := range r.IATAs { // members become known IATAs, as AddIATAs does + if !slices.Contains(s.iatas, m) { + s.iatas = append(s.iatas, m) + } + } + return true, nil +} + +func (s *zoneMemoryStore) PruneImportedRegions(_ context.Context, keep []string) ([]string, error) { + var removed []string + for slug, r := range s.regions { + if r.Imported && !slices.Contains(keep, slug) { + delete(s.regions, slug) + removed = append(removed, slug) + } + } + return removed, nil +} + +func (s *zoneMemoryStore) ListKnownIATAs(context.Context) ([]string, error) { + return slices.Clone(s.iatas), nil +} + +func (s *zoneMemoryStore) ListHeardIATAs(context.Context) ([]string, error) { + return slices.Clone(s.heard), nil +} + +func (s *zoneMemoryStore) PruneZoneBoundaries(_ context.Context, keep []string) ([]string, error) { + var removed []string + for iata := range s.rows { + if !slices.Contains(keep, iata) { + delete(s.rows, iata) + removed = append(removed, iata) + } + } + s.pruned = removed + return removed, nil +} + +func (s *zoneMemoryStore) ListZoneBoundaries(context.Context) ([]Boundary, error) { + var out []Boundary + for _, b := range s.rows { + out = append(out, b) + } + return out, nil +} + +func (s *zoneMemoryStore) SaveZoneBoundary(_ context.Context, b Boundary) error { + if s.fail { + return errors.New("offline") + } + old := s.rows[b.IATA] + if b.Feature == nil { + b.Feature = old.Feature + } + if b.CheckedAt.IsZero() { + b.CheckedAt, b.ETag = old.CheckedAt, old.ETag + } + s.rows[b.IATA] = b + return nil +} + +type zoneListMemory struct{ rows map[string]ZoneList } + +func newZoneListMemory() *zoneListMemory { return &zoneListMemory{rows: map[string]ZoneList{}} } + +func (s *zoneListMemory) ListZoneLists(context.Context) ([]ZoneList, error) { + var out []ZoneList + for _, l := range s.rows { + out = append(out, l) + } + return out, nil +} + +func (s *zoneListMemory) SaveZoneList(_ context.Context, l ZoneList) error { + old := s.rows[l.Country] + if l.Payload == nil { + l.Payload = old.Payload + } + if l.FetchedAt.IsZero() { + l.FetchedAt, l.ETag = old.FetchedAt, old.ETag + } + s.rows[l.Country] = l + return nil +} + +type fakeMeshMapper struct { + *httptest.Server + list, boundary string + listStatus, status int + etag, retryAfter string + listCalls, calls int + ifNoneMatch []string + otherStatus int // get_zones.php status for countries other than CA; 0 is unexpected + hang atomic.Bool // requests wait until the client gives up + hung atomic.Int32 +} + +func newFakeMeshMapper(t *testing.T) *fakeMeshMapper { + f := &fakeMeshMapper{listStatus: 200, status: 200, etag: `"v1"`, boundary: boundaryBody("YOW", square)} + f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if f.hang.Load() { + f.hung.Add(1) + <-r.Context().Done() + return + } + switch r.URL.Path { + case "/get_zones.php": + f.listCalls++ + if r.URL.Query().Get("country") != "CA" { + if f.otherStatus == 0 { + t.Errorf("country query: %s", r.URL.RawQuery) + } + w.WriteHeader(f.otherStatus) + return + } + w.WriteHeader(f.listStatus) + _, _ = w.Write([]byte(f.list)) + case "/get_geojson.php": + f.calls++ + f.ifNoneMatch = append(f.ifNoneMatch, r.Header.Get("If-None-Match")) + w.Header().Set("ETag", f.etag) + w.Header().Set("Retry-After", f.retryAfter) + w.WriteHeader(f.status) + if f.status == 200 { + _, _ = w.Write([]byte(f.boundary)) + } + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + f.list = `{"country":"CA","count":1,"zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}],"groups":[]}` + t.Cleanup(f.Close) + return f +} + +type zoneHarness struct { + z *Zones + store *zoneMemoryStore + changed []string + imported map[string]json.RawMessage +} + +func newZoneHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, enabled bool) *zoneHarness { + t.Helper() + h := &zoneHarness{store: store} + if store.iatas == nil { + store.iatas = []string{"YOW"} + } + if store.lists == nil { + store.lists = newZoneListMemory() + } + dir := NewDirectory(store.lists) + dir.listURL = f.URL + "/get_zones.php" + if err := dir.Restore(context.Background()); err != nil { + t.Fatal(err) + } + h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, store, dir) + h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", strings.HasPrefix(site, f.URL) } + h.z.OnChange(func(_ context.Context, iata string) { h.changed = append(h.changed, iata) }) + h.z.OnUpdate(func(imported map[string]json.RawMessage) { h.imported = imported }) + if err := h.z.Restore(context.Background()); err != nil { + t.Fatal(err) + } + return h +} + +// tick forces the region due and runs refreshes until the boundary request is made. +func (h *zoneHarness) tick(t *testing.T) Boundary { + t.Helper() + h.z.regions[0].b.NextAttempt = time.Time{} + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + return h.store.rows["YOW"] +} + +func TestZonesImportAndConditionalRefresh(t *testing.T) { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + b := h.tick(t) + if f.listCalls != 1 || f.calls != 1 || b.Feature == nil || b.ETag != `"v1"` || b.LastError != "" || b.URL != f.URL+"/" { + t.Fatalf("import failed: %+v list=%d calls=%d", b, f.listCalls, f.calls) + } + if !slices.Equal(h.changed, []string{"YOW"}) || h.imported["YOW"] == nil { + t.Fatal("listeners not told", h.changed, h.imported) + } + if !strings.Contains(string(b.Feature), `"bbox"`) || !strings.Contains(string(b.Feature), `"code":"YOW"`) { + t.Fatal("feature not validated with bbox and properties", string(b.Feature)) + } + if got := b.NextAttempt.Sub(b.AttemptedAt); got != 24*time.Hour { + t.Fatal("refresh interval", got) + } + + f.status = 304 + b = h.tick(t) + if f.listCalls != 1 || f.ifNoneMatch[1] != `"v1"` || b.Feature == nil || b.LastError != "" || len(h.changed) != 1 { + t.Fatalf("304 mishandled: %+v list=%d changed=%v", b, f.listCalls, h.changed) + } + + f.status, f.etag = 200, `"v2"` + f.boundary = boundaryBody("YOW", `{"type":"Polygon","coordinates":[[[-77,45],[-75,45],[-75,46],[-77,46],[-77,45]]]}`) + b = h.tick(t) + if b.ETag != `"v2"` || len(h.changed) != 2 || !strings.Contains(string(h.imported["YOW"]), "-77") { + t.Fatal("changed boundary not published", b.ETag, h.changed) + } +} + +func TestZonesKeepLastGoodBoundary(t *testing.T) { + for _, tc := range []struct { + name, boundary string + status int + wantErr string + wantRetry time.Duration + }{ + {"null geometry", boundaryBody("YOW", "null"), 200, "no boundary", 24 * time.Hour}, + {"not found", "", 404, "HTTP 404", 24 * time.Hour}, + {"unavailable", "", 503, "HTTP 503", 24 * time.Hour}, + {"truncated", boundaryBody("YOW", square)[:80], 200, "invalid response", 24 * time.Hour}, + {"other region", boundaryBody("YVR", square), 200, "invalid response", 24 * time.Hour}, + {"oversized", `{"type":"FeatureCollection","features":[],"pad":"` + strings.Repeat("x", MaxBoundaryBody) + `"}`, 200, "invalid response", 24 * time.Hour}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + good := h.tick(t).Feature + f.status, f.boundary = tc.status, tc.boundary + b := h.tick(t) + if b.LastError != tc.wantErr || string(b.Feature) != string(good) || string(h.imported["YOW"]) != string(good) || len(h.changed) != 1 { + t.Fatalf("got %q, feature kept=%v", b.LastError, string(b.Feature) == string(good)) + } + if got := b.NextAttempt.Sub(b.AttemptedAt); got != tc.wantRetry { + t.Fatal("retry", got) + } + }) + } +} + +func TestZonesSkipRegionsWithoutBoundary(t *testing.T) { + for _, tc := range []struct{ name, zones, want string }{ + {"not listed", `[]`, "not listed"}, + {"no boundary", `[{"code":"YOW","url":"URL/","has_boundary":false}]`, "no boundary"}, + {"foreign site", `[{"code":"yow","url":"https://example.com/","has_boundary":true}]`, "invalid site URL"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFakeMeshMapper(t) + f.list = `{"country":"CA","count":1,"zones":` + strings.ReplaceAll(tc.zones, "URL", f.URL) + `,"groups":[]}` + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + b := h.tick(t) + if f.calls != 0 || b.LastError != tc.want || b.Feature != nil || len(h.changed) != 0 { + t.Fatalf("got %+v calls=%d", b, f.calls) + } + if got := b.NextAttempt.Sub(b.AttemptedAt); got != 24*time.Hour { + t.Fatal("authoritative answer retried early", got) + } + }) + } +} + +func TestZonesListFailureBacksOff(t *testing.T) { + f := newFakeMeshMapper(t) + f.listStatus = 500 + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + for range 3 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if f.listCalls != 1 || f.calls != 0 || len(h.store.rows) != 0 { + t.Fatal("failed list retried early", f.listCalls, f.calls) + } + f.listStatus, f.list = 200, `{"country":"US","zones":[]}` + h.z.dir.lists["CA"].nextAttempt, h.z.dir.lists["CA"].fetchedAt = time.Time{}, time.Time{} + _ = h.z.Refresh(context.Background()) + if h.z.dir.lists["CA"].zones != nil { + t.Fatal("list for another country accepted") + } +} + +func TestZonesRateLimitWaitsOnlyForThatRegion(t *testing.T) { + for retryAfter, want := range map[string]time.Duration{"120": 24 * time.Hour, "172800": 48 * time.Hour} { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + f.status, f.retryAfter = 429, retryAfter + b := h.tick(t) + if b.LastError != "HTTP 429" || b.NextAttempt.Sub(b.AttemptedAt) != want { + t.Fatalf("Retry-After %ss: next attempt in %v, want %v", retryAfter, b.NextAttempt.Sub(b.AttemptedAt), want) + } + restarted := newZoneHarness(t, f, h.store, true) + if err := restarted.z.Refresh(context.Background()); err != nil || f.calls != 1 { + t.Fatal("requested again before the rate limit allows", err, f.calls) + } + } +} + +func TestZoneListSurvivesRestart(t *testing.T) { + f := newFakeMeshMapper(t) + store := &zoneMemoryStore{rows: map[string]Boundary{}} + h := newZoneHarness(t, f, store, true) + h.tick(t) + if f.listCalls != 1 { + t.Fatal("list not fetched", f.listCalls) + } + if got := h.z.dir.lists["CA"].nextAttempt.Sub(h.z.dir.lists["CA"].fetchedAt); got != 24*time.Hour { + t.Fatal("zone list refetched sooner than the rate limit allows", got) + } + restarted := newZoneHarness(t, f, store, true) + restarted.tick(t) + if f.listCalls != 1 || f.calls != 2 { + t.Fatal("restart refetched the zone list", f.listCalls, f.calls) + } +} + +func TestZonesRestoreAndPrune(t *testing.T) { + f := newFakeMeshMapper(t) + store := &zoneMemoryStore{rows: map[string]Boundary{}} + first := newZoneHarness(t, f, store, true) + first.tick(t) + store.rows["OLD"] = Boundary{IATA: "OLD", Feature: json.RawMessage(`{}`)} + + restarted := newZoneHarness(t, f, store, true) + if !slices.Equal(store.pruned, []string{"OLD"}) || !slices.Equal(restarted.changed, []string{"OLD"}) { + t.Fatal("unknown IATA not pruned", store.pruned, restarted.changed) + } + if restarted.imported["YOW"] == nil || f.calls != 1 { + t.Fatal("saved boundary not restored without a request") + } + if err := restarted.z.Refresh(context.Background()); err != nil || f.calls != 1 || f.listCalls != 1 { + t.Fatal("refreshed before due", err, f.calls) + } + + disabled := newZoneHarness(t, f, store, false) + if len(store.rows) != 0 || !slices.Equal(disabled.changed, []string{"YOW"}) || len(disabled.z.regions) != 0 { + t.Fatal("disabling did not remove imports") + } +} + +func TestZonesPickUpNewIATAs(t *testing.T) { + f := newFakeMeshMapper(t) + store := &zoneMemoryStore{iatas: []string{}, rows: map[string]Boundary{}} + h := newZoneHarness(t, f, store, true) + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if f.listCalls != 0 || f.calls != 0 { + t.Fatal("requested with no IATAs", f.listCalls, f.calls) + } + store.iatas = []string{"YOW", "ZZZ"} + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if len(h.z.regions) != 1 || h.imported["YOW"] == nil || !slices.Equal(h.changed, []string{"YOW"}) { + t.Fatal("new IATA not imported", len(h.z.regions), h.changed) + } +} + +func TestZonesPersistFailureWaitsForNextAttempt(t *testing.T) { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}, fail: true}, true) + h.z.regions[0].b.NextAttempt = time.Time{} + _ = h.z.Refresh(context.Background()) + if err := h.z.Refresh(context.Background()); err == nil { + t.Fatal("persist error hidden") + } + if h.z.regions[0].b.NextAttempt.IsZero() || h.imported["YOW"] != nil || len(h.changed) != 0 { + t.Fatal("uncommitted boundary published") + } +} + +func TestBoundaryEndpoint(t *testing.T) { + for site, want := range map[string]string{ + "https://yow.meshmapper.net/": "https://yow.meshmapper.net/get_geojson.php", + "https://yow.meshmapper.net": "https://yow.meshmapper.net/get_geojson.php", + "http://yow.meshmapper.net/": "", + "https://meshmapper.net.evil.com/": "", + "https://user@yow.meshmapper.net/": "", + "https://yow.meshmapper.net/x/": "", + "https://yow.meshmapper.net/?a=b": "", + "https://yow.meshmapper.net:8443/": "", + "https://yow.meshmapper.net/#fragment": "", + } { + got, ok := boundaryEndpoint(site) + if got != want || ok != (want != "") { + t.Errorf("%s: %q %v", site, got, ok) + } + } +} + +func TestDecodeBoundaryRejectsUnusableOutlines(t *testing.T) { + for name, body := range map[string]string{ + "two features": `{"type":"FeatureCollection","features":[{"type":"Feature","geometry":null,"properties":{"code":"YOW"}},{"type":"Feature","geometry":null,"properties":{"code":"YOW"}}]}`, + "not a polygon": boundaryBody("YOW", `{"type":"Point","coordinates":[-75,45]}`), + "open ring": boundaryBody("YOW", `{"type":"Polygon","coordinates":[[[-76,45],[-75,45],[-75,46],[-76,46]]]}`), + "antimeridian": boundaryBody("YOW", `{"type":"Polygon","coordinates":[[[179,10],[-179,10],[-179,20],[179,20],[179,10]]]}`), + "zero area": boundaryBody("YOW", `{"type":"Polygon","coordinates":[[[-76,45],[-75,45],[-74,45],[-76,45]]]}`), + } { + if _, err := decodeBoundary([]byte(body), "YOW"); err == nil { + t.Errorf("%s accepted", name) + } + } + if feature, err := decodeBoundary([]byte(boundaryBody("yow", square)), "YOW"); err != nil || feature == nil { + t.Fatal("case-insensitive code rejected", err) + } +} + +func TestClientsAllowMeshMapperTimeout(t *testing.T) { + ctx := context.Background() + scopes, err := New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(nil), scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + channels, err := NewChannels(ctx, config.MeshMapperChannelsConfig{}, newChannelMemoryStore(), NewDirectory(nil), keystore.NewMapKeyStore(nil)) + if err != nil { + t.Fatal(err) + } + for name, c := range map[string]*http.Client{ + "directory": NewDirectory(nil).client, + "zones": NewZones(config.MeshMapperZonesConfig{}, nil, nil).client, + "scopes": scopes.client, + "channels": channels.client, + } { + if c.Timeout < 60*time.Second { + t.Errorf("%s client gives up after %v; MeshMapper asks for 60-120s", name, c.Timeout) + } + } + if refreshTimeout <= requestTimeout { + t.Fatal("a refresh must outlast its request") + } +} + +func TestZoneListCutShortStillCounts(t *testing.T) { + f := newFakeMeshMapper(t) + f.hang.Store(true) + lists := newZoneListMemory() + dir := NewDirectory(lists) + dir.listURL = f.URL + "/get_zones.php" + now := time.Now().UTC() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if _, fetched, _ := dir.List(ctx, "CA", now); !fetched { + t.Fatal("first call should spend the request") + } + f.hang.Store(false) + if zones, fetched, err := dir.List(context.Background(), "CA", now.Add(15*time.Second)); zones != nil || fetched || err != nil || f.listCalls != 0 || f.hung.Load() != 1 { + t.Fatal("abandoned request retried inside the rate limit", f.listCalls, fetched, err) + } + if got := lists.rows["CA"].NextAttempt.Sub(now); got != zoneListFresh { + t.Fatal("abandoned request not persisted", lists.rows["CA"]) + } +} + +func TestZoneListFetchDoesNotBlockOtherCallers(t *testing.T) { + release := make(chan struct{}) + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + <-release + _, _ = w.Write([]byte(`{"country":"CA","zones":[]}`)) + })) + defer server.Close() + defer close(release) + dir := NewDirectory(newZoneListMemory()) + dir.listURL = server.URL + now := time.Now().UTC() + go func() { _, _, _ = dir.List(context.Background(), "CA", now) }() + for calls.Load() == 0 { + time.Sleep(time.Millisecond) + } + done := make(chan struct{}) + go func() { + defer close(done) + if zones, fetched, err := dir.List(context.Background(), "CA", now); zones != nil || fetched || err != nil { + t.Error("second caller should back off while the list is in flight", fetched, err) + } + _, _, _, _ = dir.snapshot([]string{"CA"}) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("caller waited on another task's request") + } + if calls.Load() != 1 { + t.Fatal("duplicate list request", calls.Load()) + } +} + +func TestBoundaryCutShortStillCounts(t *testing.T) { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + if err := h.z.Refresh(context.Background()); err != nil || f.listCalls != 1 { + t.Fatal("list not fetched", err) + } + f.hang.Store(true) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _ = h.z.Refresh(ctx) + f.hang.Store(false) + if err := h.z.Refresh(context.Background()); err != nil || f.calls != 0 || f.hung.Load() != 1 { + t.Fatal("abandoned boundary request retried inside the rate limit", err, f.calls) + } + if b := h.store.rows["YOW"]; b.NextAttempt.Sub(b.AttemptedAt) != 24*time.Hour { + t.Fatal("abandoned boundary request not persisted", b) + } +} diff --git a/internal/presence/broker_freshness_test.go b/internal/presence/broker_freshness_test.go new file mode 100644 index 00000000..887a6e7c --- /dev/null +++ b/internal/presence/broker_freshness_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package presence + +import ( + "context" + "github.com/google/uuid" + "testing" + "time" +) + +type brokerTestStore struct { + Store + initial []bool + seen, packets []time.Time +} + +func (s *brokerTestStore) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, packet bool) error { + s.initial = append(s.initial, packet) + return nil +} +func (s *brokerTestStore) TouchObserverBrokers(_ context.Context, _ []uuid.UUID, _ []string, seen, packets []time.Time) error { + s.seen = seen + s.packets = packets + return nil +} + +func TestBrokerPresenceDoesNotInventPacketFreshness(t *testing.T) { + ctx := context.Background() + store := &brokerTestStore{} + c := New(store, time.Second, time.Minute) + now := time.Now() + c.now = func() time.Time { return now } + id := uuid.New() + c.UpsertObserverBroker(ctx, id, "one", false) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if len(store.initial) != 1 || store.initial[0] || !store.packets[0].IsZero() { + t.Fatal("status-only activity invented a packet") + } + now = now.Add(time.Minute) + packetTime := now + c.UpsertObserverBroker(ctx, id, "one", true) + now = now.Add(time.Minute) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if !store.seen[0].Equal(now) || !store.packets[0].Equal(packetTime) { + t.Fatal("later status changed packet freshness") + } +} diff --git a/internal/presence/coalescer.go b/internal/presence/coalescer.go index c0673bb0..221aac9e 100644 --- a/internal/presence/coalescer.go +++ b/internal/presence/coalescer.go @@ -11,7 +11,8 @@ package presence import ( "context" - "log" + "fmt" + "log/slog" "sync" "time" @@ -24,29 +25,37 @@ import ( type Store interface { ingest.DB - // TouchObservers applies coalesced last_seen bumps and observation_count - // deltas for the given observer IDs in one statement. - TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error + // TouchObservers applies coalesced last_seen bumps, observation_count + // deltas and last_iata for the given observer IDs in one statement. An + // IATA only replaces one heard earlier (iataAts), so a late flush can't regress it. + TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, iatas []string, iataAts []time.Time) error // TouchObserverBrokers applies coalesced last_seen/last_packet_at bumps // for the given (observer, broker) pairs in one statement. - TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error + TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error // TouchPackets applies coalesced last_heard_at bumps for the given packet // hashes in one statement. TouchPackets(ctx context.Context, hashes [][]byte, heard []time.Time) error + + DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) } type identity struct { id uuid.UUID name string + seen time.Time } type observerBump struct { - seen time.Time - count int32 + seen time.Time + count int32 + iata string + iataAt time.Time } +type brokerBump struct{ seen, packet time.Time } + type brokerKey struct { id uuid.UUID broker string @@ -64,7 +73,7 @@ type Coalescer struct { identities map[string]identity // pubkey -> observer row dirtyObservers map[uuid.UUID]observerBump knownBrokers map[brokerKey]struct{} - dirtyBrokers map[brokerKey]time.Time + dirtyBrokers map[brokerKey]brokerBump knownIATAs map[string]struct{} packetsSeen map[string]time.Time // hash -> last observation dirtyPackets map[string]time.Time @@ -82,7 +91,7 @@ func New(store Store, flushInterval, packetTTL time.Duration) *Coalescer { identities: make(map[string]identity), dirtyObservers: make(map[uuid.UUID]observerBump), knownBrokers: make(map[brokerKey]struct{}), - dirtyBrokers: make(map[brokerKey]time.Time), + dirtyBrokers: make(map[brokerKey]brokerBump), knownIATAs: make(map[string]struct{}), packetsSeen: make(map[string]time.Time), dirtyPackets: make(map[string]time.Time), @@ -91,42 +100,52 @@ func New(store Store, flushInterval, packetTTL time.Duration) *Coalescer { // UpsertObserver serves repeat lookups from the identity cache and records a // last_seen/observation_count bump instead of writing through. -func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) { +func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte, iata string) (uuid.UUID, string, error) { key := string(pubkey) c.mu.Lock() if ident, ok := c.identities[key]; ok { bump := c.dirtyObservers[ident.id] bump.seen = c.now() bump.count++ + if iata != "" { + bump.iata, bump.iataAt = iata, bump.seen + } c.dirtyObservers[ident.id] = bump + ident.seen = bump.seen + c.identities[key] = ident c.mu.Unlock() return ident.id, ident.name, nil } c.mu.Unlock() - id, name, err := c.Store.UpsertObserver(ctx, pubkey) + id, name, err := c.Store.UpsertObserver(ctx, pubkey, iata) if err != nil { return id, name, err } c.mu.Lock() - c.identities[key] = identity{id: id, name: name} + c.identities[key] = identity{id: id, name: name, seen: c.now()} c.mu.Unlock() return id, name, nil } // UpsertObserverBroker writes through the first time a pair is seen and // records a bump afterwards. -func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { key := brokerKey{id: observerID, broker: brokerName} c.mu.Lock() if _, ok := c.knownBrokers[key]; ok { - c.dirtyBrokers[key] = c.now() + bump := c.dirtyBrokers[key] + bump.seen = c.now() + if isPacket { + bump.packet = bump.seen + } + c.dirtyBrokers[key] = bump c.mu.Unlock() return nil } c.mu.Unlock() - if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName); err != nil { + if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName, isPacket); err != nil { return err } c.mu.Lock() @@ -191,6 +210,30 @@ func (c *Coalescer) UpdateObserverStatus(ctx context.Context, p ingest.UpdateObs return id, err } +// DeleteOldObservers persists cached activity before pruning and forgets IDs so +// returning observers write through. Include identities even if an earlier +// presence flush failed or is still in flight; GREATEST prevents older flushes +// from overwriting this freshness. A failed preparation must not delete rows. +func (c *Coalescer) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { + c.mu.Lock() + observers := c.dirtyObservers + for _, ident := range c.identities { + bump := observers[ident.id] + if ident.seen.After(bump.seen) { + bump.seen = ident.seen + } + observers[ident.id] = bump + } + c.dirtyObservers = make(map[uuid.UUID]observerBump) + clear(c.identities) + clear(c.knownBrokers) + c.mu.Unlock() + if err := c.flushObservers(ctx, observers); err != nil { + return nil, err + } + return c.Store.DeleteOldObservers(ctx, cutoff) +} + // Run flushes on the configured interval until ctx is cancelled, then does a // final flush so a clean shutdown loses nothing. func (c *Coalescer) Run(ctx context.Context) { @@ -218,7 +261,7 @@ func (c *Coalescer) Flush(ctx context.Context) { brokers := c.dirtyBrokers packets := c.dirtyPackets c.dirtyObservers = make(map[uuid.UUID]observerBump) - c.dirtyBrokers = make(map[brokerKey]time.Time) + c.dirtyBrokers = make(map[brokerKey]brokerBump) c.dirtyPackets = make(map[string]time.Time) cutoff := c.now().Add(-c.packetTTL) for key, seen := range c.packetsSeen { @@ -228,31 +271,23 @@ func (c *Coalescer) Flush(ctx context.Context) { } c.mu.Unlock() - if len(observers) > 0 { - ids := make([]uuid.UUID, 0, len(observers)) - seen := make([]time.Time, 0, len(observers)) - counts := make([]int32, 0, len(observers)) - for id, bump := range observers { - ids = append(ids, id) - seen = append(seen, bump.seen) - counts = append(counts, bump.count) - } - if err := c.Store.TouchObservers(ctx, ids, seen, counts); err != nil { - log.Printf("presence: flush observers failed (%d rows dropped): %v", len(ids), err) - } + if err := c.flushObservers(ctx, observers); err != nil { + slog.Error(fmt.Sprintf("presence: flush observers failed (%d rows dropped)", len(observers)), "component", "presence", "error", err) } if len(brokers) > 0 { ids := make([]uuid.UUID, 0, len(brokers)) names := make([]string, 0, len(brokers)) seen := make([]time.Time, 0, len(brokers)) + packetTimes := make([]time.Time, 0, len(brokers)) for key, ts := range brokers { ids = append(ids, key.id) names = append(names, key.broker) - seen = append(seen, ts) + seen = append(seen, ts.seen) + packetTimes = append(packetTimes, ts.packet) } - if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen); err != nil { - log.Printf("presence: flush observer brokers failed (%d rows dropped): %v", len(ids), err) + if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen, packetTimes); err != nil { + slog.Error(fmt.Sprintf("presence: flush observer brokers failed (%d rows dropped)", len(ids)), "component", "presence", "error", err) } } @@ -264,7 +299,26 @@ func (c *Coalescer) Flush(ctx context.Context) { heard = append(heard, ts) } if err := c.Store.TouchPackets(ctx, hashes, heard); err != nil { - log.Printf("presence: flush packets failed (%d rows dropped): %v", len(hashes), err) + slog.Error(fmt.Sprintf("presence: flush packets failed (%d rows dropped)", len(hashes)), "component", "presence", "error", err) } } } + +func (c *Coalescer) flushObservers(ctx context.Context, observers map[uuid.UUID]observerBump) error { + if len(observers) == 0 { + return nil + } + ids := make([]uuid.UUID, 0, len(observers)) + seen := make([]time.Time, 0, len(observers)) + counts := make([]int32, 0, len(observers)) + iatas := make([]string, 0, len(observers)) + iataAts := make([]time.Time, 0, len(observers)) + for id, bump := range observers { + ids = append(ids, id) + seen = append(seen, bump.seen) + counts = append(counts, bump.count) + iatas = append(iatas, bump.iata) + iataAts = append(iataAts, bump.iataAt) + } + return c.Store.TouchObservers(ctx, ids, seen, counts, iatas, iataAts) +} diff --git a/internal/presence/coalescer_test.go b/internal/presence/coalescer_test.go new file mode 100644 index 00000000..6dd22672 --- /dev/null +++ b/internal/presence/coalescer_test.go @@ -0,0 +1,141 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package presence + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/google/uuid" +) + +type retentionStore struct { + Store + id uuid.UUID + touch func(context.Context, []uuid.UUID, []time.Time, []int32, []string, []time.Time) error + prune func(context.Context, time.Time) ([]uuid.UUID, error) +} + +func (s *retentionStore) UpsertObserver(context.Context, []byte, string) (uuid.UUID, string, error) { + return s.id, "fixture", nil +} +func (s *retentionStore) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, iatas []string, iataAts []time.Time) error { + return s.touch(ctx, ids, seen, counts, iatas, iataAts) +} +func (s *retentionStore) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { + return s.prune(ctx, cutoff) +} + +func TestObserverRetentionAbortOnFlushFailure(t *testing.T) { + for _, cause := range []error{errors.New("flush unavailable"), context.Canceled} { + t.Run(cause.Error(), func(t *testing.T) { + store := &retentionStore{id: uuid.New()} + store.touch = func(context.Context, []uuid.UUID, []time.Time, []int32, []string, []time.Time) error { return cause } + store.prune = func(context.Context, time.Time) ([]uuid.UUID, error) { + t.Error("deletion attempted after failed preparation") + return nil, nil + } + c := New(store, time.Second, time.Second) + if _, _, err := c.UpsertObserver(context.Background(), []byte("fixture"), "YVR"); err != nil { + t.Fatal(err) + } + if _, err := c.DeleteOldObservers(context.Background(), time.Now()); !errors.Is(err, cause) { + t.Fatalf("got %v, want %v", err, cause) + } + }) + } +} + +func TestObserverRetentionPreservesActivityAfterFlush(t *testing.T) { + for _, inFlight := range []bool{false, true} { + name := "failed flush" + if inFlight { + name = "in-flight flush" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + store := &retentionStore{id: uuid.New()} + c := New(store, time.Second, time.Second) + c.now = func() time.Time { return now } + for range 2 { + if _, _, err := c.UpsertObserver(ctx, []byte("fixture"), "YVR"); err != nil { + t.Fatal(err) + } + } + started, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + store.touch = func(_ context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, _ []string, _ []time.Time) error { + if calls.Add(1) == 1 { + close(started) + <-release + return errors.New("earlier flush failed") + } + if len(ids) != 1 || ids[0] != store.id || !seen[0].Equal(now) || counts[0] != 0 { + t.Errorf("cached activity was lost or counted twice: %v, %v, %v", ids, seen, counts) + } + return nil + } + store.prune = func(context.Context, time.Time) ([]uuid.UUID, error) { + if calls.Load() != 2 { + t.Error("deletion preceded the freshness write") + } + return nil, nil + } + go func() { c.Flush(ctx); close(done) }() + <-started + if !inFlight { + close(release) + <-done + } + _, err := c.DeleteOldObservers(ctx, now.Add(-time.Hour)) + if inFlight { + close(release) + <-done + } + if err != nil { + t.Fatal(err) + } + }) + } +} + +func (s *retentionStore) UpdateObserverStatus(context.Context, ingest.UpdateObserverStatusParams) (uuid.UUID, error) { + return s.id, nil +} + +func TestObserverLastIATAFlush(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + store := &retentionStore{id: uuid.New()} + var gotIATAs []string + var gotAts []time.Time + store.touch = func(_ context.Context, _ []uuid.UUID, _ []time.Time, _ []int32, iatas []string, ats []time.Time) error { + gotIATAs, gotAts = iatas, ats + return nil + } + c := New(store, time.Second, time.Second) + c.now = func() time.Time { return now } + pk := []byte("fixture") + upsert := func(iata string) { + t.Helper() + if _, _, err := c.UpsertObserver(ctx, pk, iata); err != nil { + t.Fatal(err) + } + now = now.Add(time.Second) + } + + upsert("YVR") // write-through + upsert("YYZ") + heard := now.Add(-time.Second) + upsert("") // status/neighbors path must not clear it or move its timestamp + c.Flush(ctx) + if len(gotIATAs) != 1 || gotIATAs[0] != "YYZ" || !gotAts[0].Equal(heard) { + t.Fatalf("flushed %v at %v, want YYZ at %s", gotIATAs, gotAts, heard) + } +} diff --git a/internal/profiling/cpu_other.go b/internal/profiling/cpu_other.go new file mode 100644 index 00000000..9c94f146 --- /dev/null +++ b/internal/profiling/cpu_other.go @@ -0,0 +1,8 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build !linux && !darwin + +package profiling + +func cpuUsage() *processCPU { return nil } diff --git a/internal/profiling/cpu_unix.go b/internal/profiling/cpu_unix.go new file mode 100644 index 00000000..77da9f73 --- /dev/null +++ b/internal/profiling/cpu_unix.go @@ -0,0 +1,19 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build linux || darwin + +package profiling + +import "syscall" + +func cpuUsage() *processCPU { + var usage syscall.Rusage + if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) != nil { + return nil + } + return &processCPU{ + UserSeconds: float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6, + SystemSeconds: float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6, + } +} diff --git a/internal/profiling/profiling.go b/internal/profiling/profiling.go new file mode 100644 index 00000000..0d61a933 --- /dev/null +++ b/internal/profiling/profiling.go @@ -0,0 +1,338 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package profiling records bounded, opt-in CPU samples to private files. +package profiling + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "time" +) + +const ( + captureDuration = 30 * time.Second + captureInterval = 30 * time.Minute + captureCooldown = 5 * time.Minute + maxProfileBytes = 8 << 20 + maxMetadataBytes = 16 << 10 + maxDirectoryBytes = 256 << 20 + maxDirectoryFiles = 512 +) + +type Recorder struct { + root *os.Root + snapshot func() any + triggers map[string]bool + requests chan string + cancel context.CancelFunc + done chan struct{} +} + +type snapshot struct { + ProcessCPU *processCPU `json:",omitempty"` + Goroutines int + Pool any `json:",omitempty"` +} + +type metadata struct { + Reason string + StartedAt time.Time + FinishedAt time.Time + GoVersion string + Before snapshot + After snapshot +} + +// Start leaves expired settings inactive so restarts cannot extend a capture window. +// Tasks named in triggers request an extra capture when they start. +func Start(parent context.Context, dir, until string, triggers []string, poolSnapshot func() any) (*Recorder, error) { + if dir == "" && until == "" { + return nil, nil + } + if dir == "" || until == "" { + return nil, errors.New("profiling requires both directory and deadline") + } + deadline, err := time.Parse(time.RFC3339Nano, until) + if err != nil { + return nil, fmt.Errorf("invalid profiling deadline: %w", err) + } + if !deadline.After(time.Now()) { + slog.Info("CPU profiling deadline has passed; profiling stays off", "component", "profiling", "until", deadline.UTC()) + return nil, nil + } + if time.Until(deadline) > 72*time.Hour { + return nil, errors.New("profiling deadline must be within 72 hours") + } + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + return nil, errors.New("private CPU profiling is supported on Linux and macOS") + } + if !filepath.IsAbs(dir) { + return nil, errors.New("profiling directory must be absolute") + } + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, err + } + info, err := os.Lstat(dir) + if err != nil { + return nil, err + } + if !info.IsDir() || info.Mode().Perm()&0077 != 0 { + return nil, errors.New("profiling directory must be private (0700) and not a symlink") + } + root, err := os.OpenRoot(dir) + if err != nil { + return nil, err + } + ctx, cancel := context.WithDeadline(parent, deadline) + r := &Recorder{root: root, snapshot: poolSnapshot, triggers: make(map[string]bool, len(triggers)), requests: make(chan string, 1), cancel: cancel, done: make(chan struct{})} + for _, name := range triggers { + r.triggers[name] = true + } + slog.Info("CPU profiling enabled", "component", "profiling", "until", deadline.UTC()) + go r.run(ctx) + return r, nil +} + +func (r *Recorder) Stop() { + if r == nil { + return + } + r.cancel() + <-r.done +} + +// request never holds up maintenance or queues a backlog of captures. +func (r *Recorder) request(reason string) { + if r == nil { + return + } + select { + case <-r.done: + return + default: + } + select { + case r.requests <- reason: + default: + } +} + +func (r *Recorder) WrapTask(name string, run func(context.Context) error) func(context.Context) error { + if r == nil { + return run + } + return func(ctx context.Context) (err error) { + select { + case <-r.done: + return run(ctx) + default: + } + if r.triggers[name] { + r.request(name) + } + pprof.Do(ctx, pprof.Labels("task", name), func(ctx context.Context) { err = run(ctx) }) + return err + } +} + +func (r *Recorder) run(ctx context.Context) { + defer close(r.done) + defer r.root.Close() + defer r.cancel() + defer slog.Info("CPU profiling stopped", "component", "profiling") + if err := schedule(ctx, r.requests, r.capture); err != nil { + slog.Warn("CPU profiling stopped after capture failure", "component", "profiling", "error", err) + } +} + +func schedule(ctx context.Context, requests <-chan string, capture func(context.Context, string) error) error { + // Offset periodic ticks from the task tickers started alongside the recorder, so a + // task's trigger isn't swallowed by a periodic capture that began seconds earlier. + first := time.NewTimer(captureInterval + captureCooldown) + defer first.Stop() + var ticker *time.Ticker + tick := first.C + reason := "periodic" + var last time.Time + var deferred <-chan time.Time + for { + if ctx.Err() != nil { + return nil + } + if wait := captureCooldown - time.Since(last); wait <= 0 { + last = time.Now() + deferred = nil // this sample covers any periodic one waiting on the cooldown + if err := capture(ctx, reason); err != nil { + return err + } + } else if reason == "periodic" && deferred == nil { + // Run a periodic sample after the cooldown rather than skipping a whole interval. + deferred = time.After(wait) + } + select { + case <-ctx.Done(): + if ticker != nil { + ticker.Stop() + } + return nil + case <-tick: + if ticker == nil { + ticker = time.NewTicker(captureInterval) + tick = ticker.C + } + reason = "periodic" + case <-deferred: + deferred = nil + reason = "periodic" + case reason = <-requests: + } + } +} + +func (r *Recorder) takeSnapshot() snapshot { + s := snapshot{Goroutines: runtime.NumGoroutine()} + s.ProcessCPU = cpuUsage() + if r.snapshot != nil { + s.Pool = r.snapshot() + } + return s +} + +func (r *Recorder) checkBudget() error { + d, err := r.root.Open(".") + if err != nil { + return err + } + entries, err := d.ReadDir(maxDirectoryFiles + 1) + d.Close() + if err != nil && err != io.EOF { + return err + } + if len(entries) > maxDirectoryFiles-2 { + return errors.New("profiling file budget exhausted") + } + var total int64 + for _, entry := range entries { + info, err := entry.Info() + if errors.Is(err, fs.ErrNotExist) { + continue // removed between ReadDir and Info + } + if err != nil { + return err + } + if !info.Mode().IsRegular() { + continue // subdirectories and specials don't hold profiles + } + total += info.Size() + } + if total > maxDirectoryBytes-maxProfileBytes-maxMetadataBytes { + return errors.New("profiling storage budget exhausted") + } + return nil +} + +func (r *Recorder) capture(ctx context.Context, reason string) error { + if err := r.checkBudget(); err != nil { + return err + } + started := time.Now().UTC() + name := started.Format("20060102T150405.000000000Z") + ".pprof" + partial := name + ".partial" + f, err := r.root.OpenFile(partial, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer r.root.Remove(partial) + w := &limitedWriter{writer: f, remaining: maxProfileBytes} + m := metadata{Reason: reason, StartedAt: started, GoVersion: runtime.Version(), Before: r.takeSnapshot()} + if err := pprof.StartCPUProfile(w); err != nil { + f.Close() + return err + } + slog.Info("CPU profiling capture started", "component", "profiling", "file", name, "reason", reason) + timer := time.NewTimer(captureDuration) + select { + case <-ctx.Done(): + case <-timer.C: + } + timer.Stop() + pprof.StopCPUProfile() + m.FinishedAt = time.Now().UTC() + m.After = r.takeSnapshot() + closeErr := f.Close() + if w.err != nil { + return w.err + } + if closeErr != nil { + return closeErr + } + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + if len(data) > maxMetadataBytes { + return errors.New("profiling metadata budget exhausted") + } + metaPartial := name + ".json.partial" + meta, err := r.root.OpenFile(metaPartial, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer r.root.Remove(metaPartial) + _, err = meta.Write(data) + closeErr = meta.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + // Sidecar first: a .pprof without .json would look complete but be unexplained. + if err := r.root.Rename(metaPartial, name+".json"); err != nil { + return err + } + if err := r.root.Rename(partial, name); err != nil { + r.root.Remove(name + ".json") + return err + } + slog.Info("CPU profile saved", "component", "profiling", "file", name, "reason", reason, "duration", m.FinishedAt.Sub(started)) + return nil +} + +type limitedWriter struct { + writer io.Writer + remaining int + err error +} + +func (w *limitedWriter) Write(p []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + if len(p) > w.remaining { + w.err = errors.New("CPU profile exceeded size limit") + return 0, w.err + } + n, err := w.writer.Write(p) + w.remaining -= n + if err == nil && n != len(p) { + err = io.ErrShortWrite + } + w.err = err + return n, err +} + +type processCPU struct { + UserSeconds float64 + SystemSeconds float64 +} diff --git a/internal/profiling/profiling_test.go b/internal/profiling/profiling_test.go new file mode 100644 index 00000000..51b490de --- /dev/null +++ b/internal/profiling/profiling_test.go @@ -0,0 +1,412 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package profiling + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "reflect" + "runtime" + "runtime/pprof" + "testing" + "testing/synctest" + "time" +) + +func TestDisabledAndExpired(t *testing.T) { + for _, tc := range []struct{ dir, until string }{ + {}, {filepath.Join(t.TempDir(), "unused"), time.Now().Add(-time.Hour).Format(time.RFC3339)}, + } { + r, err := Start(context.Background(), tc.dir, tc.until, nil, nil) + if err != nil || r != nil { + t.Fatalf("got %v, %v", r, err) + } + if tc.dir != "" { + if _, err := os.Stat(tc.dir); !os.IsNotExist(err) { + t.Fatalf("expired capture created directory: %v", err) + } + } + } +} + +func TestRejectUnsafeConfiguration(t *testing.T) { + for _, tc := range []struct{ name, dir, until string }{ + {"missing deadline", t.TempDir(), ""}, + {"missing directory", "", time.Now().Add(time.Hour).Format(time.RFC3339)}, + {"invalid deadline", t.TempDir(), "tomorrow"}, + {"unbounded window", t.TempDir(), time.Now().Add(73 * time.Hour).Format(time.RFC3339)}, + {"relative directory", "profiles", time.Now().Add(time.Hour).Format(time.RFC3339)}, + } { + t.Run(tc.name, func(t *testing.T) { + r, err := Start(context.Background(), tc.dir, tc.until, nil, nil) + if r != nil { + r.Stop() + } + if err == nil { + t.Fatal("accepted unsafe configuration") + } + }) + } + dir := t.TempDir() + if err := os.Chmod(dir, 0755); err != nil { + t.Fatal(err) + } + r, err := Start(context.Background(), dir, time.Now().Add(time.Hour).Format(time.RFC3339), nil, nil) + if r != nil { + r.Stop() + } + if err == nil { + t.Fatal("accepted public directory") + } +} + +func TestDeadlineProducesPrivateProfileAndMetadata(t *testing.T) { + requireSupportedPlatform(t) + dir := filepath.Join(t.TempDir(), "profiles") + until := time.Now().Add(500 * time.Millisecond) + r, err := Start(context.Background(), dir, until.Format(time.RFC3339Nano), nil, func() any { return map[string]int{"acquired": 2} }) + if err != nil { + t.Fatal(err) + } + select { + case <-r.done: + case <-time.After(3 * time.Second): + r.Stop() + t.Fatal("did not stop at deadline") + } + files, err := filepath.Glob(filepath.Join(dir, "*.pprof")) + if err != nil || len(files) != 1 { + t.Fatalf("profiles %v, %v", files, err) + } + b, err := os.ReadFile(files[0]) + if err != nil { + t.Fatal(err) + } + z, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(z) + z.Close() + if err != nil || len(data) == 0 { + t.Fatalf("invalid profile: %v", err) + } + for _, name := range []string{files[0], files[0] + ".json"} { + info, err := os.Stat(name) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Fatalf("permissions %v", info.Mode()) + } + } + var m struct { + Reason string + StartedAt, FinishedAt time.Time + Before, After struct{ Pool map[string]int } + } + b, err = os.ReadFile(files[0] + ".json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + if m.Reason != "periodic" || !m.FinishedAt.After(m.StartedAt) || m.Before.Pool["acquired"] != 2 || m.After.Pool["acquired"] != 2 { + t.Fatalf("bad metadata %+v", m) + } + r.request("reconfirm") + r.Stop() + files, _ = filepath.Glob(filepath.Join(dir, "*.pprof")) + if len(files) != 1 { + t.Fatal("capture after expiry") + } +} + +func TestStopReleasesProfiler(t *testing.T) { + requireSupportedPlatform(t) + active := make(chan struct{}, 2) + old := slog.Default() + slog.SetDefault(slog.New(captureObserver{Handler: slog.NewTextHandler(io.Discard, nil), active: active})) + defer slog.SetDefault(old) + dir := filepath.Join(t.TempDir(), "profiles") + r, err := Start(context.Background(), dir, time.Now().Add(time.Hour).Format(time.RFC3339), nil, nil) + if err != nil { + t.Fatal(err) + } + select { + case <-active: + case <-time.After(3 * time.Second): + r.Stop() + t.Fatal("capture did not start") + } + r.Stop() + r.Stop() + first, _ := filepath.Glob(filepath.Join(dir, "*.pprof")) + if len(first) != 1 { + t.Fatal("active capture was not saved on shutdown") + } + b, err := os.ReadFile(first[0]) + if err != nil { + t.Fatal(err) + } + z, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(io.Discard, z); err != nil { + t.Fatal(err) + } + z.Close() + dir2 := filepath.Join(t.TempDir(), "profiles") + r2, err := Start(context.Background(), dir2, time.Now().Add(400*time.Millisecond).Format(time.RFC3339Nano), nil, nil) + if err != nil { + t.Fatal(err) + } + <-r2.done + files, _ := filepath.Glob(filepath.Join(dir2, "*.pprof")) + if len(files) != 1 { + t.Fatal("profiler was not released") + } +} + +func TestProfileWriteLimit(t *testing.T) { + var b bytes.Buffer + w := &limitedWriter{writer: &b, remaining: 4} + if _, err := w.Write([]byte("12345")); err == nil { + t.Fatal("accepted oversized profile") + } + if b.Len() > 4 { + t.Fatal("exceeded file budget") + } + if w.err == nil { + t.Fatal("write failure lost") + } +} + +func TestDirectoryBudgetSurvivesRestart(t *testing.T) { + requireSupportedPlatform(t) + dir := filepath.Join(t.TempDir(), "profiles") + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + f, err := os.Create(filepath.Join(dir, "old.pprof")) + if err != nil { + t.Fatal(err) + } + if err := f.Truncate(256 << 20); err != nil { + t.Fatal(err) + } + f.Close() + r, err := Start(context.Background(), dir, time.Now().Add(time.Hour).Format(time.RFC3339), nil, nil) + if err != nil { + t.Fatal(err) + } + select { + case <-r.done: + case <-time.After(2 * time.Second): + r.Stop() + t.Fatal("budget did not stop recording") + } + entries, _ := os.ReadDir(dir) + if len(entries) != 1 { + t.Fatal("wrote past directory budget") + } +} + +func TestScheduleBoundsExtraCaptures(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + requests := make(chan string, 1) + type event struct { + at time.Duration + reason string + } + var events []event + start := time.Now() + done := make(chan error, 1) + go func() { + done <- schedule(ctx, requests, func(ctx context.Context, reason string) error { + events = append(events, event{time.Since(start), reason}) + time.Sleep(30 * time.Second) + return nil + }) + }() + synctest.Wait() + time.Sleep(time.Minute) + requests <- "reconfirm" + synctest.Wait() + if len(events) != 1 { + t.Fatalf("ignored cooldown: %+v", events) + } + time.Sleep(4 * time.Minute) + requests <- "reconfirm" + synctest.Wait() + time.Sleep(61 * time.Minute) + cancel() + synctest.Wait() + if err := <-done; err != nil { + t.Fatal(err) + } + want := []event{{0, "periodic"}, {5 * time.Minute, "reconfirm"}, {35 * time.Minute, "periodic"}, {65 * time.Minute, "periodic"}} + if !reflect.DeepEqual(events, want) { + t.Fatalf("captures %+v, want %+v", events, want) + } + }) +} + +func TestScheduleDefersPeriodicCaptureInCooldown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + requests := make(chan string, 1) + var at []time.Duration + var reasons []string + start := time.Now() + done := make(chan error, 1) + go func() { + done <- schedule(ctx, requests, func(ctx context.Context, reason string) error { + at = append(at, time.Since(start)) + reasons = append(reasons, reason) + time.Sleep(30 * time.Second) + return nil + }) + }() + synctest.Wait() + time.Sleep(33 * time.Minute) + requests <- "reconfirm" + synctest.Wait() + time.Sleep(33 * time.Minute) + cancel() + synctest.Wait() + if err := <-done; err != nil { + t.Fatal(err) + } + wantAt := []time.Duration{0, 33 * time.Minute, 38 * time.Minute, 65 * time.Minute} + wantReasons := []string{"periodic", "reconfirm", "periodic", "periodic"} + if !reflect.DeepEqual(at, wantAt) || !reflect.DeepEqual(reasons, wantReasons) { + t.Fatalf("captures %v %v, want %v %v", at, reasons, wantAt, wantReasons) + } + }) +} + +func TestScheduleOneCaptureCoversDeferredAndTrigger(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + requests := make(chan string, 1) + var at []time.Duration + start := time.Now() + done := make(chan error, 1) + go func() { + done <- schedule(ctx, requests, func(ctx context.Context, reason string) error { + at = append(at, time.Since(start)) + time.Sleep(30 * time.Second) + return nil + }) + }() + synctest.Wait() + time.Sleep(33 * time.Minute) + requests <- "reconfirm" // 33m: captured; the 35m tick is deferred to 38m + synctest.Wait() + time.Sleep(5 * time.Minute) // 38m: the deferred periodic and a fresh trigger are both due + requests <- "reconfirm" + synctest.Wait() + time.Sleep(28 * time.Minute) // past the 65m tick so cancel can't race it + cancel() + synctest.Wait() + if err := <-done; err != nil { + t.Fatal(err) + } + want := []time.Duration{0, 33 * time.Minute, 38 * time.Minute, 65 * time.Minute} + if !reflect.DeepEqual(at, want) { + t.Fatalf("captures %v, want %v", at, want) + } + }) +} + +func TestBudgetIgnoresNonRegularEntries(t *testing.T) { + requireSupportedPlatform(t) + dir := filepath.Join(t.TempDir(), "profiles") + if err := os.MkdirAll(filepath.Join(dir, "lost+found"), 0700); err != nil { + t.Fatal(err) + } + r, err := Start(context.Background(), dir, time.Now().Add(500*time.Millisecond).Format(time.RFC3339Nano), nil, nil) + if err != nil { + t.Fatal(err) + } + select { + case <-r.done: + case <-time.After(3 * time.Second): + r.Stop() + t.Fatal("did not stop at deadline") + } + profiles, _ := filepath.Glob(filepath.Join(dir, "*.pprof")) + if len(profiles) != 1 { + t.Fatalf("subdirectory stopped the recorder: %v", profiles) + } +} + +func TestTriggersAreConfigurable(t *testing.T) { + r := &Recorder{triggers: map[string]bool{"custom": true}, requests: make(chan string, 1), done: make(chan struct{})} + noop := func(context.Context) error { return nil } + _ = r.WrapTask("reconfirm", noop)(context.Background()) + select { + case got := <-r.requests: + t.Fatalf("untriggered task requested %q", got) + default: + } + _ = r.WrapTask("custom", noop)(context.Background()) + if got := <-r.requests; got != "custom" { + t.Fatalf("got %q", got) + } +} + +func TestWrappedTaskPreservesContextAndError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + want := errors.New("task failed") + r := &Recorder{triggers: map[string]bool{"reconfirm": true}, requests: make(chan string, 1), done: make(chan struct{})} + run := r.WrapTask("reconfirm", func(got context.Context) error { + if got.Err() != context.Canceled { + t.Fatal("lost cancellation") + } + if label, ok := pprof.Label(got, "task"); !ok || label != "reconfirm" { + t.Fatal("missing task label") + } + return want + }) + for i := 0; i < 10; i++ { + if err := run(ctx); err != want { + t.Fatalf("lost task error: %v", err) + } + } +} + +func requireSupportedPlatform(t *testing.T) { + t.Helper() + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("private profiling requires Unix permissions") + } +} + +type captureObserver struct { + slog.Handler + active chan<- struct{} +} + +func (h captureObserver) Handle(ctx context.Context, record slog.Record) error { + if record.Message == "CPU profiling capture started" { + h.active <- struct{}{} + } + return h.Handler.Handle(ctx, record) +} diff --git a/internal/scopestore/scopestore.go b/internal/scopestore/scopestore.go index 8abbac3b..55d6cd04 100644 --- a/internal/scopestore/scopestore.go +++ b/internal/scopestore/scopestore.go @@ -5,19 +5,38 @@ // loaded from the database at startup. package scopestore -import "sync" +import ( + "crypto/sha256" + "slices" + "strings" + "sync" +) // Entry holds a single transport scope key and its metadata. type Entry struct { Name string - TransportKey []byte // 16 bytes - KeyFingerprint []byte // 8 bytes + TransportKey []byte // 16 bytes + KeyFingerprint []byte // 8 bytes + IATAs []string // nil for manual keys; imported candidates are regional +} + +// FromName uses the same case-sensitive derivation for manual and imported scopes. +func FromName(name string) Entry { + if !strings.HasPrefix(name, "#") && !strings.HasPrefix(name, "$") { + name = "#" + name + } + h := sha256.Sum256([]byte(name)) + key := h[:16] + fingerprint := sha256.Sum256(key) + return Entry{Name: name, TransportKey: key, KeyFingerprint: fingerprint[:8]} } // ScopeStore holds all known transport scope keys in memory. type ScopeStore struct { mu sync.RWMutex entries []Entry + // IATA -> scope names, for region-filtered listings only; matching uses entries. + manual, catalogue map[string][]string } // New creates an empty ScopeStore. @@ -25,18 +44,43 @@ func New() *ScopeStore { return &ScopeStore{} } -// Load replaces all entries — call on startup after DB seeding. +// Load publishes a new immutable snapshot. The caller must not mutate it after publication. func (s *ScopeStore) Load(entries []Entry) { s.mu.Lock() defer s.mu.Unlock() s.entries = entries } -// Entries returns a copy of all loaded entries. +// Entries returns the current immutable snapshot. Callers must not modify it. func (s *ScopeStore) Entries() []Entry { s.mu.RLock() defer s.mu.RUnlock() - result := make([]Entry, len(s.entries)) - copy(result, s.entries) - return result + return s.entries +} + +// SetManualMembers publishes configured region membership of manual scopes. +func (s *ScopeStore) SetManualMembers(byIATA map[string][]string) { + s.mu.Lock() + defer s.mu.Unlock() + s.manual = byIATA +} + +// SetCatalogueMembers publishes imported catalogue membership, including names a manual scope overrides. +func (s *ScopeStore) SetCatalogueMembers(byIATA map[string][]string) { + s.mu.Lock() + defer s.mu.Unlock() + s.catalogue = byIATA +} + +// NamesForIATAs returns the sorted names of scopes that belong to any of the given IATAs. +func (s *ScopeStore) NamesForIATAs(iatas []string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + names := []string{} + for _, iata := range iatas { + names = append(names, s.manual[iata]...) + names = append(names, s.catalogue[iata]...) + } + slices.Sort(names) + return slices.Compact(names) } diff --git a/internal/scopestore/scopestore_test.go b/internal/scopestore/scopestore_test.go index 8df39001..08eef4c4 100644 --- a/internal/scopestore/scopestore_test.go +++ b/internal/scopestore/scopestore_test.go @@ -4,6 +4,7 @@ package scopestore import ( + "slices" "testing" ) @@ -38,17 +39,37 @@ func TestLoad_ReplacesEntries(t *testing.T) { } } -func TestEntries_ReturnsCopy(t *testing.T) { +func TestEntriesRetainsImmutableSnapshotWithoutAllocation(t *testing.T) { s := New() - s.Load([]Entry{ - {Name: "#bc", TransportKey: []byte{0x01}, KeyFingerprint: []byte{0x02}}, - }) - entries := s.Entries() - entries[0].Name = "mutated" + s.Load([]Entry{{Name: "#bc"}}) + previous := s.Entries() + if allocations := testing.AllocsPerRun(100, func() { + if len(s.Entries()) != 1 { + t.Fatal("missing entries") + } + }); allocations != 0 { + t.Fatalf("snapshot read allocates: %v", allocations) + } + s.Load([]Entry{{Name: "#on"}}) + if previous[0].Name != "#bc" || s.Entries()[0].Name != "#on" { + t.Fatal("replacement mutated the previous reader snapshot") + } +} - // original should be unchanged - original := s.Entries() - if original[0].Name != "#bc" { - t.Errorf("expected #bc, got %s after mutation of copy", original[0].Name) +func TestNamesForIATAs(t *testing.T) { + s := New() + if got := s.NamesForIATAs([]string{"YOW"}); got == nil || len(got) != 0 { + t.Fatalf("empty store = %#v, want empty non-nil slice", got) + } + s.SetManualMembers(map[string][]string{"YOW": {"#ottawa", "#on"}, "YVR": {"#bc"}}) + s.SetCatalogueMembers(map[string][]string{"YOW": {"#on", "#gatineau"}, "YUL": {"#qc"}}) + got := s.NamesForIATAs([]string{"YOW", "YUL", "YYZ"}) + want := []string{"#gatineau", "#on", "#ottawa", "#qc"} + if !slices.Equal(got, want) { + t.Fatalf("NamesForIATAs = %v, want %v", got, want) + } + s.SetCatalogueMembers(nil) + if got := s.NamesForIATAs([]string{"YUL"}); len(got) != 0 { + t.Fatalf("cleared catalogue still lists %v", got) } } diff --git a/internal/ws/connect_limit_test.go b/internal/ws/connect_limit_test.go new file mode 100644 index 00000000..21d878be --- /dev/null +++ b/internal/ws/connect_limit_test.go @@ -0,0 +1,174 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ws + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/coder/websocket" +) + +func connectTestServer(t *testing.T, maxConns, maxConnects int) (*httptest.Server, <-chan struct{}) { + t.Helper() + handler := Handler(hub.New(), nil, maxConns, maxConnects, nil) + done := make(chan struct{}, 64) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler(w, r) + done <- struct{}{} + })) + t.Cleanup(server.Close) + return server, done +} + +func TestFailedHandshakeDoesNotUseConnectionSlot(t *testing.T) { + server, done := connectTestServer(t, 1, 10) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + response, err := server.Client().Get(server.URL) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode < 400 || response.StatusCode == http.StatusTooManyRequests { + t.Fatalf("expected a failed handshake: %d", response.StatusCode) + } + waitConnectionEnd(t, ctx, done) + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + if _, _, err := conn.Read(ctx); err != nil { + t.Fatalf("failed handshake consumed a connection slot: %v", err) + } + conn.CloseNow() + waitConnectionEnd(t, ctx, done) +} + +func TestConnectAttemptBudgetAndRecovery(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + handler := Handler(nil, nil, 1, 2, nil) + attempt := func() *httptest.ResponseRecorder { + request := httptest.NewRequest(http.MethodGet, "/ws", nil) + request.RemoteAddr = "198.51.100.1:1234" + response := httptest.NewRecorder() + handler(response, request) + return response + } + for range 2 { + if response := attempt(); response.Code < 400 || response.Code == http.StatusTooManyRequests { + t.Fatalf("expected failed handshake within budget: %d", response.Code) + } + } + if response := attempt(); response.Code != http.StatusTooManyRequests || response.Header().Get("Retry-After") != "60" { + t.Fatalf("failed handshakes bypassed attempt budget: %d %v", response.Code, response.Header()) + } + time.Sleep(2 * time.Minute) + if response := attempt(); response.Code < 400 || response.Code == http.StatusTooManyRequests { + t.Fatalf("attempt budget did not recover: %d", response.Code) + } + }) +} + +func TestConcurrentConnectsRespectBothLimits(t *testing.T) { + for _, tc := range []struct { + name string + conns, attempts int + wantOpen, wantShed int32 + wantRejected int32 + }{ + {"concurrent cap", 2, 20, 2, 6, 0}, + {"attempt cap", 20, 2, 2, 0, 6}, + } { + t.Run(tc.name, func(t *testing.T) { + server, done := connectTestServer(t, tc.conns, tc.attempts) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var opened, shed, rejected atomic.Int32 + connections := make(chan *websocket.Conn, 8) + defer func() { + close(connections) + for conn := range connections { + conn.CloseNow() + } + for range 8 { + waitConnectionEnd(t, ctx, done) + } + }() + var group sync.WaitGroup + for range 8 { + group.Go(func() { + conn, response, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil) + if err != nil { + if response != nil && response.StatusCode == http.StatusTooManyRequests { + rejected.Add(1) + } else { + t.Errorf("unexpected handshake failure: %v", err) + } + return + } + connections <- conn // Keep accepted connections open until every attempt completes. + if _, _, err := conn.Read(ctx); err == nil { + opened.Add(1) + } else if websocket.CloseStatus(err) == websocket.StatusTryAgainLater { + shed.Add(1) + } else { + t.Errorf("unexpected WebSocket read error: %v", err) + } + }) + } + group.Wait() + if opened.Load() != tc.wantOpen || shed.Load() != tc.wantShed || rejected.Load() != tc.wantRejected { + t.Fatalf("opened=%d shed=%d rejected=%d", opened.Load(), shed.Load(), rejected.Load()) + } + }) + } +} + +func waitConnectionEnd(t *testing.T, ctx context.Context, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-ctx.Done(): + t.Fatal("connection handler did not release its slot") + } +} + +func TestConnectChurnLimited(t *testing.T) { + server, done := connectTestServer(t, 1, 10) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + url := "ws" + strings.TrimPrefix(server.URL, "http") + for range 10 { + conn, _, err := websocket.Dial(ctx, url, nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := conn.Read(ctx); err != nil { + conn.CloseNow() + t.Fatalf("read hello: %v", err) + } + conn.CloseNow() + waitConnectionEnd(t, ctx, done) + } + conn, response, err := websocket.Dial(ctx, url, nil) + if conn != nil { + conn.CloseNow() + } + if err == nil || response == nil || response.StatusCode != http.StatusTooManyRequests { + t.Fatal("eleventh connection attempt was not rejected with HTTP 429") + } + if response.Header.Get("Retry-After") != "60" { + t.Fatal("missing upgrade retry guidance") + } +} diff --git a/internal/ws/handler.go b/internal/ws/handler.go index c21826e4..9cd27bfe 100644 --- a/internal/ws/handler.go +++ b/internal/ws/handler.go @@ -9,20 +9,22 @@ // // Client → Server: // subscribe { v, type, id, scope } → server replies subscribed { v, type, id, subscriptionId } -// unsubscribe { v, type, id, subscriptionId } -// configure { v, type, id, resolvePath } → server replies configured { v, type, id, resolvePath } +// unsubscribe { v, type, id, subscriptionId } → server replies unsubscribed { v, type, id, subscriptionId } +// configure { v, type, id, resolvePath, includeObserverKey, includeRepeats } +// → server replies configured { v, type, id, resolvePath, includeObserverKey, includeRepeats } // ping { v, type, id } → server replies pong { v, type, id } // -// configure's resolvePath (bool) is a connection-wide setting, not -// per-subscription: enables/disables per-hop resolvedPath data on -// packetObservation events. Freely toggleable at any point during the -// connection; each configure call sets it to exactly the value sent -// (not additive across calls, unlike subscribe scopes). Default false. +// configure's flags are connection-wide settings, not per-subscription. +// resolvePath adds per-hop resolvedPath data to packetObservation events, +// includeObserverKey adds observation.observerPublicKey, and includeRepeats +// also streams later hearings of an already-stored observation over a new +// path, as packetObservation events with packet.isRepeat true and +// observationCount 0. Each configure sets all three to exactly the values +// sent, so an omitted flag turns off. All default false. // // Server → Client events (unsolicited): // packetObservation, observerStatus, nodeUpdate, channelMessage -// lagged { v, type, droppedCount, since, lastObservationId } -// error { v, type, code, message } +// lagged { v, type, droppedCount, since } // // Idle connections (no ping) closed after 90s. // Client should ping every 30s. @@ -31,13 +33,15 @@ package ws import ( "context" "encoding/json" - "log" + "fmt" + "log/slog" "net" "net/http" "strconv" "time" "github.com/coder/websocket" + "github.com/go-chi/httprate" "github.com/google/uuid" "github.com/MeshCore-Beacon/beacon-server/internal/api" @@ -51,24 +55,38 @@ const ( // Handler returns an http.HandlerFunc that requires the hub to be injected. // Wire it via router.New(h) so the hub is available at startup. -func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc { +func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP, maxConnectsPerMinute int, allowedOrigins []string) http.HandlerFunc { + var acceptOpts *websocket.AcceptOptions + if len(allowedOrigins) > 0 { + acceptOpts = &websocket.AcceptOptions{OriginPatterns: allowedOrigins} + } limiter := newIPLimiter(maxConnsPerIP) + attempts := httprate.NewRateLimiter(maxConnectsPerMinute, time.Minute, + httprate.WithResponseHeaders(httprate.ResponseHeaders{RetryAfter: "Retry-After"})) return func(w http.ResponseWriter, r *http.Request) { ip := r.RemoteAddr if host, _, err := net.SplitHostPort(ip); err == nil { ip = host } - if !limiter.acquire(ip) { - log.Printf("ws: connection limit reached for IP %s", ip) - http.Error(w, "too many connections from this IP", http.StatusTooManyRequests) + // Count attempts before Accept, including failed handshakes. RemoteAddr + // was already resolved by TrustedProxyIP; never read forwarding headers here. + if attempts.RespondOnLimit(w, r, httprate.CanonicalizeIP(ip)) { return } - defer limiter.release(ip) - conn, err := websocket.Accept(w, r, nil) + conn, err := websocket.Accept(w, r, acceptOpts) if err != nil { - log.Printf("ws: failed to accept connection: %v", err) + slog.Warn("ws: failed to accept connection", "component", "ws", "error", err) + return + } + defer conn.CloseNow() + if !limiter.acquire(ip) { + // This attempt passed the rate budget. Give the accepted socket a + // browser-visible backoff signal before allocating a hub client. + slog.Warn("ws: connection limit reached", "component", "ws", "client_ip", ip) + _ = conn.Close(websocket.StatusTryAgainLater, "connection limit reached") return } + defer limiter.release(ip) connID := uuid.NewString() client := h.NewClient() @@ -85,10 +103,10 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc "connectionId": connID, } helloBytes, _ := json.Marshal(hello) - log.Printf("ws[%s]: connected, hello: %s", connID, helloBytes) + slog.Debug("connected", "component", "ws", "connection_id", connID) err = conn.Write(ctx, websocket.MessageText, helloBytes) if err != nil { - log.Printf("ws[%s]: failed to send hello: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to send hello", connID), "component", "ws", "error", err) return } @@ -109,7 +127,7 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc msgBytes, _ := json.Marshal(msg) err = conn.Write(ctx, websocket.MessageText, msgBytes) if err != nil { - log.Printf("ws[%s]: failed to write hub event: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to write hub event", connID), "component", "ws", "error", err) cancel() return } @@ -126,7 +144,7 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc } lagBytes, _ := json.Marshal(lagged) if err := conn.Write(ctx, websocket.MessageText, lagBytes); err != nil { - log.Printf("ws[%s]: failed to write lagged notice: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to write lagged notice", connID), "component", "ws", "error", err) cancel() return } @@ -144,7 +162,7 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc _, msgBytes, err := conn.Read(readCtx) readCancel() if err != nil { - log.Printf("ws[%s]: read error: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: read error", connID), "component", "ws", "error", err) return } handleClientMessage(ctx, client, reader, h, conn, connID, msgBytes) @@ -160,10 +178,10 @@ type clientMessage struct { SubscriptionID string `json:"subscriptionId,omitempty"` Scope *subscribeScope `json:"scope,omitempty"` - // ResolvePath is only read for "configure" messages: enables/disables - // the resolvedPath variant of packetObservation events for the whole - // connection. See package doc. - ResolvePath bool `json:"resolvePath,omitempty"` + // Only read for "configure" messages. See package doc. + ResolvePath bool `json:"resolvePath,omitempty"` + IncludeObserverKey bool `json:"includeObserverKey,omitempty"` + IncludeRepeats bool `json:"includeRepeats,omitempty"` } // subscribeScope mirrors the scope object in the subscribe message. @@ -182,7 +200,7 @@ type subscribeScope struct { func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Reader, h *hub.Hub, conn *websocket.Conn, connID string, raw []byte) { var msg clientMessage if err := json.Unmarshal(raw, &msg); err != nil { - log.Printf("ws[%s]: bad message: %v", connID, err) + slog.Warn("bad message", "component", "ws", "connection_id", connID, "error", err) return } @@ -195,12 +213,12 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea for _, ridStr := range msg.Scope.RegionIDs { rid, err := strconv.ParseInt(ridStr, 10, 32) if err != nil { - log.Printf("ws[%s]: invalid regionId %q, skipping", connID, ridStr) + slog.Warn(fmt.Sprintf("ws[%s]: invalid regionId %q, skipping", connID, ridStr), "component", "ws") continue } region, err := reader.GetRegion(ctx, int32(rid)) if err != nil { - log.Printf("ws[%s]: region %d not found, skipping: %v", connID, rid, err) + slog.Warn(fmt.Sprintf("ws[%s]: region %d not found, skipping", connID, rid), "component", "ws", "error", err) continue } iatas = append(iatas, region.IATAs...) @@ -208,7 +226,7 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea for _, slug := range msg.Scope.RegionSlugs { region, err := reader.GetRegionBySlug(ctx, slug) if err != nil { - log.Printf("ws[%s]: region slug %q not found, skipping: %v", connID, slug, err) + slog.Warn(fmt.Sprintf("ws[%s]: region slug %q not found, skipping", connID, slug), "component", "ws", "error", err) continue } iatas = append(iatas, region.IATAs...) @@ -216,7 +234,9 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea scope := hub.Scope{ IATAs: iatas, PayloadTypes: msg.Scope.PayloadTypes, + RouteTypes: msg.Scope.RouteTypes, ChannelHashes: msg.Scope.ChannelHashes, + ObserverIDs: msg.Scope.ObserverIDs, Events: msg.Scope.Events, } subID := uuid.NewString() @@ -224,10 +244,12 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea reply, _ := json.Marshal(map[string]any{ "v": 1, "type": "subscribed", "id": msg.ID, "subscriptionId": subID, }) - log.Printf("ws[%s]: subscribed %s → %s", connID, msg.ID, subID) + if slog.Default().Enabled(context.Background(), slog.LevelDebug) { + slog.Debug(fmt.Sprintf("ws[%s]: subscribed %s → %s", connID, msg.ID, subID), "component", "ws") + } err := conn.Write(ctx, websocket.MessageText, reply) if err != nil { - log.Printf("ws[%s]: failed to send subscribed reply: %v", connID, err) + slog.Warn("failed to send subscribed reply", "component", "ws", "connection_id", connID, "error", err) } case "unsubscribe": @@ -238,29 +260,34 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea reply, _ := json.Marshal(map[string]any{ "v": 1, "type": "unsubscribed", "id": msg.ID, "subscriptionId": msg.SubscriptionID, }) - log.Printf("ws[%s]: unsubscribed %s", connID, msg.SubscriptionID) + if slog.Default().Enabled(context.Background(), slog.LevelDebug) { + slog.Debug(fmt.Sprintf("ws[%s]: unsubscribed %s", connID, msg.SubscriptionID), "component", "ws") + } if err := conn.Write(ctx, websocket.MessageText, reply); err != nil { - log.Printf("ws[%s]: failed to send unsubscribed reply: %v", connID, err) + slog.Warn("failed to send unsubscribed reply", "component", "ws", "connection_id", connID, "error", err) } case "configure": - h.SetResolvePath(client, msg.ResolvePath) + h.Configure(client, hub.ClientOptions{ResolvePath: msg.ResolvePath, IncludeObserverKey: msg.IncludeObserverKey, IncludeRepeats: msg.IncludeRepeats}) reply, _ := json.Marshal(map[string]any{ - "v": 1, "type": "configured", "id": msg.ID, "resolvePath": msg.ResolvePath, + "v": 1, "type": "configured", "id": msg.ID, + "resolvePath": msg.ResolvePath, "includeObserverKey": msg.IncludeObserverKey, "includeRepeats": msg.IncludeRepeats, }) - log.Printf("ws[%s]: configured resolvePath=%t", connID, msg.ResolvePath) + if slog.Default().Enabled(context.Background(), slog.LevelDebug) { + slog.Debug(fmt.Sprintf("ws[%s]: configured resolvePath=%t includeObserverKey=%t includeRepeats=%t", connID, msg.ResolvePath, msg.IncludeObserverKey, msg.IncludeRepeats), "component", "ws") + } if err := conn.Write(ctx, websocket.MessageText, reply); err != nil { - log.Printf("ws[%s]: failed to send configured reply: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to send configured reply", connID), "component", "ws", "error", err) } case "ping": reply, _ := json.Marshal(map[string]any{"v": 1, "type": "pong", "id": msg.ID}) err := conn.Write(ctx, websocket.MessageText, reply) if err != nil { - log.Printf("ws[%s]: failed to send pong: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to send pong", connID), "component", "ws", "error", err) } default: - log.Printf("ws[%s]: unknown message type %q", connID, msg.Type) + slog.Warn("unknown message type", "component", "ws", "connection_id", connID) } } diff --git a/internal/ws/handler_test.go b/internal/ws/handler_test.go new file mode 100644 index 00000000..3e9c5900 --- /dev/null +++ b/internal/ws/handler_test.go @@ -0,0 +1,199 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ws + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/coder/websocket" +) + +func TestAllowedOrigins(t *testing.T) { + wildcard := []string{"https://*.example.com"} + apexAndWildcard := []string{"https://example.com", "https://*.example.com"} + for _, tc := range []struct { + name string + allowed []string + origin string + wantOK bool + }{ + {"no origin header", nil, "", true}, + {"foreign origin by default", nil, "https://example.com", false}, + {"listed origin", []string{"https://example.com"}, "https://example.com", true}, + {"listed origin, other case", []string{"https://example.com"}, "https://Example.com", true}, + {"scheme mismatch", []string{"https://example.com"}, "http://example.com", false}, + {"port mismatch", []string{"https://example.com"}, "https://example.com:8443", false}, + {"unlisted origin", []string{"https://example.com"}, "https://other.example", false}, + {"wildcard subdomain", wildcard, "https://sub.example.com", true}, + {"wildcard deeper subdomain", wildcard, "https://a.b.example.com", true}, + {"wildcard other case", wildcard, "https://SUB.Example.com", true}, + {"wildcard apex", wildcard, "https://example.com", false}, + {"wildcard lookalike", wildcard, "https://evilexample.com", false}, + {"wildcard suffix host", wildcard, "https://example.com.evil.net", false}, + {"wildcard scheme mismatch", wildcard, "http://sub.example.com", false}, + {"wildcard port mismatch", wildcard, "https://sub.example.com:8443", false}, + {"apex and wildcard, apex", apexAndWildcard, "https://example.com", true}, + {"apex and wildcard, subdomain", apexAndWildcard, "https://sub.example.com", true}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(Handler(hub.New(), nil, 5, 100, tc.allowed)) + t.Cleanup(server.Close) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + header := http.Header{} + if tc.origin != "" { + header.Set("Origin", tc.origin) + } + conn, resp, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), &websocket.DialOptions{HTTPHeader: header}) + if tc.wantOK { + if err != nil { + t.Fatalf("dial: %v", err) + } + conn.CloseNow() + return + } + if err == nil { + conn.CloseNow() + t.Fatal("expected handshake to be refused") + } + if resp == nil || resp.StatusCode != http.StatusForbidden { + t.Fatalf("expected 403, got %v (%v)", resp, err) + } + }) + } +} + +func TestConfigureIncludeObserverKey(t *testing.T) { + h := hub.New() + go h.Run() + server := httptest.NewServer(Handler(h, nil, 5, 100, nil)) + t.Cleanup(server.Close) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + read := func(conn *websocket.Conn) map[string]json.RawMessage { + t.Helper() + _, data, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + var msg map[string]json.RawMessage + if err := json.Unmarshal(data, &msg); err != nil { + t.Fatal(err) + } + return msg + } + send := func(conn *websocket.Conn, msg string) { + t.Helper() + if err := conn.Write(ctx, websocket.MessageText, []byte(msg)); err != nil { + t.Fatal(err) + } + } + connect := func(configure string) *websocket.Conn { + t.Helper() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.CloseNow() }) + read(conn) // hello + if configure != "" { + send(conn, configure) + reply := read(conn) + if string(reply["type"]) != `"configured"` || string(reply["includeObserverKey"]) != "true" || string(reply["resolvePath"]) != "false" { + t.Fatalf("unexpected configured reply: %v", reply) + } + } + send(conn, `{"v":1,"type":"subscribe","id":"s","scope":{}}`) + read(conn) // subscribed + return conn + } + + plain := connect("") + keyed := connect(`{"v":1,"type":"configure","id":"c","includeObserverKey":true}`) + time.Sleep(20 * time.Millisecond) // let the hub apply both subscriptions + h.Broadcast(hub.Event{ + Type: hub.EventPacketObservation, + Payload: json.RawMessage(`{"variant":"base"}`), + PayloadWithKey: json.RawMessage(`{"variant":"key"}`), + }) + if got := string(read(plain)["data"]); got != `{"variant":"base"}` { + t.Errorf("plain client got %s", got) + } + if got := string(read(keyed)["data"]); got != `{"variant":"key"}` { + t.Errorf("opted-in client got %s", got) + } +} + +func TestConfigureIncludeRepeats(t *testing.T) { + h := hub.New() + go h.Run() + server := httptest.NewServer(Handler(h, nil, 5, 100, nil)) + t.Cleanup(server.Close) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.CloseNow() }) + read := func() map[string]json.RawMessage { + t.Helper() + _, data, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + var msg map[string]json.RawMessage + if err := json.Unmarshal(data, &msg); err != nil { + t.Fatal(err) + } + return msg + } + send := func(msg string) { + t.Helper() + if err := conn.Write(ctx, websocket.MessageText, []byte(msg)); err != nil { + t.Fatal(err) + } + } + configure := func(msg, want string) { + t.Helper() + send(msg) + if reply := read(); string(reply["type"]) != `"configured"` || string(reply["includeRepeats"]) != want { + t.Fatalf("configured reply: %v", reply) + } + time.Sleep(20 * time.Millisecond) // let the hub apply it + } + // Each repeat is followed by a normal marker, so a skipped repeat shows up as the marker. + next := func() string { + t.Helper() + h.BroadcastRepeat(hub.Event{Type: hub.EventPacketObservation, Payload: json.RawMessage(`"repeat"`)}) + h.Broadcast(hub.Event{Type: hub.EventPacketObservation, Payload: json.RawMessage(`"marker"`)}) + got := string(read()["data"]) + if got == `"repeat"` { + read() // marker + } + return got + } + + read() // hello + configure(`{"v":1,"type":"configure","id":"c","includeRepeats":true}`, "true") + send(`{"v":1,"type":"subscribe","id":"s","scope":{}}`) + read() // subscribed + time.Sleep(20 * time.Millisecond) + if got := next(); got != `"repeat"` { + t.Fatalf("opted-in client got %s", got) + } + configure(`{"v":1,"type":"configure","id":"c","resolvePath":true}`, "false") + if got := next(); got != `"marker"` { + t.Fatalf("client still received a repeat after opting out: %s", got) + } +} diff --git a/sqlc.yaml b/sqlc.yaml index 79431093..177f5b3c 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -5,7 +5,7 @@ version: "2" sql: - engine: "postgresql" queries: "db/queries" - schema: "db/migrations" + schema: ["db/migrations", "db/sqlc_scratch"] gen: go: package: "db"