diff --git a/.air.toml b/.air.toml new file mode 100644 index 0000000..0325108 --- /dev/null +++ b/.air.toml @@ -0,0 +1,22 @@ +root = "." +tmp_dir = ".air" + +[build] + cmd = "go build -o ./.air/beacon ./cmd/beacon" + entrypoint = ["./.air/beacon"] + delay = 300 + exclude_dir = [".air", ".git", "data", "node_modules", "playwright-report", "test-results"] + exclude_regex = ["_test.go"] + include_dir = ["cmd", "internal"] + include_ext = ["css", "go", "html", "js", "md", "svg"] + include_file = ["go.mod", "go.sum"] + kill_delay = "1s" + send_interrupt = true + stop_on_error = true + +[misc] + clean_on_exit = true + +[screen] + clear_on_rebuild = false + keep_scroll = true diff --git a/.gitignore b/.gitignore index 979ec4d..02e113e 100644 --- a/.gitignore +++ b/.gitignore @@ -43,8 +43,10 @@ config.toml beacon.db # Artifacts +/data/ /beacon /dist/ +/.air/ # Tailwind/daisyUI standalone CLI + plugin bundles (dev-time download, not # source); see internal/ui/uisrc/README.md to re-fetch. diff --git a/README.md b/README.md index 42d51c3..2baef2c 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,9 @@ docker run --rm \ Once it is running: -- **Dashboard:** [localhost:2112/ui](http://localhost:2112/ui) -- **Onboard manual:** [localhost:2112/ui/docs](http://localhost:2112/ui/docs) -- **MCP agent reference:** [localhost:2112/ui/mcp](http://localhost:2112/ui/mcp) +- **Dashboard:** [localhost:2112](http://localhost:2112) +- **Onboard manual:** [localhost:2112/docs](http://localhost:2112/docs) +- **MCP agent reference:** [localhost:2112/mcp/info](http://localhost:2112/mcp/info) - **Interactive API reference:** [localhost:2112/api/docs](http://localhost:2112/api/docs) - **Health:** [localhost:2112/health](http://localhost:2112/health) @@ -211,21 +211,52 @@ read [Concepts](internal/ui/docs/03-concepts.md) and ## The envelope -Every source is normalized into one JSON envelope used by buffers, filters, and -network sinks. Its canonical representation keeps wire truth intact and adds -semantic metadata rather than replacing raw values. +MQTT, SSE, WebSocket, TCP, and NDJSON consumers receive exactly three top-level +keys: -| Field group | What it carries | -|---|---| -| Wire identity | `pgn`, `source`, `dest`, `priority`, `timestamp`, `raw` | -| Route identity | `id`, `connector` after the message enters a route buffer | -| Provenance | `observed_at`, `ingress`, `origin_ingress` | -| Stable device identity | `device_name` and JavaScript-safe `device_name_hex` after address claim | -| Catalog metadata | `pgn_name`, `variant`, `transport`, `manufacturer_code`, `decode` | -| Values | Lossless raw-tick `payload` plus additive unit-scaled `physical` values | +```json +{ + "payload": { + "info": { + "timestamp": "2026-07-25T12:00:00Z", + "receivedAt": "2026-07-25T12:00:00Z", + "adapterId": "socketcan:can0", + "networkId": "can0", + "direction": "received", + "priority": 2, + "pgn": 127250, + "sourceId": 12, + "targetId": null + }, + "heading": 15708 + }, + "metadata": { + "id": 42, + "connector": "heading", + "observed_at": "2026-07-25T12:00:00Z", + "ingress": "can0", + "pgn_name": "Vessel Heading", + "decode": {"status": "decoded", "complete": true} + }, + "raw": "XC9///////8=" +} +``` + +`payload` is the verbatim JSON representation of the decoded +[`open-ships/n2k`](https://github.com/open-ships/n2k) Go struct, including every +exported `MessageInfo` field such as receive time, transport timing, adapter, +network, and direction. A consumer that knows the PGN can unmarshal `payload` +directly into the corresponding `pgn` type. The raw-tick values are unchanged. + +`metadata` holds only what Beacon adds: queue and connector identity, ingress +provenance, stable Device NAME, catalog/decode details, and physical values. + +`raw` is the assembled CAN payload as base64 bytes. It is top-level data, not +metadata. Unknown PGNs still move through HTTP, TCP, MQTT, file, observe, and transparent -routes with their raw bytes. See +routes. Their `payload` is the complete `pgn.UnknownPGN` JSON and their original +bytes remain available at top-level `raw`. See [ADR 0004](docs/adr/0004-keep-wire-values-canonical.md) for the compatibility rationale and [Concepts](internal/ui/docs/03-concepts.md#the-envelope) for the complete field contract. @@ -307,10 +338,10 @@ take away the control surface used to repair it. | Surface | Default location | Purpose | |---|---|---| -| Dashboard | `:2112/ui` | Route graph, pending/retained state, bus diagnostics, device commissioning | -| Manual | `:2112/ui/docs` | Offline getting started, CAN setup, concepts, filters, API, troubleshooting | -| MCP endpoint | `:2112/mcp` | Streamable HTTP tools for agent configuration, health, delivery, and source PGN statistics | -| MCP reference | `:2112/ui/mcp` | Embedded connection guide, tool catalog, and call examples | +| Dashboard | `:2112/dashboard` | Route graph, pending/retained state, bus diagnostics, device commissioning | +| Manual | `:2112/docs` | Offline getting started, CAN setup, concepts, filters, API, troubleshooting | +| MCP endpoint | `:2112/mcp` | Streamable HTTP tools for agent configuration, health, delivery, and source PGN metrics | +| MCP reference | `:2112/mcp/info` | Embedded connection guide, tool catalog, and call examples | | REST API | `:2112/api/v1/...` | Configuration, live state, PGN catalog, inventory, commissioning | | API reference | `:2112/api/docs` | Interactive, embedded OpenAPI 3.1 documentation | | OpenAPI document | `:2112/api/openapi.json` | Machine-readable discovery for SDKs, scripts, and agents | @@ -329,11 +360,10 @@ An MCP client can connect without a cloud relay or companion process: } ``` -The MCP server exposes twelve tools to read the complete configuration, create or +The MCP server exposes tools to read the complete configuration, create or update sources, sinks, and connector routes, delete each entity type, and read -health, delivery statistics, or per-source PGN traffic metrics. Operators and -agents can approve or clear persistent expected-traffic baselines for each -source. It uses the same validation, SQLite persistence, and hot reconciliation as the UI and REST API. +health, delivery metrics, or per-source PGN traffic metrics. It uses the same +validation, SQLite persistence, and hot reconciliation as the UI and REST API. The server, tool schemas, and reference page are all embedded in the Beacon binary; no internet connection, remote schema, CDN, or hosted MCP service is required. @@ -348,6 +378,20 @@ moves, and out-of-range values visible after a restart. The scrape-safe subset is exported as `beacon_source_pgn_*` at `/metrics`; raw payloads and fingerprint identifiers stay in the UI/MCP response to avoid unbounded Prometheus labels. +Every source and sink overview also has a stopped-by-default stream inspector. +Start captures future source-received or sink-sent messages without consuming +connector queues or blocking routing; Stop freezes the current browser-local +capture. An optional CEL expression beside Start filters the server-side +preview using the same `msg` fields as connector filters and can be changed +while streaming without clearing captured rows. Clicking a JSON key or value +shows a usable CEL expression for that field. The inspector switches between +the verbatim decoded n2k JSON and assembled CAN payload bytes in hexadecimal, +with exactly one message per line in either view. Captures can be copied, +exported as n2k JSONL, or exported as one hexadecimal CAN payload per line, +preserving message boundaries. The captured counter tracks the entire browser +session even though only its latest 200 messages remain available for display, +copy, and export. + The admin API also exposes the complete PGN and field catalog, best-effort CAN/USB hardware discovery, stable Device NAME inventory, commissioning baselines, and a machine-readable commissioning report. diff --git a/VERSION b/VERSION index 3eefcb9..9084fa2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0 diff --git a/examples/README.md b/examples/README.md index 5a11c8d..5fc85a2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -15,7 +15,7 @@ stops validating, that test fails. - **`navigation.json`** — the same shape as `minimal.json`, but the connector's filter allow-lists navigation PGNs only (heading, rapid position, COG/SOG, GNSS position: `127250`, `129025`, `129026`, - `129029`), served on `/nav`. See the filters page (`/ui/docs/filters`) + `129029`), served on `/nav`. See the filters page (`/docs/filters`) for how the filter list and CEL expressions work. - **`engine-room.json`** — one CAN source feeding *two* connectors, both filtered to engine PGNs (`127488`, `127489`, `127493`): one to an SSE @@ -38,12 +38,12 @@ stops validating, that test fails. `file_path` must be an absolute path that exists on disk (the sink opens it but does not create missing parent directories) — create `/data/log` first, or change both paths to a directory you control. See the concepts - page (`/ui/docs/concepts`) for file sink delivery semantics, rotation, and + page (`/docs/concepts`) for file sink delivery semantics, rotation, and replaying a `candump` log with `canplayer`. - **`vcan-dev.json`** — identical to `minimal.json` but pointed at `vcan0` instead of a real interface, for developing and testing beacon with no CAN hardware attached. Bring the virtual interface up first (see - `/ui/docs/can-setup`): + `/docs/can-setup`): ``` sudo modprobe vcan @@ -58,7 +58,7 @@ Replace `can0` / `vcan0` with your actual interface name, and adjust sink paths/addresses and connector filters as needed — these are starting points, not fixed configurations. All buffer limits shown are optional; if a connector's `buffer` object sets nothing at all, `max_messages` defaults -to 100000 (see `/ui/docs/concepts`). +to 100000 (see `/docs/concepts`). ## Using an example @@ -72,7 +72,7 @@ leave on the command line permanently. ``` **Offline, against an existing database file** (the file must not be held -open by a running beacon process — see `/ui/docs/api` for why): +open by a running beacon process — see `/docs/api` for why): ``` beacon import --db beacon.db examples/navigation.json # replaces the whole config diff --git a/go.mod b/go.mod index 0109fc9..fd0d557 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/go-chi/chi/v5 v5.3.1 github.com/google/cel-go v0.29.2 github.com/modelcontextprotocol/go-sdk v1.6.1 - github.com/open-ships/n2k v0.3.0 + github.com/open-ships/n2k v1.0.0 github.com/prometheus/client_golang v1.23.2 github.com/spf13/cobra v1.10.2 github.com/yuin/goldmark v1.8.4 diff --git a/go.sum b/go.sum index 898bfd6..bcbb9e4 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/open-ships/n2k v0.3.0 h1:7P/HGxAisNo4FIaFU/PCSTTLLR09rMe6ECOieBaGELQ= -github.com/open-ships/n2k v0.3.0/go.mod h1:9LVVEnTKODTR+g/zIKO9H99S6ckObWy8ZeAMtcCu8vs= +github.com/open-ships/n2k v1.0.0 h1:bMKVqRJHy9liPMZg9PPyZnM4GCkldeIZLXfCeRKCxJ4= +github.com/open-ships/n2k v1.0.0/go.mod h1:xYolOtbRoo4IMuOXPsBgS+wyir25a2n5D9doSXcIUMs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/app/app.go b/internal/app/app.go index d8f693e..f54db66 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -10,7 +10,6 @@ import ( "log/slog" "net" "net/http" - "net/url" "os" "sync" "time" @@ -122,8 +121,8 @@ func Run(ctx context.Context, opts Options) (*App, error) { _ = st.Close() return nil, fmt.Errorf("start data server: %w", err) } - // Load approved expectations before sources start so even their first - // observed stream/change event is compared and persisted. + // Load recent source lifecycle events before sources start, then persist + // new events through the registry's bounded asynchronous writer. if err := reg.AttachSourceMetricPersistence(ctx, st.DB()); err != nil { _ = ds.Stop(ctx) _ = st.Close() @@ -175,24 +174,9 @@ func Run(ctx context.Context, opts Options) (*App, error) { mux.HandleFunc("GET /health", a.handleHealth) mux.Handle(mcpserver.EndpointPath, mcpserver.Handler(cfgSvc, reg, version, log)) mux.Handle("/api/", apiHandler) - mux.Handle("/ui/", uiHandler) - // The exact-path "/ui" mount (alongside the "/ui/" subtree mount above) - // is required for GET /ui to reach uiHandler's own "GET /ui" redirect - // route at all — see ui.Handler's doc comment for why. - mux.Handle("/ui", uiHandler) - // "/docs" and "/docs/{slug}" (spec §5) are permanent redirects to their - // /ui/docs equivalents (internal/ui/docspages.go), not a second copy of - // the manual — "/docs" is one of model.ReservedPathPrefixes, so no HTTP - // sink config can ever collide with either pattern. - mux.HandleFunc("GET /docs", func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, "/ui/docs", http.StatusMovedPermanently) - }) - mux.HandleFunc("GET /docs/{slug}", func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, "/ui/docs/"+url.PathEscape(r.PathValue("slug")), http.StatusMovedPermanently) - }) - mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, "/ui/dashboard", http.StatusFound) - }) + // The UI owns the remaining root-level admin paths. More-specific API, + // MCP, health, and metrics patterns above continue to win. + mux.Handle("/", uiHandler) a.adminSrv = &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second} ln, err := net.Listen("tcp", opts.AdminAddr) if err != nil { diff --git a/internal/app/app_test.go b/internal/app/app_test.go index e664beb..ee2be8e 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -82,13 +82,7 @@ func TestHandleHealthRollup(t *testing.T) { } } -// TestDocsRedirectToUIDocs covers app.go's "/docs" and "/docs/{slug}" -// permanent redirects (spec §5) to their internal/ui/docspages.go -// equivalents — registered on the admin mux itself, not internal/ui's own -// handler, so this is the one place that composition is exercised -// end-to-end. "/docs" is one of model.ReservedPathPrefixes, so no sink -// config can ever be written that would collide with either route. -func TestDocsRedirectToUIDocs(t *testing.T) { +func TestDocsRoutesAreServedAtRoot(t *testing.T) { a := startTestApp(t) client := &http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { @@ -97,11 +91,12 @@ func TestDocsRedirectToUIDocs(t *testing.T) { } cases := []struct { - path string - want string + path string + wantStatus int + want string }{ - {"/docs", "/ui/docs"}, - {"/docs/getting-started", "/ui/docs/getting-started"}, + {"/docs", http.StatusFound, "/docs/getting-started"}, + {"/docs/getting-started", http.StatusOK, ""}, } for _, c := range cases { t.Run(c.path, func(t *testing.T) { @@ -110,8 +105,45 @@ func TestDocsRedirectToUIDocs(t *testing.T) { t.Fatal(err) } defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusMovedPermanently { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusMovedPermanently) + if resp.StatusCode != c.wantStatus { + t.Fatalf("status = %d, want %d", resp.StatusCode, c.wantStatus) + } + if loc := resp.Header.Get("Location"); loc != c.want { + t.Fatalf("Location = %q, want %q", loc, c.want) + } + }) + } +} + +func TestAdminUIIsMountedAtRoot(t *testing.T) { + a := startTestApp(t) + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + cases := []struct { + path string + wantStatus int + want string + }{ + {"/", http.StatusFound, "/dashboard"}, + {"/dashboard", http.StatusOK, ""}, + {"/sources", http.StatusOK, ""}, + {"/mcp/info", http.StatusOK, ""}, + {"/ui", http.StatusNotFound, ""}, + {"/ui/dashboard", http.StatusNotFound, ""}, + } + for _, c := range cases { + t.Run(c.path, func(t *testing.T) { + resp, err := client.Get("http://" + a.AdminAddr() + c.path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != c.wantStatus { + t.Fatalf("status = %d, want %d", resp.StatusCode, c.wantStatus) } if loc := resp.Header.Get("Location"); loc != c.want { t.Fatalf("Location = %q, want %q", loc, c.want) diff --git a/internal/e2e/api_test.go b/internal/e2e/api_test.go index 30c0412..5d48e96 100644 --- a/internal/e2e/api_test.go +++ b/internal/e2e/api_test.go @@ -224,9 +224,8 @@ func TestEndToEndAPIDrivenLifecycle(t *testing.T) { } // Step 4: connect an SSE client, inject heading + depth frames, and - // assert only the heading (127250) messages arrive, with the "info" - // object stripped from the payload (wire-freeze regression). - t.Log("step 4: SSE client observes only the filtered PGN, with info stripped") + // assert only complete heading (127250) n2k payloads arrive. + t.Log("step 4: SSE client observes complete n2k payloads for only the filtered PGN") resp, err := http.Get(dataBase + "/nav") if err != nil { t.Fatal(err) @@ -241,15 +240,15 @@ func TestEndToEndAPIDrivenLifecycle(t *testing.T) { events := sseEvents(t, resp, 2) for _, e := range events { - if e["pgn"].(float64) != 127250 { - t.Fatalf("filter leaked pgn %v", e["pgn"]) + if consumerEnvelopePGN(t, e) != 127250 { + t.Fatalf("filter leaked event %v", e) } - payload, ok := e["payload"].(map[string]any) - if !ok { - t.Fatalf("payload not an object: %v", e["payload"]) + payload, metadata := consumerEnvelopeParts(t, e) + if payload["heading"] == nil { + t.Fatalf("SSE payload lost the n2k VesselHeading fields: %v", payload) } - if _, hasInfo := payload["info"]; hasInfo { - t.Fatalf("SSE payload still contains info: %v", payload) + if metadata["connector"] != "heading" { + t.Fatalf("connector metadata is not nested under metadata: %v", metadata) } } _ = resp.Body.Close() diff --git a/internal/e2e/e2e_test.go b/internal/e2e/e2e_test.go index c047ac7..355525e 100644 --- a/internal/e2e/e2e_test.go +++ b/internal/e2e/e2e_test.go @@ -82,6 +82,39 @@ func sseEvents(t *testing.T, resp *http.Response, n int) []map[string]any { return out } +func consumerEnvelopeParts(t *testing.T, event map[string]any) (map[string]any, map[string]any) { + t.Helper() + if len(event) != 3 { + t.Fatalf("consumer envelope must have only payload, metadata, and raw at the top level: %v", event) + } + payload, ok := event["payload"].(map[string]any) + if !ok { + t.Fatalf("payload is not an object: %v", event["payload"]) + } + metadata, ok := event["metadata"].(map[string]any) + if !ok { + t.Fatalf("metadata is not an object: %v", event["metadata"]) + } + if _, ok := event["raw"]; !ok { + t.Fatalf("raw CAN bytes are not a separate top-level value: %v", event) + } + return payload, metadata +} + +func consumerEnvelopePGN(t *testing.T, event map[string]any) float64 { + t.Helper() + payload, _ := consumerEnvelopeParts(t, event) + info, ok := payload["info"].(map[string]any) + if !ok { + t.Fatalf("payload does not contain the n2k info struct: %v", payload) + } + pgnNumber, ok := info["pgn"].(float64) + if !ok { + t.Fatalf("payload.info.pgn is not a number: %v", info["pgn"]) + } + return pgnNumber +} + func TestEndToEndFilteredSSEWithReplay(t *testing.T) { fake := busfake.New() a := startApp(t, fake) @@ -99,34 +132,34 @@ func TestEndToEndFilteredSSEWithReplay(t *testing.T) { events := sseEvents(t, resp, 2) for _, e := range events { - if e["pgn"].(float64) != 127250 { - t.Fatalf("filter leaked pgn %v", e["pgn"]) + if consumerEnvelopePGN(t, e) != 127250 { + t.Fatalf("filter leaked event %v", e) + } + payload, metadata := consumerEnvelopeParts(t, e) + if payload["heading"] == nil { + t.Fatalf("SSE payload lost the n2k VesselHeading fields: %v", payload) } - // Wire-freeze regression: the payload's redundant "info" object - // (duplicating envelope header fields already present at the top - // level: pgn, source, timestamp, ...) must have been stripped at - // envelope creation, not merely omitted by convention. - payload, ok := e["payload"].(map[string]any) - if !ok { - t.Fatalf("payload not an object: %v", e["payload"]) + if metadata["connector"] != "heading" { + t.Fatalf("connector metadata is not nested under metadata: %v", metadata) } - if _, hasInfo := payload["info"]; hasInfo { - t.Fatalf("SSE payload still contains info: %v", payload) + if _, ok := e["raw"].(string); !ok { + t.Fatalf("raw CAN bytes are not a top-level base64 value: %v", e["raw"]) } } _ = resp.Body.Close() // Replay: reconnect from before the second heading. req, _ := http.NewRequest("GET", fmt.Sprintf("http://%s/nav", a.DataAddr()), nil) - req.Header.Set("Last-Event-ID", fmt.Sprintf("heading:%d", int64(events[0]["id"].(float64)))) + _, firstMetadata := consumerEnvelopeParts(t, events[0]) + req.Header.Set("Last-Event-ID", fmt.Sprintf("heading:%d", int64(firstMetadata["id"].(float64)))) resp2, err := http.DefaultClient.Do(req) if err != nil { t.Fatal(err) } defer func() { _ = resp2.Body.Close() }() replayed := sseEvents(t, resp2, 1) - if replayed[0]["pgn"].(float64) != 127250 { - t.Fatalf("replayed pgn %v", replayed[0]["pgn"]) + if consumerEnvelopePGN(t, replayed[0]) != 127250 { + t.Fatalf("replayed event %v", replayed[0]) } // Health endpoint reports components. diff --git a/internal/e2e/ui_test.go b/internal/e2e/ui_test.go index 901dab6..82a3dfe 100644 --- a/internal/e2e/ui_test.go +++ b/internal/e2e/ui_test.go @@ -69,7 +69,7 @@ func getBody(t *testing.T, target string) string { var totalMessagesPattern = regexp.MustCompile(`Total messages\s*
(\d+)
`) // connectorTotalMessages parses html (a GET -// /ui/frag/connectors/{id}/stats response body) for its TotalMessages tile. +// /frag/connectors/{id}/stats response body) for its TotalMessages tile. func connectorTotalMessages(t *testing.T, html string) int64 { t.Helper() m := totalMessagesPattern.FindStringSubmatch(html) @@ -88,7 +88,7 @@ func connectorTotalMessages(t *testing.T, html string) int64 { // dashboard and connector-detail fragments reflect it; delete the // connector; observe the dashboard fall back to its empty state — purely // through the web UI's own form endpoints (form-encoded POSTs to -// /ui/sources, /ui/sinks, /ui/connectors, mirroring exactly what a browser +// /sources, /sinks, /connectors, mirroring exactly what a browser // submitting internal/ui/templates/frag_source_form.html, // frag_sink_form.html, and frag_connector_form.html sends), against an // app.Run started with an empty store (no seed file). This proves the UI @@ -108,20 +108,20 @@ func TestUIDrivenLifecycle(t *testing.T) { // Step 0: a brand-new, unseeded store starts with zero components, so // the dashboard fragment renders its empty state pointing at - // /ui/sources/new (nothing configured at all yet). - t.Log("step 0: dashboard fragment shows the initial empty state, CTA -> /ui/sources/new") - dash := getBody(t, adminBase+"/ui/frag/dashboard") - if !strings.Contains(dash, `href="/ui/sources/new"`) { - t.Fatalf("initial dashboard fragment = %s, want empty-state CTA linking /ui/sources/new", dash) + // /sources/new (nothing configured at all yet). + t.Log("step 0: dashboard fragment shows the initial empty state, CTA -> /sources/new") + dash := getBody(t, adminBase+"/frag/dashboard") + if !strings.Contains(dash, `href="/sources/new"`) { + t.Fatalf("initial dashboard fragment = %s, want empty-state CTA linking /sources/new", dash) } - // Step 1: POST /ui/sources creates source can0 (socketcan), the UI's + // Step 1: POST /sources creates source can0 (socketcan), the UI's // own create endpoint (see internal/ui/forms.go's handleSourceCreate / // writeSource) — form field names taken from frag_source_form.html // (id/name/type/enabled) and frag_source_type_fields.html's socketcan // branch (interface). - t.Log("step 1: POST /ui/sources creates source can0") - mustStatus(t, postForm(t, adminBase+"/ui/sources", url.Values{ + t.Log("step 1: POST /sources creates source can0") + mustStatus(t, postForm(t, adminBase+"/sources", url.Values{ "id": {"can0"}, "name": {"Main bus"}, "type": {"socketcan"}, @@ -129,11 +129,11 @@ func TestUIDrivenLifecycle(t *testing.T) { "interface": {"can0"}, }), http.StatusOK) - // Step 2: POST /ui/sinks creates sink nav (http_sse /nav) — field names + // Step 2: POST /sinks creates sink nav (http_sse /nav) — field names // from frag_sink_form.html and frag_sink_type_fields.html's http_sse // branch (path). - t.Log("step 2: POST /ui/sinks creates sink nav") - mustStatus(t, postForm(t, adminBase+"/ui/sinks", url.Values{ + t.Log("step 2: POST /sinks creates sink nav") + mustStatus(t, postForm(t, adminBase+"/sinks", url.Values{ "id": {"nav"}, "name": {"Nav stream"}, "type": {"http_sse"}, @@ -142,20 +142,20 @@ func TestUIDrivenLifecycle(t *testing.T) { }), http.StatusOK) // Step 2b: with a source configured but no connector yet, the dashboard - // empty-state CTA switches from /ui/sources/new to /ui/connectors/new (see + // empty-state CTA switches from /sources/new to /connectors/new (see // internal/ui/dashboard.go's dashboardEmptyState). - t.Log("step 2b: dashboard empty state now points at /ui/connectors/new") - dash = getBody(t, adminBase+"/ui/frag/dashboard") - if !strings.Contains(dash, `href="/ui/connectors/new"`) { - t.Fatalf("dashboard fragment (source configured, no connector) = %s, want empty-state CTA linking /ui/connectors/new", dash) + t.Log("step 2b: dashboard empty state now points at /connectors/new") + dash = getBody(t, adminBase+"/frag/dashboard") + if !strings.Contains(dash, `href="/connectors/new"`) { + t.Fatalf("dashboard fragment (source configured, no connector) = %s, want empty-state CTA linking /connectors/new", dash) } - // Step 3: POST /ui/connectors creates connector heading (filter + // Step 3: POST /connectors creates connector heading (filter // pgn==127250, buffer max_messages 1000) — field names from // frag_connector_form.html (id/name/enabled/source_id/sink_id/filters/ // max_messages). - t.Log("step 3: POST /ui/connectors creates connector heading") - mustStatus(t, postForm(t, adminBase+"/ui/connectors", url.Values{ + t.Log("step 3: POST /connectors creates connector heading") + mustStatus(t, postForm(t, adminBase+"/connectors", url.Values{ "id": {"heading"}, "name": {"Heading only"}, "enabled": {"1"}, @@ -168,8 +168,8 @@ func TestUIDrivenLifecycle(t *testing.T) { // Step 4: the dashboard fragment's DAG now shows the heading connector // path instead of the empty-state hero. t.Log("step 4: dashboard fragment contains the connector path") - dash = getBody(t, adminBase+"/ui/frag/dashboard") - for _, want := range []string{"Heading only", `href="/ui/connectors/heading/"`} { + dash = getBody(t, adminBase+"/frag/dashboard") + for _, want := range []string{"Heading only", `href="/connectors/heading/"`} { if !strings.Contains(dash, want) { t.Fatalf("dashboard fragment = %s, want connector path containing %q", dash, want) } @@ -197,8 +197,8 @@ func TestUIDrivenLifecycle(t *testing.T) { events := sseEvents(t, resp, 2) for _, e := range events { - if e["pgn"].(float64) != 127250 { - t.Fatalf("filter leaked pgn %v", e["pgn"]) + if consumerEnvelopePGN(t, e) != 127250 { + t.Fatalf("filter leaked event %v", e) } } _ = resp.Body.Close() @@ -211,24 +211,24 @@ func TestUIDrivenLifecycle(t *testing.T) { // instead (see totalMessagesPattern's comment). t.Log("step 6: connector detail stats fragment reflects delivered messages") pollUntil(t, 2*time.Second, "connector detail TotalMessages >= 1", func() bool { - html := getBody(t, adminBase+"/ui/frag/connectors/heading/stats") + html := getBody(t, adminBase+"/frag/connectors/heading/stats") return connectorTotalMessages(t, html) >= 1 }) - // Step 7: POST /ui/connectors/{id}/delete removes the connector via the + // Step 7: POST /connectors/{id}/delete removes the connector via the // UI's own delete endpoint (see handleConnectorDelete). - t.Log("step 7: POST /ui/connectors/heading/delete removes it") - mustStatus(t, postForm(t, adminBase+"/ui/connectors/heading/delete", url.Values{}), http.StatusOK) + t.Log("step 7: POST /connectors/heading/delete removes it") + mustStatus(t, postForm(t, adminBase+"/connectors/heading/delete", url.Values{}), http.StatusOK) // Step 8: the dashboard fragment falls back to its empty state — with // the source (and sink) still configured, the CTA still points at - // /ui/connectors/new rather than back to /ui/sources/new. + // /connectors/new rather than back to /sources/new. t.Log("step 8: dashboard fragment shows the empty state again") - dash = getBody(t, adminBase+"/ui/frag/dashboard") + dash = getBody(t, adminBase+"/frag/dashboard") if strings.Contains(dash, "Heading only") { t.Fatalf("dashboard fragment still shows the deleted connector's card:\n%s", dash) } - if !strings.Contains(dash, `href="/ui/connectors/new"`) || !strings.Contains(dash, "Add your first connector") { - t.Fatalf("post-delete dashboard fragment = %s, want the empty-state hero linking /ui/connectors/new", dash) + if !strings.Contains(dash, `href="/connectors/new"`) || !strings.Contains(dash, "Add your first connector") { + t.Fatalf("post-delete dashboard fragment = %s, want the empty-state hero linking /connectors/new", dash) } } diff --git a/internal/mcpserver/server.go b/internal/mcpserver/server.go index 67efa8d..5e4522e 100644 --- a/internal/mcpserver/server.go +++ b/internal/mcpserver/server.go @@ -25,7 +25,7 @@ import ( const EndpointPath = "/mcp" // ToolInfo is the stable, human-readable catalog shared by the MCP server and -// the onboard /ui/mcp reference page. +// the onboard /mcp/info reference page. type ToolInfo struct { Name string Access string @@ -41,10 +41,8 @@ var toolCatalog = []ToolInfo{ {Name: "delete_sink", Access: "delete", Description: "Delete an unreferenced sink."}, {Name: "delete_connector", Access: "delete", Description: "Delete a connector route and its route-owned queue."}, {Name: "get_health", Access: "read", Description: "Read rolled-up health and live source, sink, and connector states."}, - {Name: "get_delivery_statistics", Access: "read", Description: "Read delivery rates, totals, pending delivery, retained history, limits, drops, and errors."}, - {Name: "get_source_metrics", Access: "read", Description: "Inspect all PGNs by source and sender, including unknown raw payloads, timing, load, decode quality, addressing, gaps, anomalies, field distributions, approved baselines, and lifecycle events."}, - {Name: "commit_source_traffic_baseline", Access: "write", Description: "Replace one source's persistent expected-traffic baseline with every PGN stream observed since Beacon started."}, - {Name: "clear_source_traffic_baseline", Access: "delete", Description: "Clear one source's approved traffic baseline without deleting its observations or event history."}, + {Name: "get_delivery_metrics", Access: "read", Description: "Read delivery rates, totals, pending delivery, retained history, limits, drops, and errors."}, + {Name: "get_source_metrics", Access: "read", Description: "Inspect all PGNs by source and sender, including unknown raw payloads, timing, load, decode quality, addressing, gaps, field distributions, and lifecycle events."}, } // Catalog returns a copy so callers cannot mutate the registered tool list. @@ -133,7 +131,7 @@ type sourceDefinition struct { URL string `json:"url,omitempty" jsonschema:"Remote URL for HTTP or MQTT sources."` Topic string `json:"topic,omitempty" jsonschema:"MQTT subscription topic."` Headers map[string]string `json:"headers,omitempty" jsonschema:"HTTP headers for SSE or WebSocket ingestion."` - FilePath string `json:"file_path,omitempty" jsonschema:"Absolute capture path for file replay."` + FilePath string `json:"file_path,omitempty" jsonschema:"Absolute capture path for file replay; gzip content and a missing path's .gz counterpart are handled automatically."` Address string `json:"address,omitempty" jsonschema:"host:port for tcp or udp gateway ingestion."` Format string `json:"format,omitempty" jsonschema:"Gateway stream format: ydraw or actisense."` } @@ -287,7 +285,7 @@ type healthOutput struct { Components []supervisor.Status `json:"components"` } -type deliveryStatisticsInput struct { +type deliveryMetricsInput struct { ConnectorID string `json:"connector_id,omitempty" jsonschema:"Optional connector route id. Omit to return every configured connector."` } @@ -299,26 +297,16 @@ type sourceMetricsInput struct { } type sourceMetricsOutput struct { - GeneratedAt time.Time `json:"generated_at"` - Sources map[string][]stats.SourcePGNMetric `json:"sources"` - Baselines map[string][]stats.SourceTrafficBaseline `json:"baselines"` - Events map[string][]stats.SourceMetricEvent `json:"events"` + GeneratedAt time.Time `json:"generated_at"` + Sources map[string][]stats.SourcePGNMetric `json:"sources"` + Events map[string][]stats.SourceMetricEvent `json:"events"` } -type sourceTrafficBaselineInput struct { - SourceID string `json:"source_id" jsonschema:"Configured source id."` +type deliveryMetricsOutput struct { + Connectors map[string]deliveryMetrics `json:"connectors"` } -type sourceTrafficBaselineOutput struct { - SourceID string `json:"source_id"` - Baselines []stats.SourceTrafficBaseline `json:"baselines"` -} - -type deliveryStatisticsOutput struct { - Connectors map[string]deliveryStatistics `json:"connectors"` -} - -type deliveryStatistics struct { +type deliveryMetrics struct { TotalMessages int64 `json:"total_messages"` TotalBytes int64 `json:"total_bytes"` MsgPerSec float64 `json:"msg_per_sec"` @@ -343,8 +331,8 @@ type deliveryStatistics struct { DepthHistory []int64 `json:"pending_history,omitempty"` } -func deliveryStatisticsFromSnapshot(v stats.Snapshot) deliveryStatistics { - out := deliveryStatistics{ +func deliveryMetricsFromSnapshot(v stats.Snapshot) deliveryMetrics { + out := deliveryMetrics{ TotalMessages: v.TotalMessages, TotalBytes: v.TotalBytes, MsgPerSec: v.MsgPerSec, BytesPerSec: v.BytesPerSec, PendingMessages: v.QueueDepth, PendingBytes: v.QueueBytes, @@ -467,24 +455,24 @@ func registerTools(server *sdkmcp.Server, svc *config.Service, reg *stats.Regist return nil, healthOutput{Status: supervisor.RollupHealth(components), Components: components}, nil }) - sdkmcp.AddTool(server, tool("get_delivery_statistics", "Get delivery statistics", readAnnotations), - func(ctx context.Context, _ *sdkmcp.CallToolRequest, in deliveryStatisticsInput) (*sdkmcp.CallToolResult, deliveryStatisticsOutput, error) { - out := deliveryStatisticsOutput{Connectors: map[string]deliveryStatistics{}} + sdkmcp.AddTool(server, tool("get_delivery_metrics", "Get delivery metrics", readAnnotations), + func(ctx context.Context, _ *sdkmcp.CallToolRequest, in deliveryMetricsInput) (*sdkmcp.CallToolResult, deliveryMetricsOutput, error) { + out := deliveryMetricsOutput{Connectors: map[string]deliveryMetrics{}} if in.ConnectorID != "" { if _, err := svc.GetConnector(ctx, in.ConnectorID); err != nil { - return nil, deliveryStatisticsOutput{}, publicError(log, err) + return nil, deliveryMetricsOutput{}, publicError(log, err) } snap, _ := reg.Snapshot(in.ConnectorID) - out.Connectors[in.ConnectorID] = deliveryStatisticsFromSnapshot(snap) + out.Connectors[in.ConnectorID] = deliveryMetricsFromSnapshot(snap) return nil, out, nil } connectors, err := svc.ListConnectors(ctx) if err != nil { - return nil, deliveryStatisticsOutput{}, publicError(log, err) + return nil, deliveryMetricsOutput{}, publicError(log, err) } all := reg.All() for _, connector := range connectors { - out.Connectors[connector.ID] = deliveryStatisticsFromSnapshot(all[connector.ID]) + out.Connectors[connector.ID] = deliveryMetricsFromSnapshot(all[connector.ID]) } return nil, out, nil }) @@ -494,7 +482,6 @@ func registerTools(server *sdkmcp.Server, svc *config.Service, reg *stats.Regist out := sourceMetricsOutput{ GeneratedAt: time.Now().UTC(), Sources: map[string][]stats.SourcePGNMetric{}, - Baselines: map[string][]stats.SourceTrafficBaseline{}, Events: map[string][]stats.SourceMetricEvent{}, } eventLimit := in.EventLimit @@ -506,7 +493,6 @@ func registerTools(server *sdkmcp.Server, svc *config.Service, reg *stats.Regist return nil, sourceMetricsOutput{}, publicError(log, err) } out.Sources[in.SourceID] = filterSourceMetrics(reg.SourcePGNMetrics(in.SourceID), in) - out.Baselines[in.SourceID] = filterSourceBaselines(reg.SourceTrafficBaselines(in.SourceID), in) out.Events[in.SourceID] = filterSourceMetricEvents(reg.SourceMetricEvents(in.SourceID, eventLimit), in) return nil, out, nil } @@ -517,34 +503,10 @@ func registerTools(server *sdkmcp.Server, svc *config.Service, reg *stats.Regist all := reg.AllSourcePGNMetrics() for _, source := range sources { out.Sources[source.ID] = filterSourceMetrics(all[source.ID], in) - out.Baselines[source.ID] = filterSourceBaselines(reg.SourceTrafficBaselines(source.ID), in) out.Events[source.ID] = filterSourceMetricEvents(reg.SourceMetricEvents(source.ID, eventLimit), in) } return nil, out, nil }) - - sdkmcp.AddTool(server, tool("commit_source_traffic_baseline", "Set expected traffic baseline", writeAnnotations), - func(ctx context.Context, _ *sdkmcp.CallToolRequest, in sourceTrafficBaselineInput) (*sdkmcp.CallToolResult, sourceTrafficBaselineOutput, error) { - if _, err := svc.GetSource(ctx, in.SourceID); err != nil { - return nil, sourceTrafficBaselineOutput{}, publicError(log, err) - } - baselines, err := reg.CommitSourceTrafficBaseline(ctx, in.SourceID) - if err != nil { - return nil, sourceTrafficBaselineOutput{}, publicError(log, err) - } - return nil, sourceTrafficBaselineOutput{SourceID: in.SourceID, Baselines: baselines}, nil - }) - - sdkmcp.AddTool(server, tool("clear_source_traffic_baseline", "Clear source traffic baseline", deleteAnnotations), - func(ctx context.Context, _ *sdkmcp.CallToolRequest, in sourceTrafficBaselineInput) (*sdkmcp.CallToolResult, sourceTrafficBaselineOutput, error) { - if _, err := svc.GetSource(ctx, in.SourceID); err != nil { - return nil, sourceTrafficBaselineOutput{}, publicError(log, err) - } - if err := reg.ClearSourceTrafficBaseline(ctx, in.SourceID); err != nil { - return nil, sourceTrafficBaselineOutput{}, publicError(log, err) - } - return nil, sourceTrafficBaselineOutput{SourceID: in.SourceID, Baselines: []stats.SourceTrafficBaseline{}}, nil - }) } func filterSourceMetrics(metrics []stats.SourcePGNMetric, in sourceMetricsInput) []stats.SourcePGNMetric { @@ -561,20 +523,6 @@ func filterSourceMetrics(metrics []stats.SourcePGNMetric, in sourceMetricsInput) return out } -func filterSourceBaselines(baselines []stats.SourceTrafficBaseline, in sourceMetricsInput) []stats.SourceTrafficBaseline { - out := make([]stats.SourceTrafficBaseline, 0, len(baselines)) - for _, baseline := range baselines { - if in.PGN != nil && baseline.PGN != *in.PGN { - continue - } - if in.SourceAddress != nil && baseline.SourceAddress != *in.SourceAddress { - continue - } - out = append(out, baseline) - } - return out -} - func filterSourceMetricEvents(events []stats.SourceMetricEvent, in sourceMetricsInput) []stats.SourceMetricEvent { out := make([]stats.SourceMetricEvent, 0, len(events)) for _, event := range events { diff --git a/internal/mcpserver/server_test.go b/internal/mcpserver/server_test.go index 0fd7c81..26c1331 100644 --- a/internal/mcpserver/server_test.go +++ b/internal/mcpserver/server_test.go @@ -203,13 +203,13 @@ func TestConfigureInspectAndDeleteThroughTools(t *testing.T) { tm.stats.Record("navigation", 4, 512) tm.stats.SetQueue("navigation", 3, 384) - result = callTool(t, tm.client, "get_delivery_statistics", map[string]any{"connector_id": "navigation"}) + result = callTool(t, tm.client, "get_delivery_metrics", map[string]any{"connector_id": "navigation"}) if result.IsError { - t.Fatalf("get_delivery_statistics: %s", toolErrorText(result)) + t.Fatalf("get_delivery_metrics: %s", toolErrorText(result)) } - delivery := decodeStructured[deliveryStatisticsOutput](t, result).Connectors["navigation"] + delivery := decodeStructured[deliveryMetricsOutput](t, result).Connectors["navigation"] if delivery.TotalMessages != 4 || delivery.TotalBytes != 512 || delivery.PendingMessages != 3 || delivery.PendingBytes != 384 { - t.Fatalf("delivery statistics = %+v", delivery) + t.Fatalf("delivery metrics = %+v", delivery) } result = callTool(t, tm.client, "get_health", map[string]any{}) @@ -278,50 +278,6 @@ func TestGetSourceMetricsReturnsAndFiltersSharedPGNStore(t *testing.T) { } } -func TestSourceTrafficBaselineToolsPersistAndReportEvents(t *testing.T) { - tm := newTestMCP(t) - if err := tm.svc.PutSource(context.Background(), model.Source{ - ID: "can0", Name: "CAN", Type: model.SourceSocketCAN, Interface: "can0", - }, true); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - tm.stats.RecordSource("can0", &msg.Envelope{ - PGN: 127250, PGNName: "Vessel Heading", Source: 12, Dest: 255, Priority: 2, - Raw: []byte{1, 2, 3, byte(i)}, Decode: msg.DecodeInfo{Status: "decoded", Complete: true}, - }) - } - - result := callTool(t, tm.client, "commit_source_traffic_baseline", map[string]any{"source_id": "can0"}) - if result.IsError { - t.Fatalf("commit baseline: %s", toolErrorText(result)) - } - committed := decodeStructured[sourceTrafficBaselineOutput](t, result) - if len(committed.Baselines) != 1 || committed.Baselines[0].PGN != 127250 { - t.Fatalf("committed baseline = %+v", committed) - } - - result = callTool(t, tm.client, "get_source_metrics", map[string]any{"source_id": "can0", "event_limit": 10}) - if result.IsError { - t.Fatalf("get metrics after baseline: %s", toolErrorText(result)) - } - out := decodeStructured[sourceMetricsOutput](t, result) - if len(out.Baselines["can0"]) != 1 || len(out.Events["can0"]) == 0 { - t.Fatalf("baseline/event output = %+v", out) - } - if got := out.Sources["can0"][0].BaselineStatus; got != "matching" { - t.Fatalf("baseline status = %q, want matching", got) - } - - result = callTool(t, tm.client, "clear_source_traffic_baseline", map[string]any{"source_id": "can0"}) - if result.IsError { - t.Fatalf("clear baseline: %s", toolErrorText(result)) - } - if baselines := tm.stats.SourceTrafficBaselines("can0"); len(baselines) != 0 { - t.Fatalf("baselines after clear = %+v", baselines) - } -} - func TestValidationAndDependencyFailuresAreToolErrors(t *testing.T) { tm := newTestMCP(t) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 549a454..f45ea70 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -21,44 +21,39 @@ import ( type gaugeKey struct{ kind, id string } type Set struct { - connectorMessages api.Int64Counter - connectorBytes api.Int64Counter - sourceMessages api.Int64Counter - subscriberDrops api.Int64Counter - queueDepth api.Int64ObservableGauge - queueBytes api.Int64ObservableGauge - componentState api.Int64ObservableGauge - sinkClients api.Int64UpDownCounter - sourcePGNMessages api.Int64ObservableCounter - sourcePGNFrequency api.Float64ObservableGauge - sourcePGNPeriod api.Float64ObservableGauge - sourcePGNLastSeen api.Float64ObservableGauge - sourcePGNPayload api.Float64ObservableGauge - sourcePGNGap api.Int64ObservableGauge - sourcePGNGapRatio api.Float64ObservableGauge - sourcePGNGaps api.Int64ObservableCounter - sourcePGNAnomaly api.Int64ObservableGauge - sourcePGNAnomalies api.Int64ObservableCounter - sourcePGNTiming api.Float64ObservableGauge - sourcePGNTraffic api.Float64ObservableGauge - sourcePGNDecode api.Int64ObservableCounter - sourcePGNDecodeOutcome api.Int64ObservableCounter - sourcePGNDecodeMissing api.Int64ObservableCounter - sourcePGNBursts api.Int64ObservableCounter - sourcePGNPayloadLength api.Int64ObservableCounter - sourcePGNDestination api.Int64ObservableCounter - sourcePGNPriority api.Int64ObservableCounter - sourcePGNIdentity api.Int64ObservableCounter - sourcePGNStatus api.Int64ObservableGauge - sourcePGNBaseline api.Int64ObservableGauge - sourcePGNBaselineValue api.Float64ObservableGauge - sourcePGNRaw api.Float64ObservableGauge - sourcePGNRawByte api.Float64ObservableGauge - sourcePGNField api.Float64ObservableGauge - sourcePGNFieldState api.Float64ObservableGauge - sourcePGNFieldQuality api.Int64ObservableCounter - sourcePGNFieldAnomalies api.Int64ObservableCounter - sourcePGNCategory api.Float64ObservableGauge + connectorMessages api.Int64Counter + connectorBytes api.Int64Counter + sourceMessages api.Int64Counter + subscriberDrops api.Int64Counter + queueDepth api.Int64ObservableGauge + queueBytes api.Int64ObservableGauge + componentState api.Int64ObservableGauge + sinkClients api.Int64UpDownCounter + sourcePGNMessages api.Int64ObservableCounter + sourcePGNFrequency api.Float64ObservableGauge + sourcePGNPeriod api.Float64ObservableGauge + sourcePGNLastSeen api.Float64ObservableGauge + sourcePGNPayload api.Float64ObservableGauge + sourcePGNGap api.Int64ObservableGauge + sourcePGNGapRatio api.Float64ObservableGauge + sourcePGNGaps api.Int64ObservableCounter + sourcePGNTiming api.Float64ObservableGauge + sourcePGNTraffic api.Float64ObservableGauge + sourcePGNDecode api.Int64ObservableCounter + sourcePGNDecodeOutcome api.Int64ObservableCounter + sourcePGNDecodeMissing api.Int64ObservableCounter + sourcePGNBursts api.Int64ObservableCounter + sourcePGNPayloadLength api.Int64ObservableCounter + sourcePGNDestination api.Int64ObservableCounter + sourcePGNPriority api.Int64ObservableCounter + sourcePGNIdentity api.Int64ObservableCounter + sourcePGNStatus api.Int64ObservableGauge + sourcePGNRaw api.Float64ObservableGauge + sourcePGNRawByte api.Float64ObservableGauge + sourcePGNField api.Float64ObservableGauge + sourcePGNFieldState api.Float64ObservableGauge + sourcePGNFieldQuality api.Int64ObservableCounter + sourcePGNCategory api.Float64ObservableGauge mu sync.Mutex depths map[string][2]int64 // connector -> {depth, bytes} @@ -91,8 +86,6 @@ func New(registries ...*stats.Registry) (*Set, http.Handler, error) { s.sourcePGNGap, _ = meter.Int64ObservableGauge("beacon.source.pgn.gap_active") s.sourcePGNGapRatio, _ = meter.Float64ObservableGauge("beacon.source.pgn.gap_ratio") s.sourcePGNGaps, _ = meter.Int64ObservableCounter("beacon.source.pgn.gaps") - s.sourcePGNAnomaly, _ = meter.Int64ObservableGauge("beacon.source.pgn.anomaly_active") - s.sourcePGNAnomalies, _ = meter.Int64ObservableCounter("beacon.source.pgn.anomalies") s.sourcePGNTiming, _ = meter.Float64ObservableGauge("beacon.source.pgn.timing_seconds") s.sourcePGNTraffic, _ = meter.Float64ObservableGauge("beacon.source.pgn.traffic") s.sourcePGNDecode, _ = meter.Int64ObservableCounter("beacon.source.pgn.decode.messages") @@ -104,14 +97,11 @@ func New(registries ...*stats.Registry) (*Set, http.Handler, error) { s.sourcePGNPriority, _ = meter.Int64ObservableCounter("beacon.source.pgn.priority.messages") s.sourcePGNIdentity, _ = meter.Int64ObservableCounter("beacon.source.pgn.identity_changes") s.sourcePGNStatus, _ = meter.Int64ObservableGauge("beacon.source.pgn.status") - s.sourcePGNBaseline, _ = meter.Int64ObservableGauge("beacon.source.pgn.baseline_state") - s.sourcePGNBaselineValue, _ = meter.Float64ObservableGauge("beacon.source.pgn.baseline_value") s.sourcePGNRaw, _ = meter.Float64ObservableGauge("beacon.source.pgn.raw_payload") s.sourcePGNRawByte, _ = meter.Float64ObservableGauge("beacon.source.pgn.raw_byte") s.sourcePGNField, _ = meter.Float64ObservableGauge("beacon.source.pgn.field.value") s.sourcePGNFieldState, _ = meter.Float64ObservableGauge("beacon.source.pgn.field.state") s.sourcePGNFieldQuality, _ = meter.Int64ObservableCounter("beacon.source.pgn.field.quality") - s.sourcePGNFieldAnomalies, _ = meter.Int64ObservableCounter("beacon.source.pgn.field.anomalies") s.sourcePGNCategory, _ = meter.Float64ObservableGauge("beacon.source.pgn.field.category_summary") _, err = meter.RegisterCallback(func(_ context.Context, o api.Observer) error { s.mu.Lock() @@ -136,14 +126,12 @@ func New(registries ...*stats.Registry) (*Set, http.Handler, error) { return nil }, s.sourcePGNMessages, s.sourcePGNFrequency, s.sourcePGNPeriod, s.sourcePGNLastSeen, s.sourcePGNPayload, s.sourcePGNGap, - s.sourcePGNGapRatio, s.sourcePGNGaps, s.sourcePGNAnomaly, - s.sourcePGNAnomalies, s.sourcePGNTiming, s.sourcePGNTraffic, + s.sourcePGNGapRatio, s.sourcePGNGaps, s.sourcePGNTiming, s.sourcePGNTraffic, s.sourcePGNDecode, s.sourcePGNDecodeOutcome, s.sourcePGNDecodeMissing, s.sourcePGNBursts, s.sourcePGNPayloadLength, s.sourcePGNDestination, - s.sourcePGNPriority, s.sourcePGNIdentity, s.sourcePGNStatus, s.sourcePGNBaseline, - s.sourcePGNBaselineValue, s.sourcePGNRaw, s.sourcePGNRawByte, - s.sourcePGNField, s.sourcePGNFieldState, s.sourcePGNFieldQuality, - s.sourcePGNFieldAnomalies, s.sourcePGNCategory) + s.sourcePGNPriority, s.sourcePGNIdentity, s.sourcePGNStatus, + s.sourcePGNRaw, s.sourcePGNRawByte, + s.sourcePGNField, s.sourcePGNFieldState, s.sourcePGNFieldQuality, s.sourcePGNCategory) if err != nil { return nil, nil, err } @@ -168,20 +156,8 @@ func observeSourcePGNMetrics(o api.Observer, set *Set, all map[string][]stats.So attribute.String("transport", stream.Transport), attribute.String("manufacturer_code", manufacturerCode), } - baselineAttrs := appendMetricAttribute(base, attribute.String("status", stream.BaselineStatus)) - o.ObserveInt64(set.sourcePGNBaseline, boolMetric(stream.Expected), api.WithAttributes(baselineAttrs...)) statusAttrs := appendMetricAttribute(base, attribute.String("status", stream.Status)) o.ObserveInt64(set.sourcePGNStatus, 1, api.WithAttributes(statusAttrs...)) - if stream.Expected { - for statistic, value := range map[string]float64{ - "expected_frequency_hz": stream.BaselineFrequencyHz, - "frequency_tolerance_percent": stream.BaselineTolerancePercent, - "frequency_drift_percent": stream.FrequencyDriftPercent, - } { - attrs := appendMetricAttribute(base, attribute.String("statistic", statistic)) - o.ObserveFloat64(set.sourcePGNBaselineValue, value, api.WithAttributes(attrs...)) - } - } if !stream.Observed { continue } @@ -191,7 +167,7 @@ func observeSourcePGNMetrics(o api.Observer, set *Set, all map[string][]stats.So o.ObserveFloat64(set.sourcePGNLastSeen, float64(stream.LastSeen.UnixNano())/1e9, api.WithAttributes(base...)) for statistic, value := range map[string]float64{ "shortest": stream.ShortestPeriodSeconds, "median": stream.ExpectedPeriodSeconds, - "p95": stream.PeriodP95Seconds, "p99": stream.PeriodP99Seconds, + "p90": stream.PeriodP90Seconds, "p95": stream.PeriodP95Seconds, "p99": stream.PeriodP99Seconds, "mad": stream.JitterMADSeconds, } { attrs := appendMetricAttribute(base, attribute.String("statistic", statistic)) @@ -245,12 +221,6 @@ func observeSourcePGNMetrics(o api.Observer, set *Set, all map[string][]stats.So o.ObserveInt64(set.sourcePGNGap, gap, api.WithAttributes(base...)) o.ObserveFloat64(set.sourcePGNGapRatio, stream.GapRatio, api.WithAttributes(base...)) o.ObserveInt64(set.sourcePGNGaps, stream.GapCount, api.WithAttributes(base...)) - anomaly := int64(0) - if stream.AnomalyActive { - anomaly = 1 - } - o.ObserveInt64(set.sourcePGNAnomaly, anomaly, api.WithAttributes(base...)) - o.ObserveInt64(set.sourcePGNAnomalies, stream.AnomalyCount, api.WithAttributes(base...)) if stream.Raw != nil { for length, count := range stream.Raw.LengthCounts { attrs := appendMetricAttribute(base, attribute.String("length_bytes", length)) @@ -301,20 +271,17 @@ func observeSourcePGNMetrics(o api.Observer, set *Set, all map[string][]stats.So attrs := appendMetricAttribute(fieldBase, attribute.String("statistic", statistic)) o.ObserveFloat64(set.sourcePGNField, *value, api.WithAttributes(attrs...)) } - o.ObserveInt64(set.sourcePGNFieldAnomalies, field.AnomalyCount, api.WithAttributes(fieldBase...)) } for statistic, value := range map[string]float64{ "availability_percent": field.AvailabilityPercent, "stuck_seconds": field.StuckSeconds, - "anomaly_score": field.AnomalyScore, } { attrs := appendMetricAttribute(fieldBase, attribute.String("statistic", statistic)) o.ObserveFloat64(set.sourcePGNFieldState, value, api.WithAttributes(attrs...)) } for kind, count := range map[string]int64{ "present": field.PresentMessages, "missing": field.MissingMessages, - "invalid": field.InvalidCount, "out_of_range": field.OutOfRangeCount, - "novel_category": field.NovelValueCount, + "invalid": field.InvalidCount, "novel_category": field.NovelValueCount, } { attrs := appendMetricAttribute(fieldBase, attribute.String("kind", kind)) o.ObserveInt64(set.sourcePGNFieldQuality, count, api.WithAttributes(attrs...)) @@ -334,13 +301,6 @@ func observeSourcePGNMetrics(o api.Observer, set *Set, all map[string][]stats.So } } -func boolMetric(value bool) int64 { - if value { - return 1 - } - return 0 -} - func categorySummary(field stats.FieldDistribution) (distinct int, total int64, topShare float64) { var top int64 for _, count := range field.Values { diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 4f78d46..659b92d 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -103,7 +103,6 @@ func TestPrometheusExposesSharedSourcePGNMetrics(t *testing.T) { "beacon_source_pgn_last_seen_unixtime", "beacon_source_pgn_payload_bytes", "beacon_source_pgn_gap_active", - "beacon_source_pgn_anomaly_active", "beacon_source_pgn_timing_seconds", "beacon_source_pgn_traffic", "beacon_source_pgn_decode_messages_total", @@ -111,7 +110,6 @@ func TestPrometheusExposesSharedSourcePGNMetrics(t *testing.T) { "beacon_source_pgn_payload_length_messages_total", "beacon_source_pgn_destination_messages_total", "beacon_source_pgn_priority_messages_total", - "beacon_source_pgn_baseline_state", "beacon_source_pgn_status", "beacon_source_pgn_raw_payload", "beacon_source_pgn_raw_byte", @@ -122,6 +120,7 @@ func TestPrometheusExposesSharedSourcePGNMetrics(t *testing.T) { `pgn="127250"`, `source="can0"`, `source_address="12"`, + `statistic="p90"`, } { if !strings.Contains(body, want) { t.Fatalf("source PGN exposition missing %q:\n%s", want, body) diff --git a/internal/model/model.go b/internal/model/model.go index ebcfffa..cde2f7f 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -56,7 +56,11 @@ const ( ) // ReservedPathPrefixes cannot be used by HTTP sink paths. -var ReservedPathPrefixes = []string{"/api", "/ui", "/docs", "/mcp", "/metrics", "/health"} +var ReservedPathPrefixes = []string{ + "/api", "/assets", "/cel-completions", "/config", "/connectors", + "/dashboard", "/docs", "/frag", "/health", "/mcp", "/metrics", + "/n2k", "/sinks", "/sources", +} // Duration marshals as a Go duration string ("90s", "24h"). type Duration time.Duration @@ -88,7 +92,7 @@ type Source struct { URL string `json:"url,omitempty"` // http_sse / http_ws / mqtt broker Topic string `json:"topic,omitempty"` // mqtt Headers map[string]string `json:"headers,omitempty"` // http_sse / http_ws - FilePath string `json:"file_path,omitempty"` // file: capture log to replay + FilePath string `json:"file_path,omitempty"` // file: capture log to replay; gzip is transparent Address string `json:"address,omitempty"` // tcp/udp: gateway host:port Format string `json:"format,omitempty"` // tcp/udp: "ydraw" or "actisense" } diff --git a/internal/msg/envelope.go b/internal/msg/envelope.go index 99a3b2e..6dcbb95 100644 --- a/internal/msg/envelope.go +++ b/internal/msg/envelope.go @@ -1,5 +1,5 @@ // Package msg defines the canonical message envelope that flows through -// queues, HTTP wire formats, and CEL filters. +// queues, software-facing wire formats, and CEL filters. package msg import ( @@ -51,12 +51,17 @@ const ( broadcastDest = 255 ) -// FromPGN converts a decoded n2k message into an Envelope. Known PGNs get -// their payload marshaled to JSON and Raw set to the canonical re-encoding; -// UnknownPGN keeps its original bytes and a null payload. +// FromPGN converts a decoded n2k message into an Envelope. Payload is the +// complete native JSON representation of the n2k Go struct, including every +// exported MessageInfo field. Known PGNs get Raw set to the canonical +// re-encoding; UnknownPGN keeps its original bytes. func FromPGN(m pgn.Message) (*Envelope, error) { // Handle UnknownPGN separately since it doesn't implement pgn.PGN if u, isUnknown := m.(*pgn.UnknownPGN); isUnknown { + payload, err := json.Marshal(u) + if err != nil { + return nil, fmt.Errorf("marshal unknown PGN %d payload: %w", u.Info.PGN, err) + } e := &Envelope{ PGN: u.Info.PGN, Source: u.Info.SourceId, @@ -64,7 +69,7 @@ func FromPGN(m pgn.Message) (*Envelope, error) { Priority: defaultPriority, Timestamp: u.Info.Timestamp, ObservedAt: time.Now().UTC(), - Payload: json.RawMessage("null"), + Payload: payload, Raw: append([]byte(nil), u.Data...), Decode: DecodeInfo{Status: "unknown"}, } @@ -116,10 +121,6 @@ func FromPGN(m pgn.Message) (*Envelope, error) { if err != nil { return nil, fmt.Errorf("marshal PGN %d payload: %w", e.PGN, err) } - payload, err = stripInfo(payload) - if err != nil { - return nil, fmt.Errorf("strip info from PGN %d payload: %w", e.PGN, err) - } e.Payload = payload raw, err := pgn.EncodeMessage(m) if err != nil { @@ -137,20 +138,6 @@ func FromPGN(m pgn.Message) (*Envelope, error) { return e, nil } -// stripInfo removes the redundant top-level "info" key that pgn.PGN types -// embed in their JSON encoding (MessageInfo: timestamp, priority, pgn, -// source/target id). The envelope header fields already carry that data, so -// keeping it in the payload too would duplicate it on the wire. Done once at -// envelope creation, not per read. -func stripInfo(payload json.RawMessage) (json.RawMessage, error) { - var m map[string]json.RawMessage - if err := json.Unmarshal(payload, &m); err != nil { - return nil, err - } - delete(m, "info") - return json.Marshal(m) -} - // Info rebuilds the MessageInfo for encoding this envelope back onto a CAN bus. func (e *Envelope) Info() pgn.MessageInfo { return pgn.MessageInfo{ diff --git a/internal/msg/envelope_test.go b/internal/msg/envelope_test.go index f6e3228..a3f68e3 100644 --- a/internal/msg/envelope_test.go +++ b/internal/msg/envelope_test.go @@ -1,12 +1,14 @@ package msg import ( + "bytes" "encoding/json" "sync" "testing" "time" "github.com/open-ships/n2k/pgn" + "github.com/open-ships/n2k/raw" ) func heading(v uint64) *pgn.VesselHeading { @@ -53,16 +55,115 @@ func TestFromPGNKnown(t *testing.T) { } } -func TestPayloadOmitsInfo(t *testing.T) { - e, err := FromPGN(heading(15708)) +func TestPayloadPreservesCompleteN2KStruct(t *testing.T) { + original := heading(15708) + e, err := FromPGN(original) if err != nil { t.Fatal(err) } - if _, ok := e.PayloadMap()["info"]; ok { - t.Fatalf("payload still contains info: %s", e.Payload) + var payload pgn.VesselHeading + if err := json.Unmarshal(e.Payload, &payload); err != nil { + t.Fatal(err) + } + if payload.Info.PGN != original.Info.PGN || payload.Info.SourceId != original.Info.SourceId || + payload.Info.Priority == nil || *payload.Info.Priority != *original.Info.Priority || + payload.Heading == nil || *payload.Heading != *original.Heading { + t.Fatalf("payload does not preserve the n2k message fields: %+v", payload) + } + if _, ok := e.PayloadMap()["info"]; !ok { + t.Fatalf("payload is missing info: %s", e.Payload) } if e.PayloadMap()["heading"] == nil { - t.Fatalf("stripping info lost data fields: %s", e.Payload) + t.Fatalf("payload lost data fields: %s", e.Payload) + } +} + +func TestPayloadPreservesCompleteMessageInfo(t *testing.T) { + instance, source := uint64(0), uint64(0) + pressure := int64(1020690) + original := &pgn.ActualPressure{ + Info: pgn.MessageInfo{ + Timestamp: time.Date(2026, 6, 29, 14, 10, 17, 530566931, time.UTC), + ReceivedAt: time.Date(2026, 7, 25, 12, 41, 35, 729702000, time.UTC), + TransportTimestamp: 250 * time.Millisecond, + HasTransportTimestamp: true, + AdapterID: "file:/captures/nav.log", + NetworkID: "can2", + Direction: raw.DirectionReceived, + Priority: pgn.Priority(5), + PGN: 130314, + SourceId: 6, + }, + Instance: &instance, + Source: &source, + Pressure: &pressure, + } + + e, err := FromPGN(original) + if err != nil { + t.Fatal(err) + } + nativeJSON, err := json.Marshal(original) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(e.Payload, nativeJSON) { + t.Fatalf("payload is not the verbatim n2k struct JSON\n got: %s\nwant: %s", e.Payload, nativeJSON) + } + var payloadDocument struct { + Info map[string]json.RawMessage `json:"info"` + } + if err := json.Unmarshal(e.Payload, &payloadDocument); err != nil { + t.Fatal(err) + } + for _, key := range []string{ + "timestamp", "receivedAt", "transportTimestamp", "hasTransportTimestamp", + "adapterId", "networkId", "direction", "priority", "pgn", "sourceId", "targetId", + } { + if _, ok := payloadDocument.Info[key]; !ok { + t.Fatalf("native MessageInfo field %q missing from payload.info: %s", key, e.Payload) + } + } + + document, err := json.Marshal(e) + if err != nil { + t.Fatal(err) + } + var event struct { + Payload pgn.ActualPressure `json:"payload"` + Metadata map[string]json.RawMessage `json:"metadata"` + } + if err := json.Unmarshal(document, &event); err != nil { + t.Fatal(err) + } + if !event.Payload.Info.ReceivedAt.Equal(original.Info.ReceivedAt) || + event.Payload.Info.TransportTimestamp != original.Info.TransportTimestamp || + !event.Payload.Info.HasTransportTimestamp || + event.Payload.Info.AdapterID != original.Info.AdapterID || + event.Payload.Info.NetworkID != original.Info.NetworkID || + event.Payload.Info.Direction != original.Info.Direction { + t.Fatalf("native MessageInfo was not preserved verbatim: %+v", event.Payload.Info) + } + if event.Payload.Instance == nil || *event.Payload.Instance != instance || + event.Payload.Source == nil || *event.Payload.Source != source || + event.Payload.Pressure == nil || *event.Payload.Pressure != pressure { + t.Fatalf("ActualPressure fields were not preserved: %+v", event.Payload) + } + for _, key := range []string{ + "received_at", "transport_timestamp", "has_transport_timestamp", + "adapter_id", "network_id", "direction", + } { + if _, ok := event.Metadata[key]; ok { + t.Fatalf("native MessageInfo field %q was duplicated into Beacon metadata: %s", key, document) + } + } + + var roundTrip Envelope + if err := json.Unmarshal(document, &roundTrip); err != nil { + t.Fatal(err) + } + if !bytes.Equal(roundTrip.Payload, e.Payload) { + t.Fatalf("native payload did not survive envelope ingestion\n got: %s\nwant: %s", roundTrip.Payload, e.Payload) } } @@ -75,8 +176,12 @@ func TestFromPGNUnknown(t *testing.T) { if err != nil { t.Fatal(err) } - if string(e.Payload) != "null" { - t.Fatalf("unknown payload = %s, want null", e.Payload) + var payload pgn.UnknownPGN + if err := json.Unmarshal(e.Payload, &payload); err != nil { + t.Fatalf("unknown payload is not n2k UnknownPGN JSON: %v", err) + } + if payload.Info.PGN != 130999 || payload.Info.SourceId != 9 || !bytes.Equal(payload.Data, []byte{1, 2, 3, 4}) { + t.Fatalf("unknown payload lost n2k struct fields: %+v", payload) } if len(e.Raw) != 4 { t.Fatalf("raw = %v", e.Raw) @@ -94,12 +199,114 @@ func TestEnvelopeJSONShape(t *testing.T) { e.Seq = 42 e.ConnectorID = "nav" b, _ := json.Marshal(e) - var m map[string]any + var m map[string]json.RawMessage _ = json.Unmarshal(b, &m) - for _, k := range []string{"id", "connector", "pgn", "source", "dest", "priority", "timestamp", "observed_at", "pgn_name", "decode", "physical", "payload", "raw"} { - if _, ok := m[k]; !ok { - t.Fatalf("marshaled envelope missing %q: %s", k, b) - } + if len(m) != 3 || len(m["payload"]) == 0 || len(m["metadata"]) == 0 || len(m["raw"]) == 0 { + t.Fatalf("top-level envelope must contain only payload, metadata, and raw: %s", b) + } + + var payload pgn.VesselHeading + if err := json.Unmarshal(m["payload"], &payload); err != nil { + t.Fatalf("payload cannot deserialize directly into n2k type: %v", err) + } + if payload.Info.PGN != 127250 || payload.Info.SourceId != 12 || payload.Heading == nil || *payload.Heading != 15708 { + t.Fatalf("deserialized n2k payload lost fields: %+v", payload) + } + + var metadata Metadata + if err := json.Unmarshal(m["metadata"], &metadata); err != nil { + t.Fatalf("metadata cannot deserialize: %v", err) + } + if metadata.Seq != 42 || metadata.ConnectorID != "nav" || metadata.PGNName != "Vessel Heading" || + metadata.Decode.Status != "decoded" { + t.Fatalf("metadata lost Beacon fields: %+v", metadata) + } + var raw []byte + if err := json.Unmarshal(m["raw"], &raw); err != nil || len(raw) == 0 { + t.Fatalf("raw CAN bytes are not a separate top-level value: %s (%v)", m["raw"], err) + } + var metadataFields map[string]json.RawMessage + if err := json.Unmarshal(m["metadata"], &metadataFields); err != nil { + t.Fatal(err) + } + if _, ok := metadataFields["raw"]; ok { + t.Fatalf("raw CAN bytes leaked into metadata: %s", m["metadata"]) + } +} + +func TestEnvelopeJSONRoundTripRestoresInternalFields(t *testing.T) { + e, err := FromPGN(heading(15708)) + if err != nil { + t.Fatal(err) + } + e.Seq = 42 + e.ConnectorID = "nav" + e.Ingress = "can0" + e.OriginIngress = "bridge" + + doc, err := json.Marshal(e) + if err != nil { + t.Fatal(err) + } + var got Envelope + if err := json.Unmarshal(doc, &got); err != nil { + t.Fatal(err) + } + if got.Seq != e.Seq || got.ConnectorID != e.ConnectorID || got.PGN != e.PGN || + got.Source != e.Source || got.Dest != e.Dest || got.Priority != e.Priority || + !got.Timestamp.Equal(e.Timestamp) || got.Ingress != e.Ingress || + got.OriginIngress != e.OriginIngress || !bytes.Equal(got.Payload, e.Payload) || + !bytes.Equal(got.Raw, e.Raw) { + t.Fatalf("round trip mismatch\n got: %+v\nwant: %+v", &got, e) + } +} + +func TestEnvelopeJSONAddsInfoToHandBuiltPayload(t *testing.T) { + e := &Envelope{ + PGN: 127250, + Source: 12, + Dest: 255, + Priority: 2, + Timestamp: time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC), + Payload: json.RawMessage(`{"heading":15708}`), + } + doc, err := json.Marshal(e) + if err != nil { + t.Fatal(err) + } + var wire struct { + Payload pgn.VesselHeading `json:"payload"` + } + if err := json.Unmarshal(doc, &wire); err != nil { + t.Fatal(err) + } + if wire.Payload.Info.PGN != 127250 || wire.Payload.Info.SourceId != 12 || + wire.Payload.Heading == nil || *wire.Payload.Heading != 15708 { + t.Fatalf("augmented payload = %+v", wire.Payload) + } +} + +func TestEnvelopeJSONRequiresTopLevelRaw(t *testing.T) { + e, err := FromPGN(heading(15708)) + if err != nil { + t.Fatal(err) + } + doc, err := json.Marshal(e) + if err != nil { + t.Fatal(err) + } + var wire map[string]json.RawMessage + if err := json.Unmarshal(doc, &wire); err != nil { + t.Fatal(err) + } + delete(wire, "raw") + withoutRaw, err := json.Marshal(wire) + if err != nil { + t.Fatal(err) + } + var got Envelope + if err := json.Unmarshal(withoutRaw, &got); err == nil { + t.Fatal("envelope without top-level raw was accepted") } } diff --git a/internal/msg/wire.go b/internal/msg/wire.go new file mode 100644 index 0000000..819e08d --- /dev/null +++ b/internal/msg/wire.go @@ -0,0 +1,167 @@ +package msg + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/open-ships/n2k/pgn" + + "github.com/open-ships/beacon/internal/n2kcatalog" +) + +// Metadata holds everything Beacon adds to an N2K message. The native n2k +// struct and raw CAN bytes are separate siblings. +type Metadata struct { + Seq int64 `json:"id,omitempty"` + ConnectorID string `json:"connector,omitempty"` + ObservedAt time.Time `json:"observed_at"` + Ingress string `json:"ingress,omitempty"` + OriginIngress string `json:"origin_ingress,omitempty"` + DeviceName *uint64 `json:"device_name,omitempty"` + DeviceNameHex string `json:"device_name_hex,omitempty"` + PGNName string `json:"pgn_name,omitempty"` + Variant string `json:"variant,omitempty"` + Transport string `json:"transport,omitempty"` + ManufacturerCode *uint16 `json:"manufacturer_code,omitempty"` + Decode DecodeInfo `json:"decode"` + Physical map[string]n2kcatalog.PhysicalField `json:"physical,omitempty"` +} + +type wireEnvelope struct { + Payload json.RawMessage `json:"payload"` + Metadata Metadata `json:"metadata"` + Raw []byte `json:"raw"` +} + +// MarshalJSON emits the consumer contract shared by MQTT, SSE, WebSocket, +// TCP, NDJSON files, and persisted route buffers. Only payload and metadata +// plus the raw CAN bytes exist at the top level. Payload is the verbatim JSON +// representation of the native n2k struct. +func (e *Envelope) MarshalJSON() ([]byte, error) { + payload, err := e.consumerPayload() + if err != nil { + return nil, err + } + return json.Marshal(wireEnvelope{ + Payload: payload, + Metadata: Metadata{ + Seq: e.Seq, + ConnectorID: e.ConnectorID, + ObservedAt: e.ObservedAt, + Ingress: e.Ingress, + OriginIngress: e.OriginIngress, + DeviceName: e.DeviceName, + DeviceNameHex: e.DeviceNameHex, + PGNName: e.PGNName, + Variant: e.Variant, + Transport: e.Transport, + ManufacturerCode: e.ManufacturerCode, + Decode: e.Decode, + Physical: e.Physical, + }, + Raw: e.Raw, + }) +} + +// UnmarshalJSON accepts the same three-key consumer contract. CAN identity is +// restored from payload.info, so an upstream Beacon can ingest the document +// without duplicating N2K fields into Beacon metadata. +func (e *Envelope) UnmarshalJSON(data []byte) error { + var wire struct { + Payload json.RawMessage `json:"payload"` + Metadata json.RawMessage `json:"metadata"` + Raw json.RawMessage `json:"raw"` + } + if err := json.Unmarshal(data, &wire); err != nil { + return err + } + if len(wire.Payload) == 0 { + return errors.New(`beacon envelope is missing "payload"`) + } + if len(wire.Metadata) == 0 { + return errors.New(`beacon envelope is missing "metadata"`) + } + if len(wire.Raw) == 0 { + return errors.New(`beacon envelope is missing "raw"`) + } + + var metadata Metadata + if err := json.Unmarshal(wire.Metadata, &metadata); err != nil { + return fmt.Errorf("decode beacon metadata: %w", err) + } + var payloadHeader struct { + Info *pgn.MessageInfo `json:"info"` + } + if err := json.Unmarshal(wire.Payload, &payloadHeader); err != nil { + return fmt.Errorf("decode N2K payload header: %w", err) + } + if payloadHeader.Info == nil { + return errors.New(`beacon envelope payload is missing N2K "info"`) + } + var raw []byte + if err := json.Unmarshal(wire.Raw, &raw); err != nil { + return fmt.Errorf("decode raw CAN payload: %w", err) + } + + info := *payloadHeader.Info + priority := uint8(defaultPriority) + if info.Priority != nil { + priority = *info.Priority + } + dest := uint8(broadcastDest) + if info.TargetId != nil { + dest = *info.TargetId + } + + *e = Envelope{ + Seq: metadata.Seq, + ConnectorID: metadata.ConnectorID, + PGN: info.PGN, + Source: info.SourceId, + Dest: dest, + Priority: priority, + Timestamp: info.Timestamp, + ObservedAt: metadata.ObservedAt, + Ingress: metadata.Ingress, + OriginIngress: metadata.OriginIngress, + DeviceName: metadata.DeviceName, + DeviceNameHex: metadata.DeviceNameHex, + PGNName: metadata.PGNName, + Variant: metadata.Variant, + Transport: metadata.Transport, + ManufacturerCode: metadata.ManufacturerCode, + Decode: metadata.Decode, + Physical: metadata.Physical, + Payload: bytes.Clone(wire.Payload), + Raw: bytes.Clone(raw), + } + return nil +} + +// consumerPayload preserves a native n2k payload byte-for-byte. Hand-built +// envelopes used by internal callers may omit info; add it in that case so no +// producer can emit a payload that consumers cannot deserialize as an n2k +// struct. +func (e *Envelope) consumerPayload() (json.RawMessage, error) { + var fields map[string]json.RawMessage + if len(e.Payload) != 0 { + if err := json.Unmarshal(e.Payload, &fields); err != nil { + return nil, fmt.Errorf("decode N2K payload: %w", err) + } + } + if fields == nil { + fields = make(map[string]json.RawMessage) + } + if info, ok := fields["info"]; !ok || bytes.Equal(bytes.TrimSpace(info), []byte("null")) { + rawInfo, err := json.Marshal(e.Info()) + if err != nil { + return nil, fmt.Errorf("encode N2K payload info: %w", err) + } + fields["info"] = rawInfo + return json.Marshal(fields) + } + return bytes.Clone(e.Payload), nil +} diff --git a/internal/sink/mqtt_test.go b/internal/sink/mqtt_test.go index 5cef405..fda9441 100644 --- a/internal/sink/mqtt_test.go +++ b/internal/sink/mqtt_test.go @@ -4,15 +4,14 @@ import ( "bufio" "context" "encoding/binary" + "encoding/json" "io" "log/slog" "net" - "strings" "testing" "time" "github.com/open-ships/beacon/internal/model" - "github.com/open-ships/beacon/internal/msg" "github.com/open-ships/beacon/internal/queue" ) @@ -66,7 +65,7 @@ func TestMQTTSinkBroadcastWithoutBrokerReportsDegraded(t *testing.T) { } defer rt.Stop() - rt.(Broadcaster).Broadcast([]queue.Entry{{Env: &msg.Envelope{PGN: 127250}}}) + rt.(Broadcaster).Broadcast([]queue.Entry{entry("", 0, 127250)}) state, stateErr := rt.State() if state != "degraded" || stateErr == nil { t.Fatalf("state = %q/%v, want degraded/non-nil error", state, stateErr) @@ -127,16 +126,27 @@ func TestMQTTSinkConnectsAndPublishes(t *testing.T) { t.Fatalf("state = %q/%v, want up", state, stateErr) } - rt.(Broadcaster).Broadcast([]queue.Entry{{Env: &msg.Envelope{PGN: 127250}}}) + rt.(Broadcaster).Broadcast([]queue.Entry{entry("", 0, 127250)}) select { case packet := <-published: topic, body, ok := mqttPublishBody(packet) if !ok { t.Fatalf("invalid publish packet: %#v", packet) } - if topic != "beacon/test" || !strings.Contains(string(body), `"pgn":127250`) { + if topic != "beacon/test" { t.Fatalf("publish = topic %q body %s", topic, body) } + var event map[string]any + if err := json.Unmarshal(body, &event); err != nil { + t.Fatalf("MQTT body is not JSON: %v", err) + } + if consumerEnvelopePGN(t, event) != 127250 { + t.Fatalf("publish = topic %q body %s", topic, body) + } + if _, ok := event["raw"].(string); !ok { + t.Fatalf("MQTT raw CAN bytes are not a top-level base64 value: %s", body) + } + assertNativeMessageInfoPreserved(t, event) case <-time.After(time.Second): t.Fatal("broker did not receive publish") } diff --git a/internal/sink/serve_test.go b/internal/sink/serve_test.go index 3d6ee4e..0d0b8f0 100644 --- a/internal/sink/serve_test.go +++ b/internal/sink/serve_test.go @@ -12,6 +12,8 @@ import ( "time" "github.com/coder/websocket" + "github.com/open-ships/n2k/pgn" + "github.com/open-ships/n2k/raw" "github.com/open-ships/beacon/internal/model" "github.com/open-ships/beacon/internal/msg" @@ -44,10 +46,25 @@ func (m *memReplay) Read(_ context.Context, after int64, limit int) ([]queue.Ent return out, nil } -func entry(connector string, seq int64, pgn uint32) queue.Entry { +func entry(connector string, seq int64, pgnNumber uint32) queue.Entry { + receivedAt := time.Now() + payload, _ := json.Marshal(struct { + Info pgn.MessageInfo `json:"info"` + }{ + Info: pgn.MessageInfo{ + Timestamp: receivedAt, + ReceivedAt: receivedAt, + AdapterID: "socketcan:can0", + NetworkID: "can0", + Direction: raw.DirectionReceived, + Priority: pgn.Priority(2), + PGN: pgnNumber, + SourceId: 1, + }, + }) return queue.Entry{Seq: seq, Env: &msg.Envelope{ - Seq: seq, ConnectorID: connector, PGN: pgn, Source: 1, Dest: 255, Priority: 2, - Timestamp: time.Now(), Payload: json.RawMessage(`{}`)}} + Seq: seq, ConnectorID: connector, PGN: pgnNumber, Source: 1, Dest: 255, Priority: 2, + Timestamp: receivedAt, Payload: payload, Raw: []byte{1, 2, 3}}} } func startSSE(t *testing.T) (*DataServer, Runtime) { @@ -106,6 +123,61 @@ func readSSEEvents(t *testing.T, resp *http.Response, n int, timeout time.Durati return out } +func consumerEnvelopeParts(t *testing.T, event map[string]any) (map[string]any, map[string]any) { + t.Helper() + if len(event) != 3 { + t.Fatalf("consumer envelope must have only payload, metadata, and raw at the top level: %v", event) + } + payload, ok := event["payload"].(map[string]any) + if !ok { + t.Fatalf("payload is not an object: %v", event["payload"]) + } + metadata, ok := event["metadata"].(map[string]any) + if !ok { + t.Fatalf("metadata is not an object: %v", event["metadata"]) + } + if _, ok := event["raw"]; !ok { + t.Fatalf("raw CAN bytes are not a separate top-level value: %v", event) + } + return payload, metadata +} + +func consumerEnvelopePGN(t *testing.T, event map[string]any) float64 { + t.Helper() + payload, _ := consumerEnvelopeParts(t, event) + info, ok := payload["info"].(map[string]any) + if !ok { + t.Fatalf("payload does not contain the n2k info struct: %v", payload) + } + pgnNumber, ok := info["pgn"].(float64) + if !ok { + t.Fatalf("payload.info.pgn is not a number: %v", info["pgn"]) + } + return pgnNumber +} + +func assertNativeMessageInfoPreserved(t *testing.T, event map[string]any) { + t.Helper() + payload, metadata := consumerEnvelopeParts(t, event) + info, ok := payload["info"].(map[string]any) + if !ok { + t.Fatalf("payload does not contain the n2k info struct: %v", payload) + } + for _, key := range []string{"receivedAt", "transportTimestamp", "hasTransportTimestamp", "adapterId", "networkId", "direction"} { + if key == "transportTimestamp" || key == "hasTransportTimestamp" { + continue // omitted by n2k when this synthetic message has no transport timestamp + } + if _, ok := info[key]; !ok { + t.Fatalf("native MessageInfo field %q missing from payload.info: %v", key, info) + } + } + for _, key := range []string{"received_at", "adapter_id", "network_id", "direction"} { + if _, ok := metadata[key]; ok { + t.Fatalf("native MessageInfo field %q was duplicated into metadata: %v", key, metadata) + } + } +} + func TestSSELiveBroadcast(t *testing.T) { ds, rt := startSSE(t) resp, err := http.Get(fmt.Sprintf("http://%s/events", ds.Addr())) @@ -119,9 +191,17 @@ func TestSSELiveBroadcast(t *testing.T) { time.Sleep(100 * time.Millisecond) // let the client register rt.(Broadcaster).Broadcast([]queue.Entry{entry("nav", 1, 127250)}) events := readSSEEvents(t, resp, 1, 3*time.Second) - if events[0]["pgn"].(float64) != 127250 { + if consumerEnvelopePGN(t, events[0]) != 127250 { t.Fatalf("event = %v", events[0]) } + _, metadata := consumerEnvelopeParts(t, events[0]) + if metadata["connector"] != "nav" || metadata["id"] != float64(1) { + t.Fatalf("Beacon route metadata is not nested under metadata: %v", metadata) + } + if _, ok := events[0]["raw"].(string); !ok { + t.Fatalf("raw CAN bytes are not a top-level base64 value: %v", events[0]["raw"]) + } + assertNativeMessageInfoPreserved(t, events[0]) } func TestSSEReplayWithLastEventID(t *testing.T) { @@ -138,7 +218,7 @@ func TestSSEReplayWithLastEventID(t *testing.T) { } defer func() { _ = resp.Body.Close() }() events := readSSEEvents(t, resp, 2, 3*time.Second) - if events[0]["pgn"].(float64) != 128259 || events[1]["pgn"].(float64) != 129026 { + if consumerEnvelopePGN(t, events[0]) != 128259 || consumerEnvelopePGN(t, events[1]) != 129026 { t.Fatalf("replay wrong: %v", events) } } @@ -228,3 +308,49 @@ func TestWSDeadClientDetected(t *testing.T) { waitFor(t, 3*time.Second, "dead WS client to be dropped", func() bool { return s.clientCount() == 0 }) } + +func TestWSLiveBroadcastUsesConsumerEnvelope(t *testing.T) { + ds := NewDataServer("127.0.0.1:0", slog.Default()) + if err := ds.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ds.Stop(context.Background()) }) + rt, err := New(context.Background(), model.Sink{ + ID: "ws-contract", Name: "WS", Type: model.SinkHTTPWS, Enabled: true, Path: "/ws-contract", + }, nil, ds, slog.Default(), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(rt.Stop) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, _, err := websocket.Dial(ctx, fmt.Sprintf("ws://%s/ws-contract", ds.Addr()), nil) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.CloseNow() }() + waitFor(t, 3*time.Second, "WS client to register", + func() bool { return rt.(*serveSink).clientCount() == 1 }) + + rt.(Broadcaster).Broadcast([]queue.Entry{entry("nav", 7, 127250)}) + _, data, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + var event map[string]any + if err := json.Unmarshal(data, &event); err != nil { + t.Fatal(err) + } + if consumerEnvelopePGN(t, event) != 127250 { + t.Fatalf("event = %v", event) + } + _, metadata := consumerEnvelopeParts(t, event) + if metadata["connector"] != "nav" || metadata["id"] != float64(7) { + t.Fatalf("Beacon route metadata is not nested under metadata: %v", metadata) + } + if _, ok := event["raw"].(string); !ok { + t.Fatalf("raw CAN bytes are not a top-level base64 value: %v", event["raw"]) + } + assertNativeMessageInfoPreserved(t, event) +} diff --git a/internal/sink/tcp_test.go b/internal/sink/tcp_test.go index 130270a..005d2c4 100644 --- a/internal/sink/tcp_test.go +++ b/internal/sink/tcp_test.go @@ -67,7 +67,8 @@ func TestTCPLiveTail(t *testing.T) { if err := json.Unmarshal(line, &m); err != nil { t.Fatal(err) } - if m["pgn"].(float64) != 127250 || m["connector"].(string) != "nav" { + _, metadata := consumerEnvelopeParts(t, m) + if consumerEnvelopePGN(t, m) != 127250 || metadata["connector"] != "nav" { t.Fatalf("line = %s", line) } } diff --git a/internal/source/gateway.go b/internal/source/gateway.go index 68d9828..e08b0da 100644 --- a/internal/source/gateway.go +++ b/internal/source/gateway.go @@ -1,7 +1,14 @@ package source import ( + "compress/gzip" "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" "sync" n2k "github.com/open-ships/n2k" @@ -54,10 +61,103 @@ func runReceive(ctx context.Context, publish func(*msg.Envelope), connected func } // runFile replays a capture log once at its recorded timing, then holds the -// source "up" until stopped. n2k.File auto-detects candump/canboat/YD/Actisense -// log formats. +// source "up" until stopped. If the configured path does not exist, its .gz +// counterpart is tried. Gzip is detected by content and expanded to a +// short-lived file because n2k.File accepts paths rather than readers. func runFile(ctx context.Context, cfg model.Source, publish func(*msg.Envelope), connected func()) error { - return runReceive(ctx, publish, connected, true, n2k.File(cfg.FilePath, n2k.OriginalTiming())) + sourcePath := discoverReplayFile(cfg.FilePath) + replayPath, cleanup, err := prepareReplayFile(ctx, sourcePath) + if err != nil { + return err + } + + var connectedOnce sync.Once + markConnected := func() { connectedOnce.Do(connected) } + err = runReceive(ctx, publish, markConnected, false, n2k.File(replayPath, n2k.OriginalTiming())) + cleanup() + if err != nil && ctx.Err() == nil { + return fmt.Errorf("replaying capture file %q: %w", sourcePath, err) + } + if err != nil { + return err + } + + // A clean EOF proves that even an empty capture opened successfully. The + // decompressed copy is already gone; keep only the source state up and idle. + markConnected() + if ctx.Err() == nil { + <-ctx.Done() + } + return ctx.Err() +} + +// discoverReplayFile preserves an explicitly configured file when it exists. +// Only a missing path earns the conventional .gz fallback; this avoids +// silently choosing a compressed sidecar over an operator's exact selection. +func discoverReplayFile(path string) string { + if _, err := os.Stat(path); err == nil || !errors.Is(err, os.ErrNotExist) { + return path + } + if strings.EqualFold(filepath.Ext(path), ".gz") { + return path + } + gzipPath := path + ".gz" + if _, err := os.Stat(gzipPath); err == nil || !errors.Is(err, os.ErrNotExist) { + return gzipPath + } + return path +} + +func prepareReplayFile(ctx context.Context, path string) (string, func(), error) { + f, err := os.Open(path) // #nosec G304 -- replaying the configured capture file is this source's purpose. + if err != nil { + return "", func() {}, fmt.Errorf("opening log file %q: %w", path, err) + } + defer func() { _ = f.Close() }() + + var magic [2]byte + n, readErr := f.ReadAt(magic[:], 0) + if readErr != nil && !errors.Is(readErr, io.EOF) { + return "", func() {}, fmt.Errorf("inspecting capture file %q: %w", path, readErr) + } + if n < len(magic) || magic != [2]byte{0x1f, 0x8b} { + return path, func() {}, nil + } + + zr, err := gzip.NewReader(f) + if err != nil { + return "", func() {}, fmt.Errorf("opening gzip capture file %q: %w", path, err) + } + + tmp, err := os.CreateTemp("", "beacon-replay-*.log") + if err != nil { + _ = zr.Close() + return "", func() {}, fmt.Errorf("creating temporary file for gzip capture %q: %w", path, err) + } + tmpPath := tmp.Name() + cleanup := func() { _ = os.Remove(tmpPath) } + + _, copyErr := io.Copy(tmp, contextReader{ctx: ctx, r: zr}) + closeErr := errors.Join(zr.Close(), tmp.Close()) + if err := errors.Join(copyErr, closeErr); err != nil { + cleanup() + return "", func() {}, fmt.Errorf("decompressing gzip capture file %q: %w", path, err) + } + return tmpPath, cleanup, nil +} + +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + default: + return r.r.Read(p) + } } // runTCP ingests from a TCP NMEA-2000 gateway; the dialer's reconnect loop diff --git a/internal/source/gateway_test.go b/internal/source/gateway_test.go index f9d3c91..9ebb503 100644 --- a/internal/source/gateway_test.go +++ b/internal/source/gateway_test.go @@ -1,7 +1,11 @@ package source import ( + "bytes" + "compress/gzip" "context" + "encoding/json" + "errors" "log/slog" "net" "os" @@ -11,8 +15,77 @@ import ( "time" "github.com/open-ships/beacon/internal/model" + "github.com/open-ships/beacon/internal/msg" ) +const replayHeadingLine = "(1720000000.000000) can0 09F11201#015C3DFF7FFF7FFC\n" + +func writeGzipCapture(t *testing.T, path, content string) { + t.Helper() + var compressed bytes.Buffer + zw := gzip.NewWriter(&compressed) + if _, err := zw.Write([]byte(content)); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, compressed.Bytes(), 0o600); err != nil { + t.Fatal(err) + } +} + +func replayFirstEnvelope(t *testing.T, path string) *msg.Envelope { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + published := make(chan *msg.Envelope, 1) + done := make(chan error, 1) + go func() { + done <- runFile(ctx, model.Source{ + ID: "gzip-replay", Type: model.SourceFile, FilePath: path, + }, func(e *msg.Envelope) { + published <- e + }, func() {}) + }() + + var envelope *msg.Envelope + select { + case envelope = <-published: + case <-time.After(time.Second): + cancel() + t.Fatalf("timed out replaying %q", path) + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("runFile() error = %v, want context cancellation", err) + } + case <-time.After(time.Second): + t.Fatalf("runFile() did not stop for %q", path) + } + return envelope +} + +func assertReplayMessageInfoPreserved(t *testing.T, envelope *msg.Envelope) { + t.Helper() + var payload struct { + Info struct { + ReceivedAt time.Time `json:"receivedAt"` + AdapterID string `json:"adapterId"` + NetworkID string `json:"networkId"` + Direction string `json:"direction"` + } `json:"info"` + } + if err := json.Unmarshal(envelope.Payload, &payload); err != nil { + t.Fatal(err) + } + if payload.Info.ReceivedAt.IsZero() || !strings.HasPrefix(payload.Info.AdapterID, "file:") || + payload.Info.NetworkID != "can0" || payload.Info.Direction != "received" { + t.Fatalf("file source MessageInfo was not preserved in payload.info: %s", envelope.Payload) + } +} + func waitStateError(t *testing.T, rt Runtime, wantState, wantErr string, timeout time.Duration) { t.Helper() deadline := time.Now().Add(timeout) @@ -44,23 +117,129 @@ func TestFileSourceMissingPathReportsOpenError(t *testing.T) { } func TestEmptyFileSourceReportsUpAndHolds(t *testing.T) { - path := filepath.Join(t.TempDir(), "empty.log") - if err := os.WriteFile(path, nil, 0o600); err != nil { + for _, gzipCompressed := range []bool{false, true} { + name := "plain" + if gzipCompressed { + name = "gzip" + } + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "empty.log") + if gzipCompressed { + writeGzipCapture(t, path, "") + } else if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rt, err := New(ctx, model.Source{ + ID: "empty-replay", Type: model.SourceFile, FilePath: path, + }, nil, slog.Default(), nil, nil) + if err != nil { + t.Fatal(err) + } + defer rt.Stop() + + if !eventuallyState(rt, "up", time.Second) { + state, stateErr := rt.State() + t.Fatalf("state = %q/%v, want up after clean EOF", state, stateErr) + } + }) + } +} + +func TestFileSourceReplaysGzipByContent(t *testing.T) { + for _, name := range []string{"capture.log.gz", "capture.log"} { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), name) + writeGzipCapture(t, path, replayHeadingLine) + + envelope := replayFirstEnvelope(t, path) + if envelope.PGN != 127250 { + t.Fatalf("PGN = %d, want 127250", envelope.PGN) + } + assertReplayMessageInfoPreserved(t, envelope) + }) + } +} + +func TestFileSourceDiscoversGzipSidecar(t *testing.T) { + path := filepath.Join(t.TempDir(), "capture.log") + writeGzipCapture(t, path+".gz", replayHeadingLine) + + envelope := replayFirstEnvelope(t, path) + if envelope.PGN != 127250 { + t.Fatalf("PGN = %d, want 127250", envelope.PGN) + } +} + +func TestFileSourcePrefersExactPathOverGzipSidecar(t *testing.T) { + path := filepath.Join(t.TempDir(), "capture.log") + if err := os.WriteFile(path, []byte(replayHeadingLine), 0o600); err != nil { + t.Fatal(err) + } + writeGzipCapture(t, path+".gz", "not a capture\n") + + if got := discoverReplayFile(path); got != path { + t.Fatalf("discoverReplayFile() = %q, want exact path %q", got, path) + } + envelope := replayFirstEnvelope(t, path) + if envelope.PGN != 127250 { + t.Fatalf("PGN = %d, want 127250 from exact path", envelope.PGN) + } +} + +func TestPrepareReplayFileRejectsCorruptGzip(t *testing.T) { + path := filepath.Join(t.TempDir(), "capture.log") + if err := os.WriteFile(path, []byte{0x1f, 0x8b, 0x00}, 0o600); err != nil { + t.Fatal(err) + } + + _, cleanup, err := prepareReplayFile(context.Background(), path) + cleanup() + if err == nil || !strings.Contains(err.Error(), "gzip capture file") { + t.Fatalf("prepareReplayFile() error = %v, want gzip-specific error", err) + } +} + +func TestCorruptGzipFileSourceReportsError(t *testing.T) { + path := filepath.Join(t.TempDir(), "capture.log.gz") + if err := os.WriteFile(path, []byte{0x1f, 0x8b, 0x00}, 0o600); err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() rt, err := New(ctx, model.Source{ - ID: "empty-replay", Type: model.SourceFile, FilePath: path, + ID: "corrupt-replay", Type: model.SourceFile, FilePath: path, }, nil, slog.Default(), nil, nil) if err != nil { t.Fatal(err) } defer rt.Stop() - if !eventuallyState(rt, "up", time.Second) { - state, stateErr := rt.State() - t.Fatalf("state = %q/%v, want up after clean EOF", state, stateErr) + waitStateError(t, rt, "degraded", "gzip capture file", time.Second) +} + +func TestPrepareReplayFileCleansUpDecompressedCopy(t *testing.T) { + path := filepath.Join(t.TempDir(), "capture.log.gz") + writeGzipCapture(t, path, replayHeadingLine) + + replayPath, cleanup, err := prepareReplayFile(context.Background(), path) + if err != nil { + t.Fatal(err) + } + if replayPath == path { + t.Fatal("prepareReplayFile() returned compressed source path") + } + content, err := os.ReadFile(replayPath) + if err != nil { + t.Fatal(err) + } + if string(content) != replayHeadingLine { + t.Fatalf("decompressed content = %q, want %q", content, replayHeadingLine) + } + cleanup() + if _, err := os.Stat(replayPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary replay file still exists after cleanup: %v", err) } } diff --git a/internal/stats/source_baseline.go b/internal/stats/source_baseline.go deleted file mode 100644 index 5d0ea35..0000000 --- a/internal/stats/source_baseline.go +++ /dev/null @@ -1,191 +0,0 @@ -package stats - -import ( - "fmt" - "sort" - "strconv" - "strings" - "time" -) - -func (r *Registry) sourceBaselinesFor(source string) []SourceTrafficBaseline { - r.mu.Lock() - defer r.mu.Unlock() - out := make([]SourceTrafficBaseline, 0) - for key, baseline := range r.sourceBaselines { - if key.source == source { - out = append(out, baseline) - } - } - return out -} - -func applySourceBaselines(metrics []SourcePGNMetric, baselines []SourceTrafficBaseline, now, startedAt time.Time) []SourcePGNMetric { - matched := make(map[string]bool) - byKey := make(map[string]SourceTrafficBaseline, len(baselines)) - for _, baseline := range baselines { - byKey[baseline.Identity+fmt.Sprintf(":%d", baseline.PGN)] = baseline - } - for i := range metrics { - identity := sourceBaselineIdentity(metrics[i].DeviceNameHex, metrics[i].SourceAddress) - key := identity + fmt.Sprintf(":%d", metrics[i].PGN) - baseline, ok := byKey[key] - if !ok && metrics[i].DeviceNameHex != "" { - fallback := sourceBaselineIdentity("", metrics[i].SourceAddress) + fmt.Sprintf(":%d", metrics[i].PGN) - baseline, ok = byKey[fallback] - key = fallback - } - if !ok { - metrics[i].BaselineStatus = "not_baselined" - continue - } - matched[key] = true - compareSourceBaseline(&metrics[i], baseline) - } - for _, baseline := range baselines { - key := baseline.Identity + fmt.Sprintf(":%d", baseline.PGN) - if matched[key] { - continue - } - grace := 10 * time.Second - if baseline.ExpectedFrequencyHz > 0 { - candidate := time.Duration(3 / baseline.ExpectedFrequencyHz * float64(time.Second)) - if candidate > grace { - grace = candidate - } - } - status := "awaiting" - gap := false - if now.Sub(startedAt) > grace { - status = "missing" - gap = true - } - approved := baseline.ApprovedAt - metrics = append(metrics, SourcePGNMetric{ - Observed: false, SourceID: baseline.SourceID, PGN: baseline.PGN, PGNName: baseline.PGNName, - Variant: baseline.Variant, Transport: baseline.Transport, DecodeStatus: baseline.DecodeStatus, - SourceAddress: baseline.SourceAddress, DeviceNameHex: baseline.DeviceNameHex, - FrequencyHz: baseline.ExpectedFrequencyHz, - ExpectedPeriodSeconds: baselinePeriodSeconds(baseline.ExpectedFrequencyHz), - Expected: true, BaselineStatus: status, - BaselineFrequencyHz: baseline.ExpectedFrequencyHz, - BaselineTolerancePercent: baseline.FrequencyTolerancePercent, - BaselineApprovedAt: &approved, BaselineIssues: []string{"expected stream has not been observed"}, - GapActive: gap, Status: status, - }) - } - return metrics -} - -func compareSourceBaseline(metric *SourcePGNMetric, baseline SourceTrafficBaseline) { - metric.Expected = true - metric.BaselineStatus = "matching" - metric.BaselineFrequencyHz = baseline.ExpectedFrequencyHz - metric.BaselineTolerancePercent = baseline.FrequencyTolerancePercent - approved := baseline.ApprovedAt - metric.BaselineApprovedAt = &approved - issues := make([]string, 0) - if baseline.ExpectedFrequencyHz > 0 && metric.FrequencyHz > 0 { - metric.FrequencyDriftPercent = (metric.FrequencyHz - baseline.ExpectedFrequencyHz) / baseline.ExpectedFrequencyHz * 100 - if mathAbs(metric.FrequencyDriftPercent) > baseline.FrequencyTolerancePercent { - issues = append(issues, fmt.Sprintf("frequency drift %.1f%%", metric.FrequencyDriftPercent)) - } - } - if metric.PayloadBytesLast > 0 && !containsInt(baseline.PayloadLengths, int(metric.PayloadBytesLast)) { - issues = append(issues, fmt.Sprintf("unexpected payload length %d B", metric.PayloadBytesLast)) - } - if baseline.DecodeStatus != "" && metric.DecodeStatus != baseline.DecodeStatus { - issues = append(issues, "decode status changed") - } - if baseline.Variant != "" && metric.Variant != baseline.Variant { - issues = append(issues, "decode variant changed") - } - if baseline.Transport != "" && metric.Transport != baseline.Transport { - issues = append(issues, "transport changed") - } - for destination := range metric.DestinationCounts { - if !containsUint8String(baseline.Destinations, destination) { - issues = append(issues, fmt.Sprintf("new destination %s", destination)) - } - } - for priority := range metric.PriorityCounts { - if !containsUint8String(baseline.Priorities, priority) { - issues = append(issues, fmt.Sprintf("new priority %s", priority)) - } - } - if baseline.DeviceNameHex != "" && metric.SourceAddress != baseline.SourceAddress { - issues = append(issues, fmt.Sprintf("source address changed from %d", baseline.SourceAddress)) - } - for _, field := range metric.Fields { - expected, ok := baseline.Fields[field.Field] - if !ok || field.LastNumeric == nil { - continue - } - if *field.LastNumeric < expected.Minimum || *field.LastNumeric > expected.Maximum { - issues = append(issues, fmt.Sprintf("%s outside approved range", field.Field)) - } - } - compareRawBaseline(metric, baseline, &issues) - sort.Strings(issues) - metric.BaselineIssues = issues - if len(issues) > 0 { - metric.BaselineStatus = "changed" - if metric.Status != "gap" && metric.Status != "anomaly" { - metric.Status = "changed" - } - } -} - -func compareRawBaseline(metric *SourcePGNMetric, baseline SourceTrafficBaseline, issues *[]string) { - if metric.Raw == nil || len(baseline.RawBytes) == 0 { - return - } - expected := make(map[int]BaselineRawByte, len(baseline.RawBytes)) - for _, rawByte := range baseline.RawBytes { - expected[rawByte.Offset] = rawByte - } - outside := make([]string, 0) - newChangeBits := make([]string, 0) - for _, observed := range metric.Raw.Bytes { - approved, ok := expected[observed.Offset] - if !ok || observed.Minimum < approved.Minimum || observed.Maximum > approved.Maximum { - outside = append(outside, strconv.Itoa(observed.Offset)) - } - observedMask, observedErr := strconv.ParseUint(observed.ChangedBitMaskHex, 16, 8) - approvedMask, approvedErr := strconv.ParseUint(approved.ChangedBitMaskHex, 16, 8) - if ok && observedErr == nil && approvedErr == nil && observedMask&^approvedMask != 0 { - newChangeBits = append(newChangeBits, strconv.Itoa(observed.Offset)) - } - } - if len(outside) > 0 { - *issues = append(*issues, "raw byte outside approved range at offset "+strings.Join(outside, ", ")) - } - if len(newChangeBits) > 0 { - *issues = append(*issues, "new raw bit changes at offset "+strings.Join(newChangeBits, ", ")) - } -} - -func baselinePeriodSeconds(frequencyHz float64) float64 { - if frequencyHz <= 0 { - return 0 - } - return 1 / frequencyHz -} - -func containsInt(values []int, want int) bool { - for _, value := range values { - if value == want { - return true - } - } - return false -} - -func containsUint8String(values []int, want string) bool { - for _, value := range values { - if fmt.Sprintf("%d", value) == want { - return true - } - } - return false -} diff --git a/internal/stats/source_metrics.go b/internal/stats/source_metrics.go index 9725f06..a4c9be2 100644 --- a/internal/stats/source_metrics.go +++ b/internal/stats/source_metrics.go @@ -16,7 +16,6 @@ const ( sourceIntervalSamples = 64 maxSourceFields = 128 maxCategoryValues = 16 - anomalyZThreshold = 6.0 ) // FieldDistribution is the process-local distribution of one decoded field @@ -44,86 +43,67 @@ type FieldDistribution struct { MissingMessages int64 `json:"missing_messages"` AvailabilityPercent float64 `json:"availability_percent"` InvalidCount int64 `json:"invalid_count,omitempty"` - OutOfRangeCount int64 `json:"out_of_range_count,omitempty"` NovelValueCount int64 `json:"novel_value_count,omitempty"` CatalogMinimum *float64 `json:"catalog_minimum,omitempty"` CatalogMaximum *float64 `json:"catalog_maximum,omitempty"` Values map[string]int64 `json:"values,omitempty"` Other int64 `json:"other,omitempty"` - Anomalous bool `json:"anomalous"` - AnomalyScore float64 `json:"anomaly_score,omitempty"` - AnomalyCount int64 `json:"anomaly_count,omitempty"` - LastAnomalyAt *time.Time `json:"last_anomaly_at,omitempty"` - AnomalyReason string `json:"anomaly_reason,omitempty"` } // SourcePGNMetric describes one distinct stream on a configured source. The // CAN source address is part of the identity so two devices sending the same // PGN remain independently observable when one stops transmitting. type SourcePGNMetric struct { - Observed bool `json:"observed"` - SourceID string `json:"source_id"` - PGN uint32 `json:"pgn"` - PGNName string `json:"pgn_name,omitempty"` - Variant string `json:"variant,omitempty"` - Transport string `json:"transport,omitempty"` - ManufacturerCode *uint16 `json:"manufacturer_code,omitempty"` - DecodeStatus string `json:"decode_status"` - DecodeStatuses map[string]int64 `json:"decode_statuses"` - DecodeComplete int64 `json:"decode_complete"` - DecodeIncomplete int64 `json:"decode_incomplete"` - DecodeFallback int64 `json:"decode_fallback"` - UnknownMessages int64 `json:"unknown_messages"` - MissingDecodedFields map[string]int64 `json:"missing_decoded_fields,omitempty"` - SourceAddress uint8 `json:"source_address"` - DeviceName *uint64 `json:"device_name,omitempty"` - DeviceNameHex string `json:"device_name_hex,omitempty"` - Messages int64 `json:"messages"` - FirstSeen time.Time `json:"first_seen"` - LastSeen time.Time `json:"last_seen"` - AgeSeconds float64 `json:"age_seconds"` - FrequencyHz float64 `json:"frequency_hz"` - ExpectedPeriodSeconds float64 `json:"expected_period_seconds"` - ShortestPeriodSeconds float64 `json:"shortest_period_seconds,omitempty"` - LongestPeriodSeconds float64 `json:"longest_period_seconds,omitempty"` - PeriodP95Seconds float64 `json:"period_p95_seconds,omitempty"` - PeriodP99Seconds float64 `json:"period_p99_seconds,omitempty"` - JitterMADSeconds float64 `json:"jitter_mad_seconds,omitempty"` - JitterPercent float64 `json:"jitter_percent,omitempty"` - BurstCount int64 `json:"burst_count"` - RecentMessagesPerSec float64 `json:"recent_messages_per_sec"` - RecentBytesPerSec float64 `json:"recent_bytes_per_sec"` - EstimatedBusLoadPercent float64 `json:"estimated_bus_load_percent"` - TrafficSharePercent float64 `json:"traffic_share_percent"` - Expected bool `json:"expected"` - BaselineStatus string `json:"baseline_status"` - BaselineIssues []string `json:"baseline_issues,omitempty"` - BaselineApprovedAt *time.Time `json:"baseline_approved_at,omitempty"` - BaselineFrequencyHz float64 `json:"baseline_frequency_hz,omitempty"` - BaselineTolerancePercent float64 `json:"baseline_tolerance_percent,omitempty"` - FrequencyDriftPercent float64 `json:"frequency_drift_percent,omitempty"` - PayloadBytesLast int64 `json:"payload_bytes_last"` - PayloadBytesMin int64 `json:"payload_bytes_min"` - PayloadBytesMax int64 `json:"payload_bytes_max"` - PayloadBytesMean float64 `json:"payload_bytes_mean"` - GapActive bool `json:"gap_active"` - GapRatio float64 `json:"gap_ratio,omitempty"` - GapCount int64 `json:"gap_count"` - LastGapAt *time.Time `json:"last_gap_at,omitempty"` - LongestGapSeconds float64 `json:"longest_gap_seconds,omitempty"` - AnomalyActive bool `json:"anomaly_active"` - RecentAnomaly bool `json:"recent_anomaly"` - AnomalyCount int64 `json:"anomaly_count"` - LastAnomalyAt *time.Time `json:"last_anomaly_at,omitempty"` - AnomalyScore float64 `json:"anomaly_score,omitempty"` - AnomalyField string `json:"anomaly_field,omitempty"` - AnomalyReason string `json:"anomaly_reason,omitempty"` - Status string `json:"status"` - DestinationCounts map[string]int64 `json:"destination_counts"` - PriorityCounts map[string]int64 `json:"priority_counts"` - IdentityChanges int64 `json:"identity_changes"` - Raw *RawPayloadDiagnostics `json:"raw,omitempty"` - Fields []FieldDistribution `json:"fields,omitempty"` + Observed bool `json:"observed"` + SourceID string `json:"source_id"` + PGN uint32 `json:"pgn"` + PGNName string `json:"pgn_name,omitempty"` + Variant string `json:"variant,omitempty"` + Transport string `json:"transport,omitempty"` + ManufacturerCode *uint16 `json:"manufacturer_code,omitempty"` + DecodeStatus string `json:"decode_status"` + DecodeStatuses map[string]int64 `json:"decode_statuses"` + DecodeComplete int64 `json:"decode_complete"` + DecodeIncomplete int64 `json:"decode_incomplete"` + DecodeFallback int64 `json:"decode_fallback"` + UnknownMessages int64 `json:"unknown_messages"` + MissingDecodedFields map[string]int64 `json:"missing_decoded_fields,omitempty"` + SourceAddress uint8 `json:"source_address"` + DeviceName *uint64 `json:"device_name,omitempty"` + DeviceNameHex string `json:"device_name_hex,omitempty"` + Messages int64 `json:"messages"` + FirstSeen time.Time `json:"first_seen"` + LastSeen time.Time `json:"last_seen"` + AgeSeconds float64 `json:"age_seconds"` + FrequencyHz float64 `json:"frequency_hz"` + ExpectedPeriodSeconds float64 `json:"expected_period_seconds"` + ShortestPeriodSeconds float64 `json:"shortest_period_seconds,omitempty"` + LongestPeriodSeconds float64 `json:"longest_period_seconds,omitempty"` + PeriodP90Seconds float64 `json:"period_p90_seconds,omitempty"` + PeriodP95Seconds float64 `json:"period_p95_seconds,omitempty"` + PeriodP99Seconds float64 `json:"period_p99_seconds,omitempty"` + JitterMADSeconds float64 `json:"jitter_mad_seconds,omitempty"` + JitterPercent float64 `json:"jitter_percent,omitempty"` + BurstCount int64 `json:"burst_count"` + RecentMessagesPerSec float64 `json:"recent_messages_per_sec"` + RecentBytesPerSec float64 `json:"recent_bytes_per_sec"` + EstimatedBusLoadPercent float64 `json:"estimated_bus_load_percent"` + TrafficSharePercent float64 `json:"traffic_share_percent"` + PayloadBytesLast int64 `json:"payload_bytes_last"` + PayloadBytesMin int64 `json:"payload_bytes_min"` + PayloadBytesMax int64 `json:"payload_bytes_max"` + PayloadBytesMean float64 `json:"payload_bytes_mean"` + GapActive bool `json:"gap_active"` + GapRatio float64 `json:"gap_ratio,omitempty"` + GapCount int64 `json:"gap_count"` + LastGapAt *time.Time `json:"last_gap_at,omitempty"` + LongestGapSeconds float64 `json:"longest_gap_seconds,omitempty"` + Status string `json:"status"` + DestinationCounts map[string]int64 `json:"destination_counts"` + PriorityCounts map[string]int64 `json:"priority_counts"` + IdentityChanges int64 `json:"identity_changes"` + Raw *RawPayloadDiagnostics `json:"raw,omitempty"` + Fields []FieldDistribution `json:"fields,omitempty"` } type sourceStreamKey struct { @@ -145,7 +125,6 @@ type sourceDeviceKey struct { type sourceEventState struct { messages int64 gapCount int64 - anomalyCount int64 identityChanges int64 decodeStatus string payloadLengths int @@ -197,11 +176,6 @@ type sourceField struct { values map[string]int64 lastCategory string other int64 - anomalous bool - score float64 - anomalies int64 - lastAt time.Time - reason string lowerBound *float64 upperBound *float64 numericSamples [sourceIntervalSamples]float64 @@ -210,7 +184,6 @@ type sourceField struct { presentMessages int64 missingMessages int64 invalidCount int64 - outOfRangeCount int64 novelValueCount int64 lastSeen time.Time lastChanged time.Time @@ -251,14 +224,6 @@ type sourceStream struct { gapCount int64 lastGap time.Time longestGap time.Duration - - anomalyActive bool - anomalyCount int64 - lastAnomaly time.Time - currentAnomalyScore float64 - lastAnomalyScore float64 - lastAnomalyField string - lastAnomalyReason string } func (s *sourceStream) addInterval(value time.Duration) { @@ -273,7 +238,7 @@ func (s *sourceStream) sourceEventState() sourceEventState { s.mu.Lock() defer s.mu.Unlock() state := sourceEventState{ - messages: s.messages, gapCount: s.gapCount, anomalyCount: s.anomalyCount, + messages: s.messages, gapCount: s.gapCount, identityChanges: s.identityChanges, decodeStatus: s.lastDecodeStatus, payloadLengths: len(s.wire.lengths), } @@ -427,11 +392,6 @@ func (s *sourceStream) record(now time.Time, e *msg.Envelope) { for _, missing := range e.Decode.Missing { s.missingDecodedFields[missing]++ } - s.anomalyActive = false - s.currentAnomalyScore = 0 - for _, field := range s.fields { - field.anomalous = false - } observedFields := make(map[string]bool) physicalNames := make([]string, 0, len(e.Physical)) @@ -448,9 +408,7 @@ func (s *sourceStream) record(now time.Time, e *msg.Envelope) { field.lowerBound = cloneFloat(value.Minimum) field.upperBound = cloneFloat(value.Maximum) observedFields[name] = true - if field.recordNumeric(now, value.Value, true) { - s.noteAnomaly(now, name, field.score, field.reason) - } + field.recordNumeric(now, value.Value) } values := map[string]any{} @@ -473,9 +431,7 @@ func (s *sourceStream) record(now time.Time, e *msg.Envelope) { field := s.field(name, "number", "") if field != nil { observedFields[name] = true - if field.recordNumeric(now, value, true) { - s.noteAnomaly(now, name, field.score, field.reason) - } + field.recordNumeric(now, value) } case string: if field := s.field(name, "category", ""); field != nil { @@ -559,18 +515,12 @@ func (s *sourceStream) field(name, kind, unit string) *sourceField { return field } -func (f *sourceField) recordNumeric(now time.Time, value float64, detect bool) bool { - f.anomalous = false - f.score = 0 - f.reason = "" +func (f *sourceField) recordNumeric(now time.Time, value float64) { f.presentMessages++ if math.IsNaN(value) || math.IsInf(value, 0) { f.invalidCount++ f.lastSeen = now - return false - } - if detect { - f.detectAnomaly(value) + return } if f.numeric.count > 0 { change := math.Abs(value - f.numeric.last) @@ -591,61 +541,6 @@ func (f *sourceField) recordNumeric(now time.Time, value float64, detect bool) b f.numericLen++ } f.lastSeen = now - if f.anomalous { - f.anomalies++ - f.lastAt = now - if f.reason == "below catalog minimum" || f.reason == "above catalog maximum" { - f.outOfRangeCount++ - } - } - return f.anomalous -} - -func (f *sourceField) detectAnomaly(value float64) { - if f.lowerBound != nil && value < *f.lowerBound { - f.anomalous = true - f.reason = "below catalog minimum" - f.score = 1e9 - return - } - if f.upperBound != nil && value > *f.upperBound { - f.anomalous = true - f.reason = "above catalog maximum" - f.score = 1e9 - return - } - if f.numeric.count < 5 { - return - } - sd := f.numeric.stddev() - if sd > 1e-12 { - f.score = math.Abs(value-f.numeric.mean) / sd - if f.score >= anomalyZThreshold { - f.anomalous = true - f.reason = fmt.Sprintf("value changed %.1f standard deviations from baseline", f.score) - return - } - } - change := math.Abs(value - f.numeric.last) - if f.changes.count >= 5 { - changeSD := f.changes.stddev() - if changeSD > 1e-12 { - changeScore := (change - f.changes.mean) / changeSD - if changeScore > f.score { - f.score = changeScore - } - if changeScore >= anomalyZThreshold { - f.anomalous = true - f.reason = fmt.Sprintf("step change was %.1f standard deviations above normal", changeScore) - return - } - } - } - if sd <= 1e-12 && change > math.Max(math.Abs(f.numeric.mean)*0.5, 1) { - f.anomalous = true - f.score = anomalyZThreshold - f.reason = "large change from a stable baseline" - } } func (f *sourceField) recordCategory(now time.Time, value string) { @@ -669,26 +564,13 @@ func (f *sourceField) recordCategory(now time.Time, value string) { f.lastSeen = now } -func (s *sourceStream) noteAnomaly(now time.Time, field string, score float64, reason string) { - if !s.anomalyActive { - s.anomalyCount++ - } - s.anomalyActive = true - s.lastAnomaly = now - if score >= s.currentAnomalyScore { - s.currentAnomalyScore = score - s.lastAnomalyScore = score - s.lastAnomalyField = field - s.lastAnomalyReason = reason - } -} - func (s *sourceStream) snapshot(now time.Time) SourcePGNMetric { s.mu.Lock() defer s.mu.Unlock() intervals := s.intervalValues() expected := medianInterval(intervals) + periodP90 := durationPercentile(intervals, 0.90) periodP95 := durationPercentile(intervals, 0.95) periodP99 := durationPercentile(intervals, 0.99) jitterMAD := intervalMAD(intervals, expected) @@ -710,22 +592,11 @@ func (s *sourceStream) snapshot(now time.Time) SourcePGNMetric { if expected > 0 { gapRatio = float64(age) / float64(expected) } - recentWindow := time.Minute - if candidate := 3 * expected; candidate > recentWindow { - recentWindow = candidate - } - if recentWindow > 10*time.Minute { - recentWindow = 10 * time.Minute - } - recentAnomaly := !s.lastAnomaly.IsZero() && now.Sub(s.lastAnomaly) <= recentWindow recentMessages, recentBytes, busLoad := s.rate.snapshot(now) status := "active" if len(intervals) < 3 { status = "warming" } - if recentAnomaly { - status = "anomaly" - } if gapActive { status = "gap" } @@ -739,15 +610,13 @@ func (s *sourceStream) snapshot(now time.Time) SourcePGNMetric { SourceAddress: s.key.address, Messages: s.messages, FirstSeen: s.firstSeen, LastSeen: s.lastSeen, AgeSeconds: age.Seconds(), ExpectedPeriodSeconds: expected.Seconds(), ShortestPeriodSeconds: shortest.Seconds(), - LongestPeriodSeconds: longest.Seconds(), PeriodP95Seconds: periodP95.Seconds(), + LongestPeriodSeconds: longest.Seconds(), PeriodP90Seconds: periodP90.Seconds(), + PeriodP95Seconds: periodP95.Seconds(), PeriodP99Seconds: periodP99.Seconds(), JitterMADSeconds: jitterMAD.Seconds(), BurstCount: s.burstCount, RecentMessagesPerSec: recentMessages, RecentBytesPerSec: recentBytes, EstimatedBusLoadPercent: busLoad, GapActive: gapActive, GapRatio: gapRatio, GapCount: s.gapCount, LastGapAt: timePtr(s.lastGap), LongestGapSeconds: s.longestGap.Seconds(), - AnomalyActive: s.anomalyActive, RecentAnomaly: recentAnomaly, - AnomalyCount: s.anomalyCount, LastAnomalyAt: timePtr(s.lastAnomaly), - AnomalyScore: s.lastAnomalyScore, AnomalyField: s.lastAnomalyField, AnomalyReason: s.lastAnomalyReason, Status: status, DestinationCounts: cloneUint8Totals(s.destinations), PriorityCounts: cloneUint8Totals(s.priorities), IdentityChanges: s.identityChanges, Raw: s.wire.snapshot(now), @@ -788,12 +657,10 @@ func (s *sourceStream) snapshot(now time.Time) SourcePGNMetric { func fieldSnapshot(name string, field *sourceField, now time.Time, messages int64) FieldDistribution { out := FieldDistribution{ Field: name, Kind: field.kind, Unit: field.unit, - Anomalous: field.anomalous, AnomalyScore: field.score, - AnomalyCount: field.anomalies, LastAnomalyAt: timePtr(field.lastAt), - AnomalyReason: field.reason, PresentMessages: field.presentMessages, + PresentMessages: field.presentMessages, MissingMessages: field.missingMessages, InvalidCount: field.invalidCount, - OutOfRangeCount: field.outOfRangeCount, NovelValueCount: field.novelValueCount, - CatalogMinimum: cloneFloat(field.lowerBound), CatalogMaximum: cloneFloat(field.upperBound), + NovelValueCount: field.novelValueCount, + CatalogMinimum: cloneFloat(field.lowerBound), CatalogMaximum: cloneFloat(field.upperBound), } if messages > 0 { out.AvailabilityPercent = float64(field.presentMessages) / float64(messages) * 100 @@ -877,7 +744,6 @@ func (r *Registry) SourcePGNMetrics(source string) []SourcePGNMetric { for _, stream := range streams { out = append(out, stream.snapshot(now)) } - out = applySourceBaselines(out, r.sourceBaselinesFor(source), now, r.startedAt) applyTrafficShares(out) sortSourcePGNMetrics(out) return out @@ -900,13 +766,7 @@ func (r *Registry) AllSourcePGNMetrics() map[string][]SourcePGNMetric { metric := stream.snapshot(now) out[metric.SourceID] = append(out[metric.SourceID], metric) } - for _, baseline := range r.SourceTrafficBaselines("") { - if _, ok := out[baseline.SourceID]; !ok { - out[baseline.SourceID] = nil - } - } for source := range out { - out[source] = applySourceBaselines(out[source], r.sourceBaselinesFor(source), now, r.startedAt) applyTrafficShares(out[source]) sortSourcePGNMetrics(out[source]) } @@ -927,7 +787,7 @@ func applyTrafficShares(metrics []SourcePGNMetric) { } func sortSourcePGNMetrics(metrics []SourcePGNMetric) { - priority := map[string]int{"missing": 0, "gap": 1, "anomaly": 2, "changed": 3, "awaiting": 4, "warming": 5, "active": 6} + priority := map[string]int{"missing": 0, "gap": 1, "changed": 2, "awaiting": 3, "warming": 4, "active": 5} sort.Slice(metrics, func(i, j int) bool { left, right := priority[metrics[i].Status], priority[metrics[j].Status] if left != right { @@ -976,9 +836,6 @@ func (r *Registry) sourceLifecycleEvents(now time.Time, source string, e *msg.En if after.gapCount > before.gapCount { add("gap_recovered", "warning", fmt.Sprintf("PGN %d resumed after a sending gap", e.PGN), nil) } - if after.anomalyCount > before.anomalyCount { - add("value_anomaly", "warning", fmt.Sprintf("PGN %d contained an anomalous decoded value", e.PGN), nil) - } if before.messages > 0 && after.decodeStatus != before.decodeStatus { add("decode_status_changed", "warning", fmt.Sprintf("PGN %d decode status changed from %s to %s", e.PGN, before.decodeStatus, after.decodeStatus), map[string]string{"before": before.decodeStatus, "after": after.decodeStatus}) diff --git a/internal/stats/source_metrics_test.go b/internal/stats/source_metrics_test.go index e78c8f6..cdf8fdc 100644 --- a/internal/stats/source_metrics_test.go +++ b/internal/stats/source_metrics_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "math" "path/filepath" - "strings" "testing" "time" @@ -50,6 +49,9 @@ func TestSourcePGNMetricsTrackSenderFrequencyPayloadAndValues(t *testing.T) { if math.Abs(stream.FrequencyHz-1) > 0.001 || math.Abs(stream.ExpectedPeriodSeconds-1) > 0.001 { t.Fatalf("frequency = %v Hz period = %v s, want 1", stream.FrequencyHz, stream.ExpectedPeriodSeconds) } + if math.Abs(stream.PeriodP90Seconds-1) > 0.001 || math.Abs(stream.PeriodP99Seconds-1) > 0.001 { + t.Fatalf("period p90/p99 = %v / %v, want 1", stream.PeriodP90Seconds, stream.PeriodP99Seconds) + } if stream.PayloadBytesLast != 8 || stream.PayloadBytesMin != 8 || stream.PayloadBytesMax != 8 || stream.PayloadBytesMean != 8 { t.Fatalf("payload distribution = %+v", stream) } @@ -100,12 +102,12 @@ func TestSourcePGNMetricsExposeWireDecodeAddressingAndFieldQuality(t *testing.T) if field.PresentMessages != 2 || field.MissingMessages != 1 || math.Abs(field.AvailabilityPercent-66.6667) > 0.01 { t.Fatalf("field availability = %+v", field) } - if field.OutOfRangeCount != 1 || !field.Anomalous || field.LastRateOfChange == nil { + if field.LastRateOfChange == nil || field.Maximum == nil || *field.Maximum != 1000 { t.Fatalf("field quality = %+v", field) } } -func TestSourceTrafficBaselineAndEventsSurviveRestart(t *testing.T) { +func TestSourceMetricEventsSurviveRestart(t *testing.T) { dbPath := filepath.Join(t.TempDir(), "beacon.db") now := time.Unix(1_700_000_000, 0).UTC() st, err := store.Open(dbPath) @@ -123,22 +125,6 @@ func TestSourceTrafficBaselineAndEventsSurviveRestart(t *testing.T) { }) now = now.Add(time.Second) } - baselines, err := reg.CommitSourceTrafficBaseline(t.Context(), "can0") - if err != nil { - t.Fatal(err) - } - if len(baselines) != 1 || math.Abs(baselines[0].ExpectedFrequencyHz-1) > 0.001 { - t.Fatalf("committed baselines = %+v", baselines) - } - reg.RecordSource("can0", &msg.Envelope{ - PGN: 127250, PGNName: "Vessel Heading", Source: 12, Dest: 255, Priority: 2, - Raw: []byte{1, 2, 3, 255}, Decode: msg.DecodeInfo{Status: "decoded", Complete: true}, - }) - changed := reg.SourcePGNMetrics("can0")[0] - if changed.BaselineStatus != "changed" || changed.Status != "changed" || - !strings.Contains(strings.Join(changed.BaselineIssues, " "), "raw byte") { - t.Fatalf("raw baseline change = %+v", changed) - } if err := reg.CloseSourceMetricPersistence(t.Context()); err != nil { t.Fatal(err) } @@ -156,16 +142,11 @@ func TestSourceTrafficBaselineAndEventsSurviveRestart(t *testing.T) { t.Fatal(err) } defer func() { _ = restarted.CloseSourceMetricPersistence(t.Context()) }() - if loaded := restarted.SourceTrafficBaselines("can0"); len(loaded) != 1 || loaded[0].PGN != 127250 { - t.Fatalf("loaded baselines = %+v", loaded) - } - if events := restarted.SourceMetricEvents("can0", 20); len(events) < 2 { - t.Fatalf("loaded events = %+v, want stream and baseline events", events) + if events := restarted.SourceMetricEvents("can0", 20); len(events) == 0 { + t.Fatalf("loaded events = %+v, want persisted source lifecycle events", events) } - now = now.Add(11 * time.Second) - metrics := restarted.SourcePGNMetrics("can0") - if len(metrics) != 1 || metrics[0].Observed || metrics[0].Status != "missing" || !metrics[0].GapActive { - t.Fatalf("missing expected stream = %+v", metrics) + if metrics := restarted.SourcePGNMetrics("can0"); len(metrics) != 0 { + t.Fatalf("live metrics should remain process-local after restart: %+v", metrics) } } @@ -191,58 +172,6 @@ func TestSourcePGNMetricsDetectCurrentAndRecoveredGaps(t *testing.T) { } } -func TestSourcePGNMetricsFlagLargePhysicalValueChange(t *testing.T) { - now := time.Unix(1_700_000_000, 0).UTC() - reg := newRegistryAt(func() time.Time { return now }) - record := func(value float64) { - reg.RecordSource("can0", &msg.Envelope{ - PGN: 130312, Source: 9, Payload: json.RawMessage(`{"temperature":10}`), - Physical: map[string]n2kcatalog.PhysicalField{"temperature": {Value: value, Unit: "K"}}, - }) - now = now.Add(time.Second) - } - for i := 0; i < 6; i++ { - record(10) - } - record(100) - - stream := reg.SourcePGNMetrics("can0")[0] - if !stream.AnomalyActive || !stream.RecentAnomaly || stream.Status != "anomaly" || stream.AnomalyCount != 1 { - t.Fatalf("stream anomaly = %+v", stream) - } - if stream.AnomalyField != "temperature" || stream.AnomalyReason == "" { - t.Fatalf("anomaly explanation = %+v", stream) - } - field := findField(t, stream.Fields, "temperature") - if !field.Anomalous || field.AnomalyCount != 1 || field.Maximum == nil || *field.Maximum != 100 { - t.Fatalf("field anomaly = %+v", field) - } - - record(10) - stream = reg.SourcePGNMetrics("can0")[0] - if stream.AnomalyActive || !stream.RecentAnomaly || stream.AnomalyField != "temperature" { - t.Fatalf("recent anomaly context should survive a normal sample: %+v", stream) - } -} - -func TestSourcePGNMetricsFlagLargeGenericDecodedValueChange(t *testing.T) { - now := time.Unix(1_700_000_000, 0).UTC() - reg := newRegistryAt(func() time.Time { return now }) - for i := 0; i < 6; i++ { - reg.RecordSource("gateway", &msg.Envelope{ - PGN: 99999, Source: 3, Payload: json.RawMessage(`{"sensor":10}`), - }) - now = now.Add(time.Second) - } - reg.RecordSource("gateway", &msg.Envelope{ - PGN: 99999, Source: 3, Payload: json.RawMessage(`{"sensor":1000}`), - }) - stream := reg.SourcePGNMetrics("gateway")[0] - if !stream.AnomalyActive || stream.AnomalyField != "sensor" { - t.Fatalf("generic decoded anomaly = %+v", stream) - } -} - func TestRemoveSourceDropsPGNMetrics(t *testing.T) { reg := NewRegistry() reg.RecordSource("can0", &msg.Envelope{PGN: 127250, Source: 1}) diff --git a/internal/stats/source_persistence.go b/internal/stats/source_persistence.go index 7d059f3..dfb81d7 100644 --- a/internal/stats/source_persistence.go +++ b/internal/stats/source_persistence.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "sort" - "strconv" "sync" "time" ) @@ -18,41 +17,6 @@ const ( sourceMetricEventBufferSize = 256 ) -type BaselineField struct { - Minimum float64 `json:"minimum"` - Maximum float64 `json:"maximum"` - Unit string `json:"unit,omitempty"` -} - -type BaselineRawByte struct { - Offset int `json:"offset"` - Minimum uint8 `json:"minimum"` - Maximum uint8 `json:"maximum"` - ChangedBitMaskHex string `json:"changed_bit_mask_hex"` -} - -// SourceTrafficBaseline is an operator-approved expectation for one PGN from -// one stable Device NAME (or source address when no NAME is available). -type SourceTrafficBaseline struct { - SourceID string `json:"source_id"` - Identity string `json:"identity"` - PGN uint32 `json:"pgn"` - PGNName string `json:"pgn_name,omitempty"` - SourceAddress uint8 `json:"source_address"` - DeviceNameHex string `json:"device_name_hex,omitempty"` - ExpectedFrequencyHz float64 `json:"expected_frequency_hz"` - FrequencyTolerancePercent float64 `json:"frequency_tolerance_percent"` - PayloadLengths []int `json:"payload_lengths"` - DecodeStatus string `json:"decode_status"` - Variant string `json:"variant,omitempty"` - Transport string `json:"transport,omitempty"` - Destinations []int `json:"destinations"` - Priorities []int `json:"priorities"` - Fields map[string]BaselineField `json:"fields,omitempty"` - RawBytes []BaselineRawByte `json:"raw_bytes,omitempty"` - ApprovedAt time.Time `json:"approved_at"` -} - type SourceMetricEvent struct { ID int64 `json:"id,omitempty"` Time time.Time `json:"time"` @@ -66,11 +30,6 @@ type SourceMetricEvent struct { Details map[string]string `json:"details,omitempty"` } -type sourceBaselineKey struct { - source, identity string - pgn uint32 -} - type sourceMetricEventRing struct { items []SourceMetricEvent } @@ -92,16 +51,12 @@ type sourceMetricPersistence struct { closed bool } -// AttachSourceMetricPersistence loads baselines and recent change events, then -// starts a non-blocking event writer. Call this before serving baseline APIs. +// AttachSourceMetricPersistence loads recent source lifecycle events and +// starts their non-blocking persistence writer. func (r *Registry) AttachSourceMetricPersistence(ctx context.Context, db *sql.DB) error { if r == nil || db == nil { return nil } - baselines, err := loadSourceBaselines(ctx, db) - if err != nil { - return err - } events, err := loadSourceMetricEvents(ctx, db) if err != nil { return err @@ -114,7 +69,6 @@ func (r *Registry) AttachSourceMetricPersistence(ctx context.Context, db *sql.DB r.mu.Unlock() return errors.New("source metric persistence already attached") } - r.sourceBaselines = baselines for _, event := range events { ring := r.sourceMetricEvents[event.SourceID] if ring == nil { @@ -129,27 +83,6 @@ func (r *Registry) AttachSourceMetricPersistence(ctx context.Context, db *sql.DB return nil } -func loadSourceBaselines(ctx context.Context, db *sql.DB) (map[sourceBaselineKey]SourceTrafficBaseline, error) { - rows, err := db.QueryContext(ctx, `SELECT doc FROM source_metric_baselines`) - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - out := make(map[sourceBaselineKey]SourceTrafficBaseline) - for rows.Next() { - var doc string - if err := rows.Scan(&doc); err != nil { - return nil, err - } - var baseline SourceTrafficBaseline - if err := json.Unmarshal([]byte(doc), &baseline); err != nil { - return nil, err - } - out[sourceBaselineKey{baseline.SourceID, baseline.Identity, baseline.PGN}] = baseline - } - return out, rows.Err() -} - func loadSourceMetricEvents(ctx context.Context, db *sql.DB) ([]SourceMetricEvent, error) { rows, err := db.QueryContext(ctx, `SELECT id, doc FROM ( SELECT id, doc FROM source_metric_events ORDER BY id DESC LIMIT ? @@ -252,190 +185,6 @@ func (r *Registry) SourceMetricEvents(source string, limit int) []SourceMetricEv return out } -func sourceBaselineIdentity(deviceNameHex string, address uint8) string { - if deviceNameHex != "" { - return "name:" + deviceNameHex - } - return fmt.Sprintf("address:%d", address) -} - -func baselineFromMetric(metric SourcePGNMetric, approvedAt time.Time) SourceTrafficBaseline { - lengths := make([]int, 0) - if metric.Raw != nil { - for length := range metric.Raw.LengthCounts { - parsed, err := strconv.Atoi(length) - if err == nil { - lengths = append(lengths, parsed) - } - } - } - if len(lengths) == 0 { - for value := metric.PayloadBytesMin; value <= metric.PayloadBytesMax && value-metric.PayloadBytesMin < 256; value++ { - lengths = append(lengths, int(value)) - } - } - sort.Ints(lengths) - destinations := sortedIntKeys(metric.DestinationCounts) - priorities := sortedIntKeys(metric.PriorityCounts) - fields := make(map[string]BaselineField) - for _, field := range metric.Fields { - if field.P05 == nil || field.P95 == nil { - continue - } - span := *field.P95 - *field.P05 - margin := mathMax(span*0.25, mathMax(mathAbs(*field.P50)*0.02, 1e-9)) - fields[field.Field] = BaselineField{Minimum: *field.P05 - margin, Maximum: *field.P95 + margin, Unit: field.Unit} - } - rawBytes := make([]BaselineRawByte, 0) - if metric.Raw != nil { - for _, rawByte := range metric.Raw.Bytes { - rawBytes = append(rawBytes, BaselineRawByte{ - Offset: rawByte.Offset, Minimum: rawByte.Minimum, Maximum: rawByte.Maximum, - ChangedBitMaskHex: rawByte.ChangedBitMaskHex, - }) - } - } - return SourceTrafficBaseline{ - SourceID: metric.SourceID, Identity: sourceBaselineIdentity(metric.DeviceNameHex, metric.SourceAddress), - PGN: metric.PGN, PGNName: metric.PGNName, SourceAddress: metric.SourceAddress, - DeviceNameHex: metric.DeviceNameHex, ExpectedFrequencyHz: metric.FrequencyHz, - FrequencyTolerancePercent: 25, PayloadLengths: lengths, DecodeStatus: metric.DecodeStatus, - Variant: metric.Variant, Transport: metric.Transport, Destinations: destinations, - Priorities: priorities, Fields: fields, RawBytes: rawBytes, ApprovedAt: approvedAt, - } -} - -func sortedIntKeys(values map[string]int64) []int { - out := make([]int, 0, len(values)) - for value := range values { - parsed, err := strconv.ParseUint(value, 10, 8) - if err == nil { - out = append(out, int(parsed)) - } - } - sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) - return out -} - -func mathAbs(value float64) float64 { - if value < 0 { - return -value - } - return value -} - -func mathMax(left, right float64) float64 { - if left > right { - return left - } - return right -} - -func (r *Registry) CommitSourceTrafficBaseline(ctx context.Context, source string) ([]SourceTrafficBaseline, error) { - if r == nil { - return nil, errors.New("source metrics unavailable") - } - metrics := r.SourcePGNMetrics(source) - approvedAt := r.now().UTC() - baselines := make([]SourceTrafficBaseline, 0, len(metrics)) - for _, metric := range metrics { - if metric.Observed { - baselines = append(baselines, baselineFromMetric(metric, approvedAt)) - } - } - r.mu.Lock() - persistence := r.sourcePersistence - r.mu.Unlock() - if persistence == nil { - return nil, errors.New("source metric persistence unavailable") - } - tx, err := persistence.db.BeginTx(ctx, nil) - if err != nil { - return nil, err - } - defer func() { _ = tx.Rollback() }() - if _, err := tx.ExecContext(ctx, `DELETE FROM source_metric_baselines WHERE source_id = ?`, source); err != nil { - return nil, err - } - for _, baseline := range baselines { - doc, err := json.Marshal(baseline) - if err != nil { - return nil, err - } - if _, err := tx.ExecContext(ctx, `INSERT INTO source_metric_baselines - (source_id, identity, pgn, approved_at, doc) VALUES (?, ?, ?, ?, ?)`, - baseline.SourceID, baseline.Identity, baseline.PGN, baseline.ApprovedAt.UnixNano(), string(doc)); err != nil { - return nil, err - } - } - if err := tx.Commit(); err != nil { - return nil, err - } - r.mu.Lock() - for key := range r.sourceBaselines { - if key.source == source { - delete(r.sourceBaselines, key) - } - } - for _, baseline := range baselines { - r.sourceBaselines[sourceBaselineKey{baseline.SourceID, baseline.Identity, baseline.PGN}] = baseline - } - r.mu.Unlock() - r.recordSourceMetricEvent(SourceMetricEvent{Time: approvedAt, SourceID: source, - Kind: "baseline_committed", Severity: "info", - Summary: fmt.Sprintf("Set expected traffic baseline from %d observed streams", len(baselines))}) - return baselines, nil -} - -func (r *Registry) ClearSourceTrafficBaseline(ctx context.Context, source string) error { - if r == nil { - return nil - } - r.mu.Lock() - persistence := r.sourcePersistence - r.mu.Unlock() - if persistence == nil { - return errors.New("source metric persistence unavailable") - } - if _, err := persistence.db.ExecContext(ctx, `DELETE FROM source_metric_baselines WHERE source_id = ?`, source); err != nil { - return err - } - r.mu.Lock() - for key := range r.sourceBaselines { - if key.source == source { - delete(r.sourceBaselines, key) - } - } - r.mu.Unlock() - r.recordSourceMetricEvent(SourceMetricEvent{Time: r.now().UTC(), SourceID: source, - Kind: "baseline_cleared", Severity: "info", Summary: "Cleared source traffic baseline"}) - return nil -} - -func (r *Registry) SourceTrafficBaselines(source string) []SourceTrafficBaseline { - if r == nil { - return []SourceTrafficBaseline{} - } - r.mu.Lock() - out := make([]SourceTrafficBaseline, 0) - for key, baseline := range r.sourceBaselines { - if source == "" || key.source == source { - out = append(out, baseline) - } - } - r.mu.Unlock() - sort.Slice(out, func(i, j int) bool { - if out[i].SourceID != out[j].SourceID { - return out[i].SourceID < out[j].SourceID - } - if out[i].PGN != out[j].PGN { - return out[i].PGN < out[j].PGN - } - return out[i].Identity < out[j].Identity - }) - return out -} - func (r *Registry) CloseSourceMetricPersistence(ctx context.Context) error { if r == nil { return nil diff --git a/internal/stats/stats.go b/internal/stats/stats.go index adee033..954517a 100644 --- a/internal/stats/stats.go +++ b/internal/stats/stats.go @@ -363,11 +363,12 @@ type Registry struct { sink map[string]*counters events map[string]*eventRing sourceStreams map[sourceStreamKey]*sourceStream - sourceBaselines map[sourceBaselineKey]SourceTrafficBaseline sourceMetricEvents map[string]*sourceMetricEventRing sourcePersistence *sourceMetricPersistence sourceAddressNames map[sourceAddressKey]uint64 sourceDeviceAddresses map[sourceDeviceKey]uint8 + streamSubscribers map[string]map[uint64]chan []byte + nextStreamSubscriber uint64 } // NewRegistry returns an empty Registry using the real wall clock. @@ -386,10 +387,10 @@ func newRegistryAt(now func() time.Time) *Registry { sink: map[string]*counters{}, events: map[string]*eventRing{}, sourceStreams: map[sourceStreamKey]*sourceStream{}, - sourceBaselines: map[sourceBaselineKey]SourceTrafficBaseline{}, sourceMetricEvents: map[string]*sourceMetricEventRing{}, sourceAddressNames: map[sourceAddressKey]uint64{}, sourceDeviceAddresses: map[sourceDeviceKey]uint8{}, + streamSubscribers: map[string]map[uint64]chan []byte{}, } } @@ -470,6 +471,7 @@ func (r *Registry) RecordSource(source string, e *msg.Envelope) { r.recordSourceMetricEvent(event) } r.recordEvent("source", source, ev) + r.publishStream("source", source, e) } func (r *Registry) RecordSink(sink, connector string, e *msg.Envelope) { @@ -480,6 +482,7 @@ func (r *Registry) RecordSink(sink, connector string, e *msg.Envelope) { ev := eventFromEnvelope(now, "sent", connector, e) r.getSink(sink).record(now, 1, int64(ev.SizeBytes)) r.recordEvent("sink", sink, ev) + r.publishStream("sink", sink, e) } func (r *Registry) RecordConnectorEvent(connector, stage string, e *msg.Envelope) { @@ -637,6 +640,7 @@ func (r *Registry) RemoveSource(source string) { r.mu.Lock() delete(r.source, source) delete(r.events, "source:"+source) + r.closeStreamLocked("source", source) for key := range r.sourceStreams { if key.source == source { delete(r.sourceStreams, key) @@ -652,6 +656,7 @@ func (r *Registry) RemoveSink(sink string) { r.mu.Lock() delete(r.sink, sink) delete(r.events, "sink:"+sink) + r.closeStreamLocked("sink", sink) r.mu.Unlock() } diff --git a/internal/stats/stats_test.go b/internal/stats/stats_test.go index b2ccae8..d9bf5d2 100644 --- a/internal/stats/stats_test.go +++ b/internal/stats/stats_test.go @@ -1,6 +1,7 @@ package stats import ( + "encoding/json" "testing" "time" @@ -51,6 +52,62 @@ func TestRemoveDropsComponentCountersAndEvents(t *testing.T) { } } +func TestStreamSubscriptionsReceiveCanonicalFutureEnvelopesByBoundary(t *testing.T) { + r := NewRegistry() + sourceStream, stopSource := r.SubscribeStream("source", "in", 2) + defer stopSource() + sinkStream, stopSink := r.SubscribeStream("sink", "out", 2) + defer stopSink() + + envelope := &msg.Envelope{ + PGN: 130314, + Source: 6, + Dest: 255, + Priority: 5, + Payload: json.RawMessage( + `{"info":{"timestamp":"2026-06-29T10:10:17.530566931-04:00","priority":5,"pgn":130314,"sourceId":6,"targetId":null},"instance":0,"source":0,"pressure":1020690}`, + ), + Raw: []byte{0xff, 0x00, 0x00, 0x12}, + } + + r.RecordSource("in", envelope) + sourceDocument := <-sourceStream + select { + case unexpected := <-sinkStream: + t.Fatalf("source event crossed into sink stream: %s", unexpected) + default: + } + + var wire struct { + Payload json.RawMessage `json:"payload"` + Metadata json.RawMessage `json:"metadata"` + Raw []byte `json:"raw"` + } + if err := json.Unmarshal(sourceDocument, &wire); err != nil { + t.Fatal(err) + } + if string(wire.Payload) != string(envelope.Payload) { + t.Fatalf("stream payload = %s, want verbatim %s", wire.Payload, envelope.Payload) + } + if string(wire.Raw) != string(envelope.Raw) { + t.Fatalf("stream raw = %v, want %v", wire.Raw, envelope.Raw) + } + if len(wire.Metadata) == 0 { + t.Fatal("stream envelope omitted metadata") + } + + r.RecordSink("out", "route", envelope) + if sinkDocument := <-sinkStream; len(sinkDocument) == 0 { + t.Fatal("sink stream received an empty document") + } + + // Removing an entity closes its active preview streams. + r.RemoveSink("out") + if _, ok := <-sinkStream; ok { + t.Fatal("sink stream remained open after entity removal") + } +} + func TestTotalsAndRates(t *testing.T) { r := NewRegistry() for i := 0; i < 10; i++ { diff --git a/internal/stats/stream.go b/internal/stats/stream.go new file mode 100644 index 0000000..af4ca98 --- /dev/null +++ b/internal/stats/stream.go @@ -0,0 +1,96 @@ +package stats + +import ( + "encoding/json" + "sync" + + "github.com/open-ships/beacon/internal/msg" +) + +// SubscribeStream subscribes to future source-received or sink-sent +// envelopes. It is an observability tap: publishers never block and drop a +// preview copy when a subscriber cannot keep up. Each message is the +// canonical three-key consumer envelope JSON. +func (r *Registry) SubscribeStream(kind, id string, buffer int) (<-chan []byte, func()) { + if buffer < 1 { + buffer = 1 + } + if r == nil || id == "" || (kind != "source" && kind != "sink") { + ch := make(chan []byte) + close(ch) + return ch, func() {} + } + + key := streamKey(kind, id) + r.mu.Lock() + subscriberID := r.nextStreamSubscriber + r.nextStreamSubscriber++ + ch := make(chan []byte, buffer) + subscribers := r.streamSubscribers[key] + if subscribers == nil { + subscribers = map[uint64]chan []byte{} + r.streamSubscribers[key] = subscribers + } + subscribers[subscriberID] = ch + r.mu.Unlock() + + var once sync.Once + return ch, func() { + once.Do(func() { + r.mu.Lock() + subscribers := r.streamSubscribers[key] + if current, ok := subscribers[subscriberID]; ok { + delete(subscribers, subscriberID) + close(current) + } + if len(subscribers) == 0 { + delete(r.streamSubscribers, key) + } + r.mu.Unlock() + }) + } +} + +func (r *Registry) publishStream(kind, id string, envelope *msg.Envelope) { + if r == nil || envelope == nil { + return + } + key := streamKey(kind, id) + + // Avoid serialization on the normal data path when nobody is watching. + r.mu.Lock() + hasSubscribers := len(r.streamSubscribers[key]) != 0 + r.mu.Unlock() + if !hasSubscribers { + return + } + + document, err := json.Marshal(envelope) + if err != nil { + return + } + + r.mu.Lock() + for _, subscriber := range r.streamSubscribers[key] { + copyOfDocument := append([]byte(nil), document...) + select { + case subscriber <- copyOfDocument: + default: + // A UI preview must never apply backpressure to vessel traffic. + } + } + r.mu.Unlock() +} + +func (r *Registry) closeStreamLocked(kind, id string) { + key := streamKey(kind, id) + for subscriberID, subscriber := range r.streamSubscribers[key] { + delete(r.streamSubscribers[key], subscriberID) + close(subscriber) + } + delete(r.streamSubscribers, key) +} + +func streamKey(kind, id string) string { + return kind + ":" + id +} diff --git a/internal/store/store.go b/internal/store/store.go index 6b1e627..f0dbe97 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -68,6 +68,7 @@ var migrations = []string{ doc TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS source_metric_events_source_ts ON source_metric_events(source_id, ts DESC);`, + `DROP TABLE IF EXISTS source_metric_baselines;`, } func Open(path string) (*Store, error) { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 8facace..de5124f 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "database/sql" "encoding/json" "path/filepath" "testing" @@ -151,3 +152,47 @@ func TestReopenKeepsData(t *testing.T) { t.Fatal("data lost across reopen") } } + +func TestSourceTrafficBaselineTableIsRemovedOnUpgrade(t *testing.T) { + path := filepath.Join(t.TempDir(), "pre-removal.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY)`); err != nil { + t.Fatal(err) + } + for i, migration := range migrations[:3] { + if _, err := db.Exec(migration); err != nil { + t.Fatalf("migration %d setup: %v", i+1, err) + } + if _, err := db.Exec(`INSERT INTO schema_migrations(version) VALUES (?)`, i+1); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec(` + INSERT INTO source_metric_baselines (source_id, identity, pgn, approved_at, doc) + VALUES ('source-1', 'address:10', 127250, 1, '{}') + `); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = s.Close() }() + + var count int + if err := s.DB().QueryRow( + `SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'source_metric_baselines'`, + ).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatalf("source_metric_baselines table count = %d, want 0", count) + } +} diff --git a/internal/ui/assets/README.md b/internal/ui/assets/README.md index 6b06d3a..935ce3d 100644 --- a/internal/ui/assets/README.md +++ b/internal/ui/assets/README.md @@ -1,7 +1,7 @@ # Vendored assets Everything in this directory is served by `internal/ui.Handler` under -`/ui/assets/` via `go:embed`, so beacon's web UI (`/ui/`) is fully +`/assets/` via `go:embed`, so beacon's web UI (`/`) is fully self-contained and works with no network access beyond the browser talking to beacon itself. @@ -27,11 +27,14 @@ full page reloads and handles the fragment swaps. Hand-authored progressive enhancement for the connector form's CEL filter textarea. It supplies accessible, keyboard-driven typeahead for envelope fields, CEL helpers, PGN numbers, and decoded payload keys. The schema-backed -items are fetched from `/ui/cel-completions`; a small built-in fallback keeps +items are fetched from `/cel-completions`; a small built-in fallback keeps the core interaction useful if that request fails. It also debounces CEL compilation while the operator types and overlays compiler error ranges as red wavy underlines. No configuration write depends on this script, and filters -are still validated server-side on Save. +are still validated server-side on Save. The same file progressively enhances +source and sink overview pages with their stopped-by-default stream inspector, +live CEL filtering and field suggestions, bounded browser-local capture, +JSON/CAN view switching, clipboard copy, and export. ## app.css diff --git a/internal/ui/assets/app.css b/internal/ui/assets/app.css index 3a74761..f0821b3 100644 --- a/internal/ui/assets/app.css +++ b/internal/ui/assets/app.css @@ -540,9 +540,12 @@ code { min-width: 0; } +/* Status and metrics share one full-width card, so this is a single column. + It stays a grid (rather than collapsing to a block) to keep the row gap if + a second card is ever added back alongside it. */ .overview-grid { display: grid; - grid-template-columns: minmax(0, 0.9fr) minmax(0, 1.1fr); + grid-template-columns: minmax(0, 1fr); gap: var(--space-5); } @@ -557,6 +560,238 @@ code { padding: var(--space-4); } +.stream-panel { + gap: var(--space-4); +} + +.stream-panel-heading { + align-items: flex-start; +} + +.stream-panel-heading > div:first-child { + display: grid; + flex: 1 1 0; + gap: var(--space-1); + min-width: 0; +} + +.stream-panel-heading p { + max-width: 58rem; + margin: 0; + color: #666; + font-size: 0.84rem; +} + +.stream-primary-controls, +.stream-export-controls, +.stream-view-switch { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: var(--space-2); +} + +.stream-primary-controls { + align-items: flex-start; + flex: 0 0 auto; + flex-wrap: nowrap; + justify-content: flex-end; +} + +.stream-primary-controls > [hidden] { + display: none; +} + +.stream-filter-control { + display: grid; + flex: 0 1 28rem; + gap: var(--space-1); + width: clamp(16rem, 28vw, 28rem); +} + +.stream-filter-input { + width: 100%; + min-height: 2rem; + border: 1px solid var(--border-strong); + border-radius: 6px; + background: var(--surface); + color: var(--gray-dark); + font-family: var(--font-mono); + font-size: 0.78rem; + line-height: 1; + padding: 0.42rem 0.66rem; +} + +.stream-filter-input:focus { + border-color: var(--accent); + outline: 3px solid rgba(35, 55, 255, 0.12); +} + +.stream-filter-input[aria-invalid="true"] { + border-color: var(--error); +} + +.stream-filter-input[aria-invalid="true"]:focus { + outline-color: rgba(180, 35, 24, 0.14); +} + +.stream-filter-input:disabled { + background: var(--surface-muted); + color: var(--gray); +} + +.stream-filter-feedback { + color: var(--error); + font-size: 0.75rem; + font-weight: 700; + line-height: 1.25; +} + +.stream-filter-feedback[hidden] { + display: none; +} + +.stream-copy-feedback { + align-self: center; + color: var(--success); + font-size: 0.75rem; + font-weight: 750; +} + +.stream-toolbar { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: var(--space-3); + border-block: 1px solid var(--border); + padding-block: var(--space-3); +} + +.stream-status { + margin-left: auto; + color: #555; + font-size: 0.8rem; + font-weight: 750; +} + +.stream-cel-inspector { + display: flex; + align-items: center; + justify-content: space-between; + flex-wrap: wrap; + gap: var(--space-3); + border: 1px solid var(--border-strong); + border-radius: var(--radius); + background: var(--surface-muted); + padding: var(--space-2) var(--space-3); +} + +.stream-cel-inspector[hidden] { + display: none; +} + +.stream-cel-inspector-copy { + display: grid; + gap: var(--space-1); + min-width: 0; +} + +.stream-cel-inspector-label { + color: #555; + font-size: 0.75rem; + font-weight: 750; +} + +.stream-cel-expression { + display: block; + max-width: min(60rem, calc(100vw - 4rem)); + overflow-x: auto; + border: 1px solid var(--border); + background: #fff; + color: #111827; + font-size: 0.78rem; + padding: 0.32rem 0.5rem; + white-space: nowrap; +} + +.stream-cel-inspector-actions { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: var(--space-2); +} + +.stream-empty { + display: grid; + justify-items: center; + gap: var(--space-1); + border: 1px dashed var(--border-strong); + border-radius: var(--radius); + background: var(--surface-muted); + color: #666; + padding: var(--space-8) var(--space-4); + text-align: center; +} + +.stream-empty[hidden], +.stream-list[hidden] { + display: none; +} + +.stream-empty strong { + color: var(--black); +} + +.stream-list { + display: flex; + flex-direction: column; + gap: 0; + max-height: 34rem; + overflow: auto; + overscroll-behavior: contain; + border: 1px solid #64748b; + border-radius: var(--radius); + background: #fff; +} + +.stream-message { + display: block; + flex: 0 0 auto; + min-width: 0; + overflow: visible; + background: transparent; +} + +code.stream-message-content { + display: block; + box-sizing: border-box; + width: max-content; + min-width: 100%; + margin: 0; + border-radius: 0; + background: #fff; + color: #111827; + padding: 0.3rem 0.75rem; + font-size: 0.78rem; + line-height: 1.3; + white-space: pre; +} + +.stream-message-content[data-stream-json] { + cursor: pointer; +} + +.stream-json-cel-target { + border-radius: 2px; + transition: background-color 100ms ease, color 100ms ease; +} + +.stream-message-content[data-stream-json]:hover .stream-json-cel-target:hover { + background: #e8ecff; + color: var(--accent-dark); + outline: 1px solid #9da8ff; +} + .config-list { display: grid; grid-template-columns: minmax(120px, 18%) minmax(0, 1fr); @@ -618,9 +853,14 @@ code { border-left-color: var(--warning); } +/* Full-width now that it shares the status card, and the tile count varies by + kind: two for a source, up to eight for a connector. auto-fit packs the + connector's row, while the capped track keeps a source's pair from + stretching to half the page each. */ .overview-stats { display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); + grid-template-columns: repeat(auto-fit, minmax(135px, 160px)); + justify-content: start; gap: var(--space-2); } @@ -680,12 +920,379 @@ code { vertical-align: top; } -.source-stream-gap td { - background: rgba(180, 35, 24, 0.055); +.source-device-table { + width: 100%; + min-width: 1160px; + table-layout: fixed; +} + +.source-device-table th, +.source-device-table td { + vertical-align: top; +} + +.source-device-table td { + overflow-wrap: anywhere; +} + +.source-device-table .source-device-col-address { + width: 5rem; +} + +.source-device-table .source-device-col-name { + width: 13rem; +} + +.source-device-table .source-device-col-manufacturer { + width: 12rem; +} + +.source-device-table .source-device-col-role { + width: 12rem; +} + +.source-device-table .source-device-col-rate { + width: 7rem; +} + +.source-device-table .source-device-col-share { + width: 9rem; +} + +.source-device-table .source-device-col-last-seen { + width: 7rem; +} + +.source-device-address, +.source-device-stat, +.source-device-table time { + font-variant-numeric: tabular-nums; + white-space: nowrap; +} + +.source-device-sort-button { + display: inline-flex; + align-items: center; + justify-content: flex-start; + gap: 0.35rem; + width: 100%; + border: 0; + background: transparent; + color: inherit; + cursor: pointer; + font: inherit; + padding: 0; + text-align: left; + text-transform: inherit; +} + +.source-device-sort-button:hover, +.source-device-sort-button:focus-visible { + color: var(--black); } -.source-stream-anomaly td { - background: rgba(166, 95, 0, 0.06); +.source-device-sort-button:focus-visible { + border-radius: 3px; + outline: 2px solid var(--accent); + outline-offset: 3px; +} + +.source-device-sort-indicator { + min-width: 1em; + font-size: 1rem; + font-weight: 500; + line-height: 1; + opacity: 0.72; + text-align: center; +} + +.source-device-browser { + display: grid; + gap: var(--space-4); + min-width: 0; +} + +.source-device-row { + cursor: pointer; +} + +.source-device-row:hover td, +.source-device-row:focus td { + background: var(--surface-muted); +} + +.source-device-row:focus { + outline: 3px solid rgba(35, 55, 255, 0.22); + outline-offset: -3px; +} + +.source-device-row-selected td { + background: rgba(35, 55, 255, 0.065); +} + +.source-device-row-selected td:first-child { + box-shadow: inset 4px 0 0 var(--accent); +} + +/* Sized to the table's content (~1120px) rather than the viewport: a + full-width dialog strands each column header far from its own data and + leaves dead space below short PGN lists. Height follows the content and + only scrolls once a device has more PGNs than the viewport can hold. */ +.source-device-detail-panel { + width: min(1120px, calc(100vw - 2rem)); + /* fit-content, not auto: the UA stylesheet pins dialogs with both block + insets at 0, so height:auto stretches to the full viewport instead of + wrapping the content. */ + height: fit-content; + max-width: none; + max-height: calc(100vh - 3rem); + margin: auto; + overflow-x: hidden; + overflow-y: auto; + overscroll-behavior: contain; + scrollbar-gutter: stable; + border: 1px solid var(--border-strong); + border-radius: var(--radius); + background: var(--surface-muted); + box-shadow: 0 24px 80px rgba(16, 19, 29, 0.34); + padding: 0; +} + +.source-device-detail-panel::backdrop { + background: rgba(16, 19, 29, 0.56); + backdrop-filter: blur(2px); +} + +.source-device-dialog-surface { + display: grid; + grid-template-rows: auto auto auto; + gap: var(--space-4); + min-width: 0; + overflow: visible; + padding: var(--space-4); +} + +.source-device-detail-header { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: var(--space-4); + align-items: start; +} + +.source-device-detail-heading { + display: grid; + gap: 0.25rem; +} + +.source-device-detail-heading h4, +.source-device-detail-heading p, +.source-device-pgn-heading h4, +.source-device-pgn-heading p { + margin: 0; +} + +.source-device-detail-heading h4, +.source-device-pgn-heading h4 { + color: var(--black); + font-size: 1.15rem; +} + +.source-device-detail-heading p:not(.eyebrow), +.source-device-pgn-heading p { + color: #666; + font-size: 0.84rem; +} + +.source-device-detail-metadata { + display: grid; + grid-column: 1 / -1; + grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); + gap: var(--space-2); + margin: 0; +} + +.source-device-detail-metadata > div { + min-width: 0; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--surface); + padding: var(--space-2) var(--space-3); +} + +.source-device-detail-metadata dt { + color: #666; + font-size: 0.7rem; + font-weight: 800; + text-transform: uppercase; +} + +.source-device-detail-metadata dd { + min-width: 0; + margin: 0.2rem 0 0; + overflow-wrap: anywhere; +} + +.source-device-pgn-heading { + align-items: start; +} + +/* The generic .section-heading cap wraps this caption to two lines inside the + dialog, which leaves the PGN count badge floating below the title. */ +.source-device-pgn-heading p { + max-width: none; +} + +.source-device-pgn-heading > div { + display: grid; + gap: 0.25rem; +} + +.source-device-pgn-table { + width: 100%; + min-width: 0; + table-layout: fixed; + box-shadow: none; +} + +/* Balanced against the dialog's 1120px content width: the PGN column carries + wrapping proprietary-range names, timing holds a 3-up percentile grid, and + activity only needs a dot plus a relative timestamp. */ +.source-device-pgn-table th:nth-child(1) { + width: 25%; +} + +.source-device-pgn-table th:nth-child(2) { + width: 12%; +} + +.source-device-pgn-table th:nth-child(3) { + width: 14%; +} + +.source-device-pgn-table th:nth-child(4) { + width: 22%; +} + +.source-device-pgn-table th:nth-child(5) { + width: 16%; +} + +.source-device-pgn-table th:nth-child(6) { + width: 11%; +} + +.source-device-pgn-table th, +.source-device-pgn-table td { + vertical-align: top; + overflow-wrap: anywhere; + padding: clamp(0.48rem, 0.9vh, 0.78rem) clamp(0.52rem, 0.75vw, 0.82rem); + font-size: clamp(0.84rem, 0.84vw, 1rem); + line-height: 1.3; +} + +.source-device-pgn-table th:first-child { + font-size: clamp(0.9rem, 0.92vw, 1.08rem); +} + +.source-device-pgn-table td.source-device-pgn-identity { + font-size: clamp(1rem, 0.98vw, 1.15rem); + line-height: 1.32; +} + +.source-device-pgn-identity .badge { + font-size: 0.85em; +} + +.source-device-pgn-link { + color: var(--accent); + font-weight: 800; + text-decoration: none; +} + +.source-device-pgn-link code { + color: inherit; + font-size: 1.12em; +} + +.source-device-pgn-link:hover, +.source-device-pgn-link:focus-visible { + color: var(--accent-dark); + text-decoration: underline; +} + +.source-device-pgn-viewport { + min-width: 0; + min-height: 0; + overflow-x: clip; + overflow-y: visible; +} + +.source-device-pgn-table thead th { + position: sticky; + top: 0; + z-index: 1; +} + +.source-device-pgn-timing { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 0.35rem; +} + +.source-device-pgn-timing > span { + display: grid; + gap: 0.12rem; + min-width: 0; +} + +.source-device-pgn-timing small { + color: #666; + font-size: 0.68em; + font-weight: 800; + text-transform: uppercase; +} + +.source-device-pgn-timing strong { + font-size: 0.95em; + white-space: nowrap; +} + +/* Recency dot, graded against each PGN's own p90 period. Shares the dot + language used by .usage-dot/.connector-dot; the aria-label/title carries the + same state in words so color is never the sole signal. */ +.source-device-pgn-activity { + display: inline-flex; + align-items: center; + gap: 0.45rem; + white-space: nowrap; +} + +.source-device-pgn-dot { + flex: none; + width: 0.62rem; + height: 0.62rem; + border-radius: 999px; + background: var(--gray); + box-shadow: 0 0 0 3px var(--surface-muted); +} + +.source-device-pgn-dot-fresh { + background: var(--success); + box-shadow: 0 0 0 3px var(--success-bg); +} + +.source-device-pgn-dot-late { + background: var(--warning); + box-shadow: 0 0 0 3px var(--warning-bg); +} + +.source-device-pgn-dot-stale { + background: var(--error); + box-shadow: 0 0 0 3px var(--error-bg); +} + +.source-stream-gap td { + background: rgba(180, 35, 24, 0.055); } .source-stream-changed td, @@ -702,18 +1309,6 @@ code { font-weight: 800; } -.source-anomaly-note { - display: block; - max-width: 11rem; - margin-top: 0.35rem; - overflow: hidden; - color: var(--warning); - font-size: 0.75rem; - font-weight: 750; - text-overflow: ellipsis; - white-space: nowrap; -} - .source-baseline-state { display: block; margin-top: 0.32rem; @@ -804,10 +1399,6 @@ code { line-height: 1.35; } -.source-value-anomaly { - border-left-color: var(--warning); -} - .event-table td:last-child code { white-space: normal; } @@ -1033,7 +1624,15 @@ code { } .overview-card.component-status-surface { - box-shadow: inset 4px 0 0 var(--component-state-accent), var(--shadow); + border-color: var(--border); + background: var(--surface); + box-shadow: var(--shadow); +} + +.overview-card .status-line .badge { + min-height: 1.75rem; + font-size: 0.95rem; + padding: 0.3rem 0.65rem; } .component-status-row { @@ -1882,10 +2481,6 @@ code { flex-direction: column; } - .overview-grid { - grid-template-columns: 1fr; - } - .mcp-install-grid, .mcp-call-grid { grid-template-columns: 1fr; @@ -1895,6 +2490,22 @@ code { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .stream-status { + width: 100%; + margin-left: 0; + } + + .stream-primary-controls { + flex-wrap: wrap; + justify-content: flex-start; + width: 100%; + } + + .stream-filter-control { + flex: 1 1 20rem; + width: auto; + } + .config-list { grid-template-columns: 1fr; } diff --git a/internal/ui/assets/app.js b/internal/ui/assets/app.js index 1b26180..30384c2 100644 --- a/internal/ui/assets/app.js +++ b/internal/ui/assets/app.js @@ -49,7 +49,7 @@ function loadCatalog() { if (catalogPromise) return catalogPromise; - catalogPromise = fetch("/ui/cel-completions", { + catalogPromise = fetch("/cel-completions", { headers: { Accept: "application/json" } }) .then(function (response) { @@ -352,7 +352,7 @@ validationRequest = new AbortController(); var body = new URLSearchParams(); body.set("filters", value); - fetch("/ui/frag/validate-filters", { + fetch("/frag/validate-filters", { method: "POST", headers: { Accept: "application/json", @@ -503,14 +503,894 @@ return 100; } + var activeSourceDeviceRow = null; + + function sourceDeviceRowFromTarget(target) { + return target && target.closest && target.closest("[data-source-device-detail-row]"); + } + + function reconcileSourceDeviceRows(responseText) { + var currentBody = document.getElementById("source-device-rows"); + if (!currentBody) return; + + var response = new DOMParser().parseFromString(responseText, "text/html"); + var snapshot = response.querySelector("[data-source-device-row-snapshot]"); + if (!snapshot) return; + + var nextRows = Array.prototype.slice.call( + snapshot.querySelectorAll("[data-source-device-detail-row]") + ); + var currentRows = Array.prototype.slice.call( + currentBody.querySelectorAll("[data-source-device-detail-row]") + ); + var currentByAddress = Object.create(null); + var nextAddresses = Object.create(null); + + currentRows.forEach(function (row) { + currentByAddress[row.dataset.deviceAddress] = row; + }); + + if (nextRows.length) { + var empty = currentBody.querySelector("[data-source-device-empty-row]"); + if (empty) empty.remove(); + } + + nextRows.forEach(function (nextRow, index) { + var address = nextRow.dataset.deviceAddress; + var currentRow = currentByAddress[address]; + nextAddresses[address] = true; + + if (currentRow) { + // Keep the row and its click handler stable. Replacing only its cells + // avoids losing a click between pointer-down and pointer-up. + if (currentRow !== activeSourceDeviceRow) { + var cells = Array.prototype.map.call(nextRow.children, function (cell) { + return cell.cloneNode(true); + }); + currentRow.replaceChildren.apply(currentRow, cells); + } + return; + } + + var inserted = nextRow.cloneNode(true); + var before = null; + for (var nextIndex = index + 1; nextIndex < nextRows.length; nextIndex += 1) { + before = currentByAddress[nextRows[nextIndex].dataset.deviceAddress]; + if (before) break; + } + currentBody.insertBefore(inserted, before); + currentByAddress[address] = inserted; + if (window.htmx && typeof window.htmx.process === "function") { + window.htmx.process(inserted); + } + }); + + currentRows.forEach(function (row) { + if (!nextAddresses[row.dataset.deviceAddress] && row !== activeSourceDeviceRow) { + row.remove(); + } + }); + + if (!nextRows.length && !currentBody.querySelector("[data-source-device-empty-row]")) { + var nextEmpty = snapshot.querySelector("[data-source-device-empty-row]"); + if (nextEmpty) currentBody.appendChild(nextEmpty.cloneNode(true)); + } + } + + function syncSourceDevicePGNSortControls(responseText) { + var response = new DOMParser().parseFromString(responseText, "text/html"); + var nextControls = response.querySelectorAll("[data-source-device-pgn-sort-control]"); + + Array.prototype.forEach.call(nextControls, function (nextControl) { + var key = nextControl.dataset.sourceDevicePgnSortControl; + var currentControl = document.querySelector( + '[data-source-device-pgn-sort-control="' + key + '"]' + ); + if (!currentControl) return; + + var currentHeading = currentControl.closest("th"); + var nextHeading = nextControl.closest("th"); + if (currentHeading && nextHeading) { + currentHeading.setAttribute("aria-sort", nextHeading.getAttribute("aria-sort") || "none"); + } + + var replacement = nextControl.cloneNode(true); + currentControl.replaceWith(replacement); + if (window.htmx && typeof window.htmx.process === "function") { + window.htmx.process(replacement); + } + }); + } + + var streamCaptureLimit = 200; + + function initStreamPanels(root) { + var scope = root && root.querySelectorAll ? root : document; + var panels = Array.prototype.slice.call(scope.querySelectorAll("[data-stream-panel]")); + if (scope.matches && scope.matches("[data-stream-panel]")) panels.unshift(scope); + panels.forEach(initStreamPanel); + } + + function initStreamPanel(panel) { + if (panel.dataset.streamReady === "true") return; + panel.dataset.streamReady = "true"; + + var startButton = panel.querySelector("[data-stream-start]"); + var stopButton = panel.querySelector("[data-stream-stop]"); + var clearButton = panel.querySelector("[data-stream-clear]"); + var copyButton = panel.querySelector("[data-stream-copy]"); + var copyFeedback = panel.querySelector("[data-stream-copy-feedback]"); + var filterInput = panel.querySelector("[data-stream-filter]"); + var filterFeedback = panel.querySelector("[data-stream-filter-feedback]"); + var celInspector = panel.querySelector("[data-stream-cel-inspector]"); + var celPath = panel.querySelector("[data-stream-cel-path]"); + var celExpression = panel.querySelector("[data-stream-cel-expression]"); + var celUseButton = panel.querySelector("[data-stream-cel-use]"); + var celCopyButton = panel.querySelector("[data-stream-cel-copy]"); + var celCloseButton = panel.querySelector("[data-stream-cel-close]"); + var status = panel.querySelector("[data-stream-status]"); + var empty = panel.querySelector("[data-stream-empty]"); + var emptyTitle = empty && empty.querySelector("strong"); + var emptyDetail = empty && empty.querySelector("span"); + var list = panel.querySelector("[data-stream-list]"); + var viewButtons = Array.prototype.slice.call(panel.querySelectorAll("[data-stream-view]")); + var exportButtons = Array.prototype.slice.call(panel.querySelectorAll("[data-stream-export]")); + if (!startButton || !stopButton || !clearButton || !status || !empty || !list) return; + + var entries = []; + var totalCaptured = 0; + var stream = null; + var starting = false; + var appliedFilter = ""; + var displayFormat = "json"; + var renderPending = false; + var filterTimer = null; + var validationSequence = 0; + var copyFeedbackTimer = null; + + function isStreaming() { + return stream !== null; + } + + function setStatus(label) { + var summary = totalCaptured.toLocaleString() + " captured"; + if (totalCaptured > entries.length) { + summary += " \u00b7 latest " + entries.length.toLocaleString() + " shown"; + } + status.textContent = label + " \u00b7 " + summary; + } + + function setFilterError(message) { + if (!filterInput || !filterFeedback) return; + filterInput.setAttribute("aria-invalid", message ? "true" : "false"); + filterFeedback.textContent = message || ""; + filterFeedback.hidden = !message; + } + + function updateControls() { + startButton.hidden = isStreaming(); + stopButton.hidden = !isStreaming(); + startButton.disabled = starting; + stopButton.disabled = false; + if (filterInput) filterInput.disabled = starting; + clearButton.disabled = entries.length === 0; + exportButtons.forEach(function (button) { + if (button.dataset.streamExport === "can") { + button.disabled = !entries.some(function (entry) { + return streamRawBytes(entry.raw).length > 0; + }); + } else { + button.disabled = entries.length === 0; + } + }); + if (copyButton) { + copyButton.disabled = entries.length === 0 || ( + displayFormat === "can" && !entries.some(function (entry) { + return streamRawBytes(entry.raw).length > 0; + }) + ); + } + } + + function updateEmpty() { + if (entries.length > 0) { + empty.hidden = true; + list.hidden = false; + return; + } + empty.hidden = false; + list.hidden = true; + if (isStreaming()) { + emptyTitle.textContent = "Waiting for messages\u2026"; + emptyDetail.textContent = "New " + panel.dataset.streamKind + " messages will appear here."; + } else { + emptyTitle.textContent = "Streaming is stopped."; + emptyDetail.textContent = "Choose Start to capture messages arriving at this " + panel.dataset.streamKind + "."; + } + } + + function queueRender() { + if (renderPending) return; + renderPending = true; + window.requestAnimationFrame(function () { + renderPending = false; + renderEntries(); + }); + } + + function renderEntries() { + var fragment = document.createDocumentFragment(); + entries.forEach(function (entry) { + fragment.appendChild(streamMessageElement(entry, displayFormat)); + }); + list.replaceChildren(fragment); + updateEmpty(); + updateControls(); + setStatus(isStreaming() ? "Streaming" : "Stopped"); + } + + function validateFilter(filterExpression) { + if (!filterExpression) { + return Promise.resolve({ valid: true, diagnostics: [] }); + } + var body = new URLSearchParams(); + body.set("filters", filterExpression); + return fetch("/frag/validate-filters", { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/x-www-form-urlencoded" + }, + body: body.toString(), + credentials: "same-origin" + }).then(function (response) { + if (!response.ok) throw new Error("validation request failed"); + return response.json(); + }); + } + + function validationError(result, suffix) { + var diagnostics = Array.isArray(result.diagnostics) ? result.diagnostics : []; + var message = diagnostics.length > 0 + ? "Invalid CEL: " + diagnostics[0].message + : "Invalid CEL filter."; + return suffix ? message + " " + suffix : message; + } + + function openStream(filterExpression) { + if (stream) stream.close(); + var streamURL = new URL(panel.dataset.streamUrl, window.location.href); + if (filterExpression) streamURL.searchParams.set("filter", filterExpression); + var nextStream = new EventSource(streamURL.toString()); + stream = nextStream; + appliedFilter = filterExpression; + starting = false; + setFilterError(""); + setStatus("Connecting"); + updateControls(); + updateEmpty(); + + nextStream.onopen = function () { + if (stream === nextStream) setStatus("Streaming"); + }; + nextStream.onmessage = function (event) { + if (stream !== nextStream) return; + var envelope; + try { + envelope = JSON.parse(event.data); + } catch (_) { + return; + } + if (!envelope || typeof envelope !== "object" || !("payload" in envelope)) return; + // Keep the exact Go-produced payload text alongside the parsed object. + // JSON.parse would round uint64/int64 values above JavaScript's safe + // integer range, corrupting both the inspector and a later export. + envelope.streamPayloadJSON = streamPayloadJSON(event.data) || JSON.stringify(envelope.payload); + totalCaptured += 1; + entries.unshift(envelope); + if (entries.length > streamCaptureLimit) entries.length = streamCaptureLimit; + queueRender(); + }; + nextStream.onerror = function () { + if (stream === nextStream) setStatus("Reconnecting"); + }; + } + + function start() { + if (stream || starting || !panel.dataset.streamUrl) return; + var filterExpression = filterInput ? filterInput.value.trim() : ""; + setFilterError(""); + if (!filterExpression) { + openStream(""); + return; + } + + starting = true; + var sequence = ++validationSequence; + setStatus("Checking filter"); + updateControls(); + validateFilter(filterExpression) + .then(function (result) { + if (!starting || sequence !== validationSequence) return; + if (!result.valid) { + starting = false; + setFilterError(validationError(result)); + setStatus("Filter error"); + updateControls(); + return; + } + openStream(filterExpression); + }) + .catch(function () { + if (!starting || sequence !== validationSequence) return; + starting = false; + setFilterError("CEL validation is unavailable. Try again."); + setStatus("Filter error"); + updateControls(); + }); + } + + function applyRunningFilter() { + window.clearTimeout(filterTimer); + filterTimer = null; + if (!stream || !filterInput) return; + var filterExpression = filterInput.value.trim(); + if (filterExpression === appliedFilter) { + setFilterError(""); + return; + } + + var sequence = ++validationSequence; + validateFilter(filterExpression) + .then(function (result) { + if (!stream || sequence !== validationSequence) return; + if (!result.valid) { + setFilterError(validationError(result, "The current stream filter is unchanged.")); + return; + } + openStream(filterExpression); + }) + .catch(function () { + if (!stream || sequence !== validationSequence) return; + setFilterError("CEL validation is unavailable. The current stream filter is unchanged."); + }); + } + + function scheduleRunningFilter() { + window.clearTimeout(filterTimer); + filterTimer = window.setTimeout(applyRunningFilter, 350); + } + + function stop() { + window.clearTimeout(filterTimer); + filterTimer = null; + validationSequence += 1; + if (stream) stream.close(); + stream = null; + starting = false; + updateControls(); + updateEmpty(); + setStatus("Stopped"); + } + + function clear() { + entries = []; + totalCaptured = 0; + list.replaceChildren(); + if (celInspector) celInspector.hidden = true; + if (copyFeedback) copyFeedback.textContent = ""; + updateControls(); + updateEmpty(); + setStatus(isStreaming() ? "Streaming" : "Stopped"); + } + + startButton.addEventListener("click", start); + stopButton.addEventListener("click", stop); + clearButton.addEventListener("click", clear); + if (filterInput) { + filterInput.addEventListener("input", function () { + setFilterError(""); + if (isStreaming()) scheduleRunningFilter(); + }); + filterInput.addEventListener("keydown", function (event) { + if (event.key !== "Enter") return; + event.preventDefault(); + if (isStreaming()) applyRunningFilter(); + else if (!startButton.disabled) start(); + }); + } + + list.addEventListener("click", function (event) { + var content = event.target.closest && event.target.closest(".stream-message-content[data-stream-json]"); + if (!content || !celInspector || !celPath || !celExpression) return; + var target = event.target.closest("[data-cel-expression]"); + celPath.textContent = target ? target.dataset.celPath : "msg.payload"; + celExpression.textContent = target ? target.dataset.celExpression : "has(msg.payload)"; + celInspector.hidden = false; + }); + + if (celUseButton && celExpression && filterInput) { + celUseButton.addEventListener("click", function () { + filterInput.value = celExpression.textContent; + setFilterError(""); + filterInput.focus(); + if (isStreaming()) applyRunningFilter(); + }); + } + + if (celCopyButton && celExpression) { + celCopyButton.addEventListener("click", function () { + writeClipboard(celExpression.textContent).then(function () { + if (copyFeedback) copyFeedback.textContent = "CEL copied."; + }).catch(function () { + if (copyFeedback) copyFeedback.textContent = "Copy failed."; + }); + }); + } + + if (celCloseButton && celInspector) { + celCloseButton.addEventListener("click", function () { + celInspector.hidden = true; + }); + } + + viewButtons.forEach(function (button) { + button.addEventListener("click", function () { + displayFormat = button.dataset.streamView; + viewButtons.forEach(function (candidate) { + var selected = candidate === button; + candidate.setAttribute("aria-pressed", selected ? "true" : "false"); + candidate.classList.toggle("btn-primary", selected); + candidate.classList.toggle("btn-ghost", !selected); + }); + if (displayFormat !== "json" && celInspector) celInspector.hidden = true; + queueRender(); + }); + }); + + exportButtons.forEach(function (button) { + button.addEventListener("click", function () { + exportStream(entries, button.dataset.streamExport, panel.dataset.streamKind, panel.dataset.streamId); + }); + }); + + if (copyButton) { + copyButton.addEventListener("click", function () { + var output = streamClipboardText(entries, displayFormat); + if (!output) return; + writeClipboard(output).then(function () { + window.clearTimeout(copyFeedbackTimer); + if (copyFeedback) { + var copiedLines = output.trimEnd().split("\n").length; + copyFeedback.textContent = "Copied " + copiedLines.toLocaleString() + " retained message" + + (copiedLines === 1 ? "." : "s."); + } + copyFeedbackTimer = window.setTimeout(function () { + if (copyFeedback) copyFeedback.textContent = ""; + }, 2500); + }).catch(function () { + if (copyFeedback) copyFeedback.textContent = "Copy failed."; + }); + }); + } + + panel.stopStreamCapture = stop; + updateControls(); + updateEmpty(); + } + + function streamMessageElement(envelope, format) { + var article = document.createElement("article"); + article.className = "stream-message"; + var payload = envelope.payload && typeof envelope.payload === "object" ? envelope.payload : {}; + var info = payload.info && typeof payload.info === "object" ? payload.info : {}; + var metadata = envelope.metadata && typeof envelope.metadata === "object" ? envelope.metadata : {}; + var identityParts = []; + if (info.pgn !== undefined && info.pgn !== null) identityParts.push("PGN " + info.pgn); + if (metadata.pgn_name) identityParts.push(metadata.pgn_name); + if (info.sourceId !== undefined && info.sourceId !== null) identityParts.push("source " + info.sourceId); + var bytes = streamRawBytes(envelope.raw); + article.title = identityParts.join(" \u00b7 ") || "N2K message"; + var code = document.createElement("code"); + code.className = "stream-message-content"; + if (format === "can") { + code.textContent = bytes.length ? streamBytesHex(bytes, true) : "No raw CAN payload available."; + } else { + code.dataset.streamJson = "true"; + code.title = "Click a JSON key or value to inspect its CEL filter"; + renderStreamJSON(code, envelope.streamPayloadJSON || JSON.stringify(envelope.payload)); + } + article.appendChild(code); + return article; + } + + function renderStreamJSON(code, text) { + var targets = streamJSONCELTargets(text); + if (targets.length === 0) { + code.textContent = text; + return; + } + var cursor = 0; + targets.forEach(function (target) { + if (target.start < cursor) return; + if (target.start > cursor) { + code.appendChild(document.createTextNode(text.slice(cursor, target.start))); + } + var span = document.createElement("span"); + span.className = "stream-json-cel-target"; + span.dataset.celPath = target.path; + span.dataset.celExpression = target.expression; + span.textContent = text.slice(target.start, target.end); + span.title = target.expression; + code.appendChild(span); + cursor = target.end; + }); + if (cursor < text.length) { + code.appendChild(document.createTextNode(text.slice(cursor))); + } + } + + function streamJSONCELTargets(text) { + var targets = []; + var cursor = 0; + + function skipWhitespace() { + while (cursor < text.length && /\s/.test(text[cursor])) cursor += 1; + } + + function readString() { + var start = cursor; + cursor += 1; + var escaped = false; + while (cursor < text.length) { + var character = text[cursor]; + cursor += 1; + if (escaped) { + escaped = false; + } else if (character === "\\") { + escaped = true; + } else if (character === '"') { + break; + } + } + var raw = text.slice(start, cursor); + return { start: start, end: cursor, raw: raw, value: JSON.parse(raw) }; + } + + function presenceExpression(path) { + return path.endsWith("]") ? path + " != null" : "has(" + path + ")"; + } + + function childPath(path, key) { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(key) + ? path + "." + key + : path + "[" + JSON.stringify(key) + "]"; + } + + function primitiveExpression(path, raw) { + if (/^-?\d+$/.test(raw) && typeof BigInt === "function") { + try { + var integer = BigInt(raw); + if (integer > BigInt("9223372036854775807") || integer < BigInt("-9223372036854775808")) { + return presenceExpression(path); + } + } catch (_) { + return presenceExpression(path); + } + } + return path + " == " + raw; + } + + function parseValue(path) { + skipWhitespace(); + var start = cursor; + var character = text[cursor]; + if (character === "{") { + cursor += 1; + skipWhitespace(); + while (cursor < text.length && text[cursor] !== "}") { + var key = readString(); + skipWhitespace(); + if (text[cursor] !== ":") throw new Error("invalid JSON object"); + cursor += 1; + var nextPath = childPath(path, key.value); + var value = parseValue(nextPath); + targets.push({ + start: key.start, + end: key.end, + path: nextPath, + expression: value.expression + }); + skipWhitespace(); + if (text[cursor] === ",") { + cursor += 1; + skipWhitespace(); + } else { + break; + } + } + if (text[cursor] !== "}") throw new Error("invalid JSON object"); + cursor += 1; + return { start: start, end: cursor, expression: presenceExpression(path) }; + } + if (character === "[") { + cursor += 1; + skipWhitespace(); + var arrayIndex = 0; + while (cursor < text.length && text[cursor] !== "]") { + parseValue(path + "[" + arrayIndex + "]"); + arrayIndex += 1; + skipWhitespace(); + if (text[cursor] === ",") { + cursor += 1; + skipWhitespace(); + } else { + break; + } + } + if (text[cursor] !== "]") throw new Error("invalid JSON array"); + cursor += 1; + return { start: start, end: cursor, expression: presenceExpression(path) }; + } + if (character === '"') { + var stringValue = readString(); + var stringExpression = path + " == " + stringValue.raw; + targets.push({ + start: stringValue.start, + end: stringValue.end, + path: path, + expression: stringExpression + }); + return { start: start, end: cursor, expression: stringExpression }; + } + + while (cursor < text.length && !/[\s,\]}]/.test(text[cursor])) cursor += 1; + if (cursor === start) throw new Error("invalid JSON value"); + var raw = text.slice(start, cursor); + var expression = primitiveExpression(path, raw); + targets.push({ start: start, end: cursor, path: path, expression: expression }); + return { start: start, end: cursor, expression: expression }; + } + + try { + parseValue("msg.payload"); + skipWhitespace(); + if (cursor !== text.length) return []; + } catch (_) { + return []; + } + return targets.sort(function (left, right) { + return left.start - right.start || left.end - right.end; + }); + } + + function streamRawBytes(raw) { + if (typeof raw !== "string" || raw === "") return new Uint8Array(0); + try { + var decoded = window.atob(raw); + var bytes = new Uint8Array(decoded.length); + for (var index = 0; index < decoded.length; index += 1) { + bytes[index] = decoded.charCodeAt(index); + } + return bytes; + } catch (_) { + return new Uint8Array(0); + } + } + + function streamBytesHex(bytes, spaced) { + return Array.prototype.map.call(bytes, function (byte) { + return byte.toString(16).padStart(2, "0").toUpperCase(); + }).join(spaced ? " " : ""); + } + + function streamPayloadJSON(documentText) { + // MarshalJSON emits payload as the first top-level field. Extract that + // value without parsing it so integer tokens and native n2k key order stay + // exactly as Go serialized them. + var keyIndex = documentText.indexOf('"payload"'); + if (keyIndex < 0) return ""; + var colon = documentText.indexOf(":", keyIndex + 9); + if (colon < 0) return ""; + var start = colon + 1; + while (start < documentText.length && /\s/.test(documentText[start])) start += 1; + if (start >= documentText.length) return ""; + + var opening = documentText[start]; + if (opening !== "{" && opening !== "[" && opening !== '"') { + var primitiveEnd = start; + while (primitiveEnd < documentText.length && documentText[primitiveEnd] !== "," && documentText[primitiveEnd] !== "}") { + primitiveEnd += 1; + } + return documentText.slice(start, primitiveEnd).trim(); + } + + var depth = 0; + var inString = false; + var escaped = false; + for (var index = start; index < documentText.length; index += 1) { + var character = documentText[index]; + if (inString) { + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === '"') { + inString = false; + if (opening === '"' && depth === 0) return documentText.slice(start, index + 1); + } + continue; + } + if (character === '"') { + inString = true; + } else if (character === "{" || character === "[") { + depth += 1; + } else if (character === "}" || character === "]") { + depth -= 1; + if (depth === 0) return documentText.slice(start, index + 1); + } + } + return ""; + } + + function streamClipboardText(entries, format) { + var chronological = entries.slice().reverse(); + var lines; + if (format === "can") { + lines = chronological + .map(function (entry) { return streamRawBytes(entry.raw); }) + .filter(function (bytes) { return bytes.length > 0; }) + .map(function (bytes) { return streamBytesHex(bytes, true); }); + } else { + lines = chronological.map(function (entry) { + return entry.streamPayloadJSON || JSON.stringify(entry.payload); + }); + } + return lines.length ? lines.join("\n") + "\n" : ""; + } + + function writeClipboard(text) { + if (navigator.clipboard && typeof navigator.clipboard.writeText === "function") { + return navigator.clipboard.writeText(text); + } + return new Promise(function (resolve, reject) { + var textarea = document.createElement("textarea"); + textarea.value = text; + textarea.setAttribute("readonly", ""); + textarea.style.position = "fixed"; + textarea.style.opacity = "0"; + document.body.appendChild(textarea); + textarea.select(); + var copied = document.execCommand("copy"); + textarea.remove(); + if (copied) resolve(); + else reject(new Error("copy failed")); + }); + } + + function exportStream(entries, format, kind, id) { + var chronological = entries.slice().reverse(); + var body; + var extension; + var mime; + if (format === "can") { + body = chronological + .map(function (entry) { return streamRawBytes(entry.raw); }) + .filter(function (bytes) { return bytes.length > 0; }) + .map(function (bytes) { return streamBytesHex(bytes, false); }) + .join("\n") + "\n"; + extension = "can.hex"; + mime = "text/plain"; + } else { + body = chronological.map(function (entry) { + return entry.streamPayloadJSON || JSON.stringify(entry.payload); + }).join("\n") + "\n"; + extension = "n2k.jsonl"; + mime = "application/x-ndjson"; + } + + var safeID = String(id || "stream").replace(/[^A-Za-z0-9_.-]+/g, "-"); + var stamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z"); + var name = String(kind || "component") + "-" + safeID + "-" + stamp + "." + extension; + var url = URL.createObjectURL(new Blob([body], { type: mime })); + var link = document.createElement("a"); + link.href = url; + link.download = name; + document.body.appendChild(link); + link.click(); + link.remove(); + window.setTimeout(function () { URL.revokeObjectURL(url); }, 0); + } + + function initSourceDeviceDialog(root) { + var scope = root && root.querySelector ? root : document; + var dialog = scope.matches && scope.matches("[data-source-device-dialog]") + ? scope + : scope.querySelector("[data-source-device-dialog]"); + if (!dialog || dialog.dataset.dialogReady === "true") return; + dialog.dataset.dialogReady = "true"; + + var close = dialog.querySelector("[data-source-device-dialog-close]"); + function requestClose(event) { + if (event) event.preventDefault(); + if (close) close.click(); + else dialog.close(); + } + + dialog.addEventListener("cancel", requestClose); + dialog.addEventListener("click", function (event) { + if (event.target === dialog) requestClose(event); + }); + + if (typeof dialog.showModal === "function") { + if (!dialog.open) dialog.showModal(); + } else { + dialog.setAttribute("open", ""); + } + } + + document.addEventListener("pointerdown", function (event) { + activeSourceDeviceRow = sourceDeviceRowFromTarget(event.target); + }); + ["pointerup", "pointercancel"].forEach(function (eventName) { + document.addEventListener(eventName, function () { + window.setTimeout(function () { + activeSourceDeviceRow = null; + }, 0); + }); + }); + + document.addEventListener("keydown", function (event) { + var row = sourceDeviceRowFromTarget(event.target); + if (row && (event.key === "Enter" || event.key === " " || event.key === "Spacebar")) { + event.preventDefault(); + row.click(); + return; + } + if (event.key !== "Escape") return; + var panel = document.getElementById("source-device-detail-panel"); + var close = panel && panel.tagName === "DIALOG" && panel.open && + panel.querySelector("[data-source-device-dialog-close]"); + if (close) { + event.preventDefault(); + close.click(); + } + }); + if (document.readyState === "loading") { document.addEventListener("DOMContentLoaded", function () { initCELAutocomplete(document); + initSourceDeviceDialog(document); + initStreamPanels(document); }); } else { initCELAutocomplete(document); + initSourceDeviceDialog(document); + initStreamPanels(document); } document.addEventListener("htmx:load", function (event) { - initCELAutocomplete(event.detail && event.detail.elt ? event.detail.elt : document); + var root = event.detail && event.detail.elt ? event.detail.elt : document; + initCELAutocomplete(root); + initSourceDeviceDialog(root); + initStreamPanels(root); + }); + document.addEventListener("htmx:beforeCleanupElement", function (event) { + var root = event.detail && event.detail.elt; + if (!root || !root.querySelectorAll) return; + var panels = Array.prototype.slice.call(root.querySelectorAll("[data-stream-panel]")); + if (root.matches && root.matches("[data-stream-panel]")) panels.unshift(root); + panels.forEach(function (panel) { + if (typeof panel.stopStreamCapture === "function") panel.stopStreamCapture(); + }); + }); + document.addEventListener("htmx:afterRequest", function (event) { + var trigger = event.detail && event.detail.elt; + if (!trigger) return; + if (!event.detail.successful) return; + if (trigger.matches("[data-source-device-row-refresh]")) { + reconcileSourceDeviceRows(event.detail.xhr.responseText); + return; + } + if (trigger.matches("[data-source-device-pgn-sort-control]")) { + syncSourceDevicePGNSortControls(event.detail.xhr.responseText); + } }); })(); diff --git a/internal/ui/dashboard.go b/internal/ui/dashboard.go index 5d16788..ec9d200 100644 --- a/internal/ui/dashboard.go +++ b/internal/ui/dashboard.go @@ -1,6 +1,6 @@ // dashboard.go implements beacon's live home view: the source -> connector // -> sink DAG templates/frag_dashboard.html renders, and the GET -// /ui/frag/dashboard handler templates/dashboard.html polls every 2s +// /frag/dashboard handler templates/dashboard.html polls every 2s // (hx-trigger="load, every 2s") — see dashboard.html's comment for why it // ships an empty container rather than rendering this fragment inline, the // same "ships empty, fetches client-side" shape the overview pages use. @@ -266,15 +266,15 @@ func inventoryDeviceRows(records []inventory.Record) []busDeviceRow { // endpoint sides exist. func dashboardEmptyState(hasSources, hasSinks bool) (title, message, cta, href string) { if !hasSources { - return "Add your first source", "Connect a source to start receiving data.", "Add a source", "/ui/sources/new" + return "Add your first source", "Connect a source to start receiving data.", "Add a source", "/sources/new" } if !hasSinks { - return "Add your first sink", "Choose where beacon should deliver routed messages.", "Add a sink", "/ui/sinks/new" + return "Add your first sink", "Choose where beacon should deliver routed messages.", "Add a sink", "/sinks/new" } - return "Add your first connector", "Wire a source to a sink with a connector to start routing data.", "Add a connector", "/ui/connectors/new" + return "Add your first connector", "Wire a source to a sink with a connector to start routing data.", "Add a connector", "/connectors/new" } -// handleDashboardFrag serves GET /ui/frag/dashboard: the dashboard page's +// handleDashboardFrag serves GET /frag/dashboard: the dashboard page's // hx-trigger="load, every 2s" polling target (see templates/dashboard.html). // Lists every configured source/sink/connector fresh on every poll — a // source/sink/connector created or deleted elsewhere in the UI must show up diff --git a/internal/ui/dashboard_test.go b/internal/ui/dashboard_test.go index acfaa0d..1ef5ccb 100644 --- a/internal/ui/dashboard_test.go +++ b/internal/ui/dashboard_test.go @@ -62,7 +62,7 @@ func newDashboardTestServer(t *testing.T) (*httptest.Server, *config.Service, *s handler := ui.Handler(svc, reg, rec.Statuses, nil, "test", nil) mux := http.NewServeMux() - mux.Handle("/ui/", handler) + mux.Handle("/", handler) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) return srv, svc, reg, rec @@ -90,7 +90,7 @@ func markerSnippet(t *testing.T, body, marker string) string { func dashboardFrag(t *testing.T, srv *httptest.Server) string { t.Helper() - resp, err := http.Get(srv.URL + "/ui/frag/dashboard") + resp, err := http.Get(srv.URL + "/frag/dashboard") if err != nil { t.Fatal(err) } @@ -104,13 +104,13 @@ func TestDashboardFragEmptyStateNoSourcesPointsAtSources(t *testing.T) { srv, _, _, _ := newDashboardTestServer(t) body := dashboardFrag(t, srv) - for _, want := range []string{"Add your first source", `href="/ui/sources/new"`} { + for _, want := range []string{"Add your first source", `href="/sources/new"`} { if !strings.Contains(body, want) { t.Fatalf("empty dashboard fragment missing %q:\n%s", want, body) } } empty := markerSnippet(t, body, "Add your first source") - if !strings.Contains(empty, `href="/ui/sources/new"`) { + if !strings.Contains(empty, `href="/sources/new"`) { t.Fatalf("initial empty-state CTA should point at sources:\n%s", empty) } } @@ -122,7 +122,7 @@ func TestDashboardFragEmptyStateWithSourceNeedsSink(t *testing.T) { }, true)) body := dashboardFrag(t, srv) - for _, want := range []string{"Add your first sink", `href="/ui/sinks/new"`} { + for _, want := range []string{"Add your first sink", `href="/sinks/new"`} { if !strings.Contains(body, want) { t.Fatalf("dashboard fragment (sources, no connectors) missing %q:\n%s", want, body) } @@ -137,7 +137,7 @@ func TestDashboardFragEmptyStateWithSourceAndSinkPointsAtConnectors(t *testing.T seedSourceSink(t, svc) body := dashboardFrag(t, srv) - for _, want := range []string{"Add your first connector", `href="/ui/connectors/new"`} { + for _, want := range []string{"Add your first connector", `href="/connectors/new"`} { if !strings.Contains(body, want) { t.Fatalf("dashboard fragment (source+sink, no connectors) missing %q:\n%s", want, body) } @@ -159,12 +159,12 @@ func TestDashboardFragRendersConnectorDAG(t *testing.T) { for _, want := range []string{ "dag-board", "dag-node-source", "dag-node-connector", "dag-node-sink", `data-node-id="src1"`, `data-connector-id="heading"`, `data-node-id="sink1"`, - `href="/ui/sources/src1/"`, `href="/ui/sinks/sink1/"`, - `href="/ui/sources/new"`, `href="/ui/sinks/new"`, `href="/ui/connectors/new"`, + `href="/sources/src1/"`, `href="/sinks/sink1/"`, + `href="/sources/new"`, `href="/sinks/new"`, `href="/connectors/new"`, "metadata-stack", "metadata-table", "Detail", "can0", "127.0.0.1:9000", - "Heading only", `href="/ui/connectors/heading/"`, "Source One", "Sink One", - `href="/ui/sources/src1/">Source One`, `href="/ui/sinks/sink1/">Sink One`, + "Heading only", `href="/connectors/heading/"`, "Source One", "Sink One", + `href="/sources/src1/">Source One`, `href="/sinks/sink1/">Sink One`, "badge-success\">enabled", "msg/s", "B/s", "queued", } { if !strings.Contains(body, want) { @@ -228,7 +228,7 @@ func TestDashboardFragConnectorErrorBadge(t *testing.T) { for _, want := range []string{ `dag-node-connector component-status-surface state-error`, `dag-link state-error`, - ``, + ``, } { if !strings.Contains(body, want) { t.Errorf("dashboard fragment missing connector error colorization %q:\n%s", want, body) @@ -292,13 +292,13 @@ func TestDashboardFragEndpointNodeStates(t *testing.T) { } for _, want := range []string{ - ``, - ``, - ``, - ``, - ``, - ``, - ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, ``, ``, ``, @@ -312,12 +312,12 @@ func TestDashboardFragEndpointNodeStates(t *testing.T) { for _, tc := range []struct { path, want string }{ - {"/ui/sources", ``}, - {"/ui/sinks", ``}, - {"/ui/connectors", ``}, - {"/ui/frag/sources/up-src/overview", `
`}, - {"/ui/frag/sinks/err-sink/overview", `
`}, - {"/ui/frag/connectors/up-conn/overview", `
`}, + {"/sources", ``}, + {"/sinks", ``}, + {"/connectors", ``}, + {"/frag/sources/up-src/overview", `
`}, + {"/frag/sinks/err-sink/overview", `
`}, + {"/frag/connectors/up-conn/overview", `
`}, } { resp, err := http.Get(srv.URL + tc.path) if err != nil { @@ -346,11 +346,19 @@ func TestDashboardFragMetadataTablesCountEndpointUsage(t *testing.T) { "metadata-stack", "Sources", "Sinks", "Connectors", "Used source", "2 connectors", "Unused source", "0 connectors", "Used sink", "Unused sink", "usage-dot-used", "usage-dot-unused", + ``, + ``, + ``, } { if !strings.Contains(body, want) { t.Fatalf("dashboard metadata tables missing %q:\n%s", want, body) } } + for _, notWant := range []string{`>View`, `>Edit`, `Actions`} { + if strings.Contains(body, notWant) { + t.Fatalf("dashboard metadata tables unexpectedly contain row action %q:\n%s", notWant, body) + } + } } // --- Dashboard page shell --- @@ -358,7 +366,7 @@ func TestDashboardFragMetadataTablesCountEndpointUsage(t *testing.T) { func TestDashboardPageHostsPollingContainer(t *testing.T) { srv, _, _, _ := newDashboardTestServer(t) - resp, err := http.Get(srv.URL + "/ui/dashboard") + resp, err := http.Get(srv.URL + "/dashboard") if err != nil { t.Fatal(err) } @@ -366,7 +374,7 @@ func TestDashboardPageHostsPollingContainer(t *testing.T) { body := mustBody(t, resp) for _, want := range []string{ `id="dashboard-panel"`, - `hx-get="/ui/frag/dashboard"`, + `hx-get="/frag/dashboard"`, `hx-trigger="load, every 2s"`, `hx-swap="innerHTML"`, } { diff --git a/internal/ui/docs/01-getting-started.md b/internal/ui/docs/01-getting-started.md index e1d189c..4007486 100644 --- a/internal/ui/docs/01-getting-started.md +++ b/internal/ui/docs/01-getting-started.md @@ -36,7 +36,10 @@ Gateway streams use an `address` in `host:port` form and a `format` of NMEA 2000 address; a `tcp_gateway` sink is writable, claims an address, and participates on the remote bus. A `file` source takes an absolute `file_path`, replays its capture once at the recorded timing, then remains -up and idle. +up and idle. Gzip-compressed captures are detected from their contents, so +the filename does not need a `.gz` suffix. When the configured path is +missing, Beacon also tries the same path with `.gz` appended; an existing +exact path always takes precedence. ## Running beacon @@ -76,7 +79,7 @@ exists on the host (see the CAN setup page). The image's healthcheck polls ## Your first pipeline, via the UI -1. Open `/ui` (redirects to the dashboard). +1. Open `/` (redirects to `/dashboard`). 2. **Sources** → Add source. For a quick test without hardware, use a `socketcan` source pointed at `vcan0` (see the CAN setup page for bringing up a virtual interface). diff --git a/internal/ui/docs/03-concepts.md b/internal/ui/docs/03-concepts.md index 89b5f0a..041dcb9 100644 --- a/internal/ui/docs/03-concepts.md +++ b/internal/ui/docs/03-concepts.md @@ -32,33 +32,84 @@ filters and its own buffer. ## The envelope -Every message flowing through beacon — in a connector's buffer, over SSE/WS/TCP/MQTT, -and as the value a CEL filter's `msg` variable is bound to — is a JSON -object with this shape: - -| JSON field | Type | Present when | Meaning | -|---|---|---|---| -| `id` | integer | queued/replayed messages only | the message's sequence number within its connector's buffer | -| `connector` | string | queued/replayed messages only | the id of the connector that delivered this message | -| `pgn` | integer | always | the NMEA 2000 PGN number | -| `source` | integer | always | source device address (0-253) | -| `dest` | integer | always | destination address (255 = broadcast) | -| `priority` | integer | always | 0 (highest) to 7 (lowest) | -| `timestamp` | string | always | RFC 3339 timestamp | -| `observed_at` | string | always | when this Beacon ingress observed the message | -| `ingress` / `origin_ingress` | string | when known | current and first-known ingress provenance | -| `device_name` | integer | after an address claim is known | stable 64-bit ISO Device NAME; unlike `source`, it survives address changes | -| `device_name_hex` | string | after an address claim is known | the same NAME as 16 uppercase hex digits, safe for JavaScript and database keys | -| `pgn_name` / `variant` / `transport` | string | cataloged PGNs | catalog identity and N2K transport kind | -| `manufacturer_code` | integer | proprietary PGNs | manufacturer discriminator decoded from the payload | -| `decode` | object | always | `decoded` or `unknown`, plus catalog completeness/fallback/missing metadata | -| `physical` | object | decoded numeric fields | unit-scaled values with unit, physical quantity, and catalog range; raw payload ticks remain unchanged | -| `payload` | object or `null` | always | the decoded PGN fields, one JSON key per field, `null` for a PGN beacon doesn't know how to decode | -| `raw` | string (base64) | usually | the assembled CAN payload bytes: original for an undecodable PGN and canonical re-encoding for a decoded PGN. Semantic CAN delivery skips an unknown decoder; transparent SocketCAN mode forwards it losslessly | - -`id` and `connector` are only populated once a message has passed through a -connector's buffer — a freshly-decoded message from a source doesn't have -them yet. +Every message sent over SSE, WebSocket, TCP, MQTT, or NDJSON has exactly three +top-level keys: + +| JSON field | Type | Meaning | +|---|---|---| +| `payload` | object | the verbatim JSON representation of the decoded `open-ships/n2k` Go struct | +| `metadata` | object | every field Beacon adds for routing, provenance, enrichment, and replay | +| `raw` | string or null | base64-encoded assembled CAN bytes, kept separate because they are data rather than metadata | + +Every `payload` includes the n2k struct's complete `info` object. In n2k v1 this +includes timestamp, receive/transport timing, adapter ID, network ID, +direction, priority, PGN, source ID, and target ID. All remaining keys are the +original generated fields for that PGN, using n2k's JSON names and raw wire +values. Beacon does not strip, rename, or relocate any n2k field. A Go consumer +can unmarshal the nested object directly: + +```go +var event struct { + Payload pgn.VesselHeading `json:"payload"` + Metadata json.RawMessage `json:"metadata"` +} +err := json.Unmarshal(document, &event) +``` + +For an unknown PGN, `payload` is the complete `pgn.UnknownPGN` JSON rather than +`null`. + +`metadata` can contain: + +| JSON field | Present when | Meaning | +|---|---|---| +| `id` / `connector` | queued/replayed messages | route sequence and connector identity | +| `observed_at` | always | when this Beacon ingress observed the message | +| `ingress` / `origin_ingress` | known | current and first-known ingress provenance | +| `device_name` / `device_name_hex` | address claim known | stable ISO Device NAME in numeric and JavaScript-safe hexadecimal forms | +| `pgn_name` / `variant` / `transport` | cataloged PGNs | catalog identity and N2K transport kind | +| `manufacturer_code` | proprietary PGNs | manufacturer discriminator decoded from the payload | +| `decode` | always | decoded/unknown status and catalog completeness details | +| `physical` | decoded numeric fields | additive unit-scaled values; payload raw ticks stay unchanged | + +`id` and `connector` appear only after a message passes through a connector +buffer. Top-level `raw` is original for unknown PGNs and a canonical +re-encoding for decoded PGNs. + +## Source and sink stream inspector + +Each source and sink overview has a **Stream contents** panel. It starts +stopped and captures only messages that arrive after **Start**: + +- A source panel observes envelopes at that source's received boundary. +- A sink panel observes envelopes after successful or accepted delivery to + that sink. +- **Stop** closes the live preview while keeping the current capture in the + browser. Start and Stop replace one another so only the currently relevant + action is shown. **Clear** removes that local capture. + +The optional **CEL filter** beside **Start** is applied by Beacon before +messages reach the browser. It uses the same `msg` shape as connector filters, +for example `msg.pgn == 129026` or +`msg.source == 11 && msg.decode_status == "decoded"`. +The filter remains editable while streaming. A valid change reconnects the +best-effort preview with the new expression without clearing captured rows; an +invalid change leaves the current stream filter active. + +The inspector is best-effort and non-blocking: it never consumes a connector's +durable queue or slows vessel traffic when the browser cannot keep up. The +captured count continues for the full browser session while the panel keeps the +latest 200 messages in the current browser tab for display, copy, and export. + +**JSONL** shows one compact, verbatim nested `payload` per line, ready to +unmarshal into the corresponding n2k Go struct. **CAN bytes** shows one +top-level `raw` payload as hexadecimal per line. **Export JSONL** downloads the +captured payload lines oldest first. **Export CAN** downloads one uppercase +hexadecimal payload per line; the line boundary preserves the message boundary +that would be lost by concatenating the bytes into one binary blob. Click any +JSON key or value to inspect a relevant CEL filter and optionally apply it. +**Copy stream** copies every retained line in arrival order using the active +JSONL or spaced-hex CAN representation. ## Buffering and pruning diff --git a/internal/ui/docs/04-filters.md b/internal/ui/docs/04-filters.md index ba3fa97..aafe7c1 100644 --- a/internal/ui/docs/04-filters.md +++ b/internal/ui/docs/04-filters.md @@ -7,8 +7,10 @@ list evaluates to `true`. An empty list passes everything. ## Variables -Every expression sees a single variable, `msg`, matching the envelope shape -(see the concepts page): +Every expression sees a single variable, `msg`. This is Beacon's convenient +flat filtering view; the external MQTT/SSE/WS/TCP envelope separately nests PGN +and native n2k `MessageInfo` fields under `payload`, Beacon context under +`metadata`, and the assembled CAN bytes under top-level `raw`: | Variable | CEL type | Notes | |---|---|---| @@ -25,7 +27,7 @@ Every expression sees a single variable, `msg`, matching the envelope shape | `msg.manufacturer_code` | int | proprietary manufacturer code, or zero | | `msg.decode_status` | string | `decoded` or `unknown` | | `msg.physical` | map | scaled fields keyed like payload, each with `value`, `unit`, and `physical_quantity` | -| `msg.payload` | map | decoded PGN fields, keyed by their JSON field name (camelCase, e.g. `heading`, `speedWaterReferenced`) | +| `msg.payload` | map | complete native n2k struct JSON, including every exported `info` field | `msg.payload`'s keys depend entirely on which PGN the message is — check the specific PGN's decoded field names (the same names its JSON `payload` diff --git a/internal/ui/docs/05-api.md b/internal/ui/docs/05-api.md index 94a17c7..b53f500 100644 --- a/internal/ui/docs/05-api.md +++ b/internal/ui/docs/05-api.md @@ -6,7 +6,7 @@ start from its own documentation rather than this page: - `/api/docs` — an interactive, offline reference (no CDN dependency) - `/api/openapi.json` — the machine-readable OpenAPI 3.1 document backing it -- `/ui/mcp` — the offline MCP connection guide, tool catalog, and call examples +- `/mcp/info` — the offline MCP connection guide, tool catalog, and call examples ## MCP (for AI agents) @@ -23,16 +23,14 @@ session headers: ``` The tools are `get_config`, `put_source`, `put_sink`, `put_connector`, -`delete_source`, `delete_sink`, `delete_connector`, `get_health`, and -`get_delivery_statistics`, plus `get_source_metrics`, -`commit_source_traffic_baseline`, and `clear_source_traffic_baseline`. Source -metrics can be filtered by configured source id, PGN, and NMEA 2000 source -address. They include learned timing/jitter, rates and estimated bus load, -addressing, decode quality, payload-size and raw-byte distributions, last-seen -age, gaps/bursts, anomalies, decoded-field quantiles and availability, approved -expectations, and recent lifecycle events. Input and output JSON Schemas are -returned by MCP `tools/list`. Configuration and baseline writes persist to -SQLite; configuration changes also reconcile immediately through the +`delete_source`, `delete_sink`, `delete_connector`, `get_health`, +`get_delivery_metrics`, and `get_source_metrics`. Source metrics can be +filtered by configured source id, PGN, and NMEA 2000 source address. They +include learned timing/jitter, rates and estimated bus load, addressing, +decode quality, payload-size and raw-byte distributions, last-seen age, +gaps/bursts, decoded-field quantiles and availability, and recent lifecycle +events. Input and output JSON Schemas are returned by MCP `tools/list`. +Configuration writes persist to SQLite and reconcile immediately through the supervisor. MCP needs no cloud relay, separate process, remote schema, or internet access. @@ -155,10 +153,10 @@ stream. The same process-local store is available to agents through `get_source_metrics` and at `/metrics` under the `beacon_source_pgn_*` metric families. Frequency and gap thresholds are learned from recent arrival intervals. Live observation windows reset when Beacon -restarts; operator-approved expected-traffic baselines and bounded lifecycle -events persist in SQLite. Prometheus exports bounded numeric summaries and -finite labels, while raw hexdumps and payload fingerprints remain available in -the UI and MCP response rather than becoming high-cardinality time series. +restarts; bounded lifecycle events persist in SQLite. Prometheus exports +bounded numeric summaries and finite labels, while raw hexdumps and payload +fingerprints remain available in the UI and MCP response rather than becoming +high-cardinality time series. ## Health and system info diff --git a/internal/ui/docs/06-troubleshooting.md b/internal/ui/docs/06-troubleshooting.md index 2d65194..90b8cbd 100644 --- a/internal/ui/docs/06-troubleshooting.md +++ b/internal/ui/docs/06-troubleshooting.md @@ -26,16 +26,6 @@ age versus the stream's learned period. The equivalent Prometheus signal is `beacon_source_pgn_gap_active`; agents can query the same store with the MCP `get_source_metrics` tool. -After the source has run under normal conditions, use **Set expected traffic -baseline** to save the observed senders, PGNs, rates, payload shapes, decode -metadata, addressing, and field ranges as the expected baseline. The overview -will then surface a PGN that never returns after startup, frequency drift, new -payload lengths or destinations, Device NAME/address changes, and values outside -the approved range. Baselines can also be managed with the MCP -`commit_source_traffic_baseline` and `clear_source_traffic_baseline` tools. -Setting the baseline replaces the previous one with streams observed since -Beacon started; do not set it while an expected device is absent. - ## CAN write failures A `socketcan`/`usbcan` sink confirms every push; on failure the connector diff --git a/internal/ui/docspages.go b/internal/ui/docspages.go index cbf68ec..35d1091 100644 --- a/internal/ui/docspages.go +++ b/internal/ui/docspages.go @@ -1,13 +1,11 @@ -// docspages.go implements beacon's onboard operator manual at GET /ui/docs -// and GET /ui/docs/{slug}: the markdown files embedded from docs/*.md, +// docspages.go implements beacon's onboard operator manual at GET /docs +// and GET /docs/{slug}: the markdown files embedded from docs/*.md, // rendered to HTML once per file and cached (the embedded content never // changes at runtime, so there is nothing to invalidate), and served inside // templates/docs.html's sidebar +
layout — see pages.go's "docs" -// entry and ui.go's route registrations. internal/app additionally -// 301-redirects the bare "/docs" and "/docs/{slug}" paths (on the admin -// mux, spec §5) to these /ui/docs equivalents; "/docs" is one of -// model.ReservedPathPrefixes, so no HTTP sink can ever be configured to -// collide with either mount. +// entry and ui.go's root-level route registrations. "/docs" is reserved +// from HTTP sink paths, so configured data endpoints cannot collide with +// the manual. package ui import ( @@ -195,9 +193,9 @@ func docNavItems() []docNavItem { return items } -// handleDocsIndex serves GET /ui/docs: a 302 to the first manual page (the +// handleDocsIndex serves GET /docs: a 302 to the first manual page (the // same convention as a directory index), so operators can bookmark or link -// to plain "/ui/docs" without needing to know page order or which file is +// to plain "/docs" without needing to know page order or which file is // first. func handleDocsIndex() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -205,11 +203,11 @@ func handleDocsIndex() http.HandlerFunc { if len(docPages) > 0 { first = docPages[0].Slug } - http.Redirect(w, r, "/ui/docs/"+first, http.StatusFound) + http.Redirect(w, r, "/docs/"+first, http.StatusFound) } } -// handleDocPage serves GET /ui/docs/{slug}: the manual page whose slug +// handleDocPage serves GET /docs/{slug}: the manual page whose slug // matches, or a 404 for an unknown one. func handleDocPage(version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/ui/flash.go b/internal/ui/flash.go index 7be8548..f927e9e 100644 --- a/internal/ui/flash.go +++ b/internal/ui/flash.go @@ -28,7 +28,7 @@ func setFlash(w http.ResponseWriter, msg string) { http.SetCookie(w, &http.Cookie{ // #nosec G124 -- Beacon supports local HTTP; HttpOnly and SameSite protect this non-sensitive flash value. Name: flashCookie, Value: base64.URLEncoding.EncodeToString([]byte(msg)), - Path: "/ui", + Path: "/", HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: 60, // self-expire if a redirect never lands on a reader @@ -46,7 +46,7 @@ func takeFlash(w http.ResponseWriter, r *http.Request) string { http.SetCookie(w, &http.Cookie{ // #nosec G124 -- must match the local-HTTP-compatible cookie being cleared. Name: flashCookie, Value: "", - Path: "/ui", + Path: "/", HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: -1, // delete now that it has been read diff --git a/internal/ui/forms.go b/internal/ui/forms.go index b45733f..7191203 100644 --- a/internal/ui/forms.go +++ b/internal/ui/forms.go @@ -2,7 +2,7 @@ // page/fragment data types templates/sources.html, sinks.html, // connectors.html, connector_detail.html, and every frag_*.html render // against, the model<->form conversions, and the HTTP handlers Handler -// (ui.go) wires up at /ui/sources, /ui/sinks, /ui/connectors, /ui/frag/*. +// (ui.go) wires up at /sources, /sinks, /connectors, /frag/*. // // Sources and sinks are handled by deliberately parallel, non-generic code // (sourceX / sinkX pairs) rather than a shared generic implementation: the @@ -392,7 +392,7 @@ func renderSourcesPage(w http.ResponseWriter, r *http.Request, svc *config.Servi current = "Edit " + form.ID } page = page.withBreadcrumbs( - breadcrumbItem{Label: "Sources", Href: "/ui/sources"}, + breadcrumbItem{Label: "Sources", Href: "/sources"}, breadcrumbItem{Label: current}, ) } @@ -404,14 +404,14 @@ func renderSourcesPage(w http.ResponseWriter, r *http.Request, svc *config.Servi renderPage(w, log, "sources", data) } -// handleSourcesPage serves GET /ui/sources: the full list page. +// handleSourcesPage serves GET /sources: the full list page. func handleSourcesPage(svc *config.Service, statuses func() []supervisor.Status, version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { renderSourcesPage(w, r, svc, statuses, version, log, nil) } } -// handleSourceNewPage serves GET /ui/sources/new: the canonical create page. +// handleSourceNewPage serves GET /sources/new: the canonical create page. func handleSourceNewPage(svc *config.Service, statuses func() []supervisor.Status, version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { can, serial := discoverHardware() @@ -420,7 +420,7 @@ func handleSourceNewPage(svc *config.Service, statuses func() []supervisor.Statu } } -// handleSourceEditPage serves GET /ui/sources/{id}/edit: the canonical edit page. +// handleSourceEditPage serves GET /sources/{id}/edit: the canonical edit page. func handleSourceEditPage(svc *config.Service, statuses func() []supervisor.Status, version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { can, serial := discoverHardware() @@ -439,7 +439,7 @@ func handleSourceEditPage(svc *config.Service, statuses func() []supervisor.Stat } } -// handleSourceTypeFieldsFrag serves GET /ui/frag/source-type-fields: the +// handleSourceTypeFieldsFrag serves GET /frag/source-type-fields: the // type select's hx-get target. hx-include="closest form" resends every // field currently in the form as a query parameter, so the newly selected // type's own field keeps its value if it happens to still be in the DOM — @@ -468,14 +468,14 @@ func handleSourceTypeFieldsFrag(log *slog.Logger) http.HandlerFunc { } } -// handleSourceCreate serves POST /ui/sources. +// handleSourceCreate serves POST /sources. func handleSourceCreate(svc *config.Service, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { writeSource(w, r, svc, log, true, "") } } -// handleSourceUpdate serves POST /ui/sources/{id}. +// handleSourceUpdate serves POST /sources/{id}. func handleSourceUpdate(svc *config.Service, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { writeSource(w, r, svc, log, false, r.PathValue("id")) @@ -521,7 +521,7 @@ func writeSource(w http.ResponseWriter, r *http.Request, svc *config.Service, lo return } if isCreate { - flashRedirect(w, fmt.Sprintf("Source %q created", v.ID), "/ui/dashboard") + flashRedirect(w, fmt.Sprintf("Source %q created", v.ID), "/dashboard") return } sources, err := svc.ListSources(r.Context()) @@ -566,7 +566,7 @@ func entityWriteErrorMessage(err error, kind, id string) string { } } -// handleSourceDelete serves POST /ui/sources/{id}/delete. +// handleSourceDelete serves POST /sources/{id}/delete. func handleSourceDelete(svc *config.Service, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") @@ -845,7 +845,7 @@ func renderSinksPage(w http.ResponseWriter, r *http.Request, svc *config.Service current = "Edit " + form.ID } page = page.withBreadcrumbs( - breadcrumbItem{Label: "Sinks", Href: "/ui/sinks"}, + breadcrumbItem{Label: "Sinks", Href: "/sinks"}, breadcrumbItem{Label: current}, ) } @@ -857,14 +857,14 @@ func renderSinksPage(w http.ResponseWriter, r *http.Request, svc *config.Service renderPage(w, log, "sinks", data) } -// handleSinksPage serves GET /ui/sinks: the full list page. +// handleSinksPage serves GET /sinks: the full list page. func handleSinksPage(svc *config.Service, statuses func() []supervisor.Status, version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { renderSinksPage(w, r, svc, statuses, version, log, nil) } } -// handleSinkNewPage serves GET /ui/sinks/new: the canonical create page. +// handleSinkNewPage serves GET /sinks/new: the canonical create page. func handleSinkNewPage(svc *config.Service, statuses func() []supervisor.Status, version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { can, serial := discoverHardware() @@ -873,7 +873,7 @@ func handleSinkNewPage(svc *config.Service, statuses func() []supervisor.Status, } } -// handleSinkEditPage serves GET /ui/sinks/{id}/edit: the canonical edit page. +// handleSinkEditPage serves GET /sinks/{id}/edit: the canonical edit page. func handleSinkEditPage(svc *config.Service, statuses func() []supervisor.Status, version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { can, serial := discoverHardware() @@ -892,7 +892,7 @@ func handleSinkEditPage(svc *config.Service, statuses func() []supervisor.Status } } -// handleSinkTypeFieldsFrag serves GET /ui/frag/sink-type-fields. The same +// handleSinkTypeFieldsFrag serves GET /frag/sink-type-fields. The same // type-switch caveat as handleSourceTypeFieldsFrag applies: a previously // selected type's field values are discarded on switch, not carried over. func handleSinkTypeFieldsFrag(log *slog.Logger) http.HandlerFunc { @@ -921,14 +921,14 @@ func handleSinkTypeFieldsFrag(log *slog.Logger) http.HandlerFunc { } } -// handleSinkCreate serves POST /ui/sinks. +// handleSinkCreate serves POST /sinks. func handleSinkCreate(svc *config.Service, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { writeSink(w, r, svc, log, true, "") } } -// handleSinkUpdate serves POST /ui/sinks/{id}. +// handleSinkUpdate serves POST /sinks/{id}. func handleSinkUpdate(svc *config.Service, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { writeSink(w, r, svc, log, false, r.PathValue("id")) @@ -966,7 +966,7 @@ func writeSink(w http.ResponseWriter, r *http.Request, svc *config.Service, log return } if isCreate { - flashRedirect(w, fmt.Sprintf("Sink %q created", v.ID), "/ui/dashboard") + flashRedirect(w, fmt.Sprintf("Sink %q created", v.ID), "/dashboard") return } sinks, err := svc.ListSinks(r.Context()) @@ -982,7 +982,7 @@ func writeSink(w http.ResponseWriter, r *http.Request, svc *config.Service, log writeFragmentSuccess(w, log, "sink-panel-oob", "sink-form-container", data) } -// handleSinkDelete serves POST /ui/sinks/{id}/delete. +// handleSinkDelete serves POST /sinks/{id}/delete. func handleSinkDelete(svc *config.Service, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") @@ -1333,7 +1333,7 @@ func renderConnectorsPage(w http.ResponseWriter, r *http.Request, svc *config.Se current = "Edit " + form.ID } page = page.withBreadcrumbs( - breadcrumbItem{Label: "Connectors", Href: "/ui/connectors"}, + breadcrumbItem{Label: "Connectors", Href: "/connectors"}, breadcrumbItem{Label: current}, ) } @@ -1345,14 +1345,14 @@ func renderConnectorsPage(w http.ResponseWriter, r *http.Request, svc *config.Se renderPage(w, log, "connectors", data) } -// handleConnectorsPage serves GET /ui/connectors: the full list page. +// handleConnectorsPage serves GET /connectors: the full list page. func handleConnectorsPage(svc *config.Service, reg *stats.Registry, statuses func() []supervisor.Status, version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { renderConnectorsPage(w, r, svc, reg, statuses, version, log, nil) } } -// handleConnectorNewPage serves GET /ui/connectors/new: the canonical create page. +// handleConnectorNewPage serves GET /connectors/new: the canonical create page. func handleConnectorNewPage(svc *config.Service, reg *stats.Registry, statuses func() []supervisor.Status, version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { sources, sinks, err := listSourcesAndSinks(r.Context(), svc) @@ -1366,7 +1366,7 @@ func handleConnectorNewPage(svc *config.Service, reg *stats.Registry, statuses f } } -// handleConnectorEditPage serves GET /ui/connectors/{id}/edit: the canonical edit page. +// handleConnectorEditPage serves GET /connectors/{id}/edit: the canonical edit page. func handleConnectorEditPage(svc *config.Service, reg *stats.Registry, statuses func() []supervisor.Status, version string, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { sources, sinks, err := listSourcesAndSinks(r.Context(), svc) @@ -1399,7 +1399,7 @@ type filterValidateData struct { Err string } -// handleValidateFiltersFrag serves POST /ui/frag/validate-filters. Requests +// handleValidateFiltersFrag serves POST /frag/validate-filters. Requests // accepting application/json receive source ranges for live editor // underlines; other callers retain the original HTML status fragment. func handleValidateFiltersFrag(svc *config.Service, log *slog.Logger) http.HandlerFunc { @@ -1427,14 +1427,14 @@ func handleValidateFiltersFrag(svc *config.Service, log *slog.Logger) http.Handl } } -// handleConnectorCreate serves POST /ui/connectors. +// handleConnectorCreate serves POST /connectors. func handleConnectorCreate(svc *config.Service, reg *stats.Registry, statuses func() []supervisor.Status, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { writeConnector(w, r, svc, reg, statuses, log, true, "") } } -// handleConnectorUpdate serves POST /ui/connectors/{id}. +// handleConnectorUpdate serves POST /connectors/{id}. func handleConnectorUpdate(svc *config.Service, reg *stats.Registry, statuses func() []supervisor.Status, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { writeConnector(w, r, svc, reg, statuses, log, false, r.PathValue("id")) @@ -1479,7 +1479,7 @@ func writeConnector(w http.ResponseWriter, r *http.Request, svc *config.Service, return } if isCreate { - flashRedirect(w, fmt.Sprintf("Connector %q created", v.ID), "/ui/dashboard") + flashRedirect(w, fmt.Sprintf("Connector %q created", v.ID), "/dashboard") return } connectors, err := svc.ListConnectors(r.Context()) @@ -1495,7 +1495,7 @@ func writeConnector(w http.ResponseWriter, r *http.Request, svc *config.Service, writeFragmentSuccess(w, log, "connector-panel-oob", "connector-form-container", data) } -// handleConnectorDelete serves POST /ui/connectors/{id}/delete. Unlike +// handleConnectorDelete serves POST /connectors/{id}/delete. Unlike // handleSourceDelete/handleSinkDelete there is no ErrInUse case: // config.Service.DeleteConnector never returns it ("nothing else // references a connector"). @@ -1542,7 +1542,7 @@ type connectorStatsData struct { SparkPoints string } -// handleConnectorStatsFrag serves GET /ui/frag/connectors/{id}/stats, an +// handleConnectorStatsFrag serves GET /frag/connectors/{id}/stats, an // hx-trigger="load, every 2s" polling target using hx-swap="outerHTML". // A known connector that has never recorded (not yet started, or // idle since boot) renders zero-valued tiles rather than a notice — diff --git a/internal/ui/forms_test.go b/internal/ui/forms_test.go index 4b87efa..cdd3c7e 100644 --- a/internal/ui/forms_test.go +++ b/internal/ui/forms_test.go @@ -1,6 +1,7 @@ package ui_test import ( + "bufio" "context" "encoding/base64" "encoding/json" @@ -41,7 +42,7 @@ func newUIServerWithService(t *testing.T) (*httptest.Server, *config.Service) { handler := ui.Handler(svc, stats.NewRegistry(), fakeReconciler{}.Statuses, nil, "test", nil) mux := http.NewServeMux() - mux.Handle("/ui/", handler) + mux.Handle("/", handler) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) return srv, svc @@ -67,7 +68,7 @@ func newUIServerWithServiceAndRegistry(t *testing.T) (*httptest.Server, *config. handler := ui.Handler(svc, reg, fakeReconciler{}.Statuses, nil, "test", nil) mux := http.NewServeMux() - mux.Handle("/ui/", handler) + mux.Handle("/", handler) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) return srv, svc, reg @@ -117,13 +118,13 @@ func TestSourcesPageRendersConfiguredEntities(t *testing.T) { ID: "can0", Name: "Engine CAN", Type: model.SourceSocketCAN, Enabled: true, Interface: "can0", }, true)) - resp, err := http.Get(srv.URL + "/ui/sources") + resp, err := http.Get(srv.URL + "/sources") if err != nil { t.Fatal(err) } mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) - for _, want := range []string{"Engine CAN", "can0", "socketcan", "Detail", "badge-success", "component-status-row state-restarting", `href="/ui/sources/new"`, `href="/ui/sources/can0/"`, `href="/ui/sources/can0/edit"`} { + for _, want := range []string{"Engine CAN", "can0", "socketcan", "Detail", "badge-success", "component-status-row state-restarting", `href="/sources/new"`, `href="/sources/can0/"`, `href="/sources/can0/edit"`} { if !strings.Contains(body, want) { t.Fatalf("sources page missing %q:\n%s", want, body) } @@ -133,21 +134,21 @@ func TestSourcesPageRendersConfiguredEntities(t *testing.T) { func TestSourceNewPageOpensCreateForm(t *testing.T) { srv, _ := newUIServerWithService(t) - resp, err := http.Get(srv.URL + "/ui/sources/new") + resp, err := http.Get(srv.URL + "/sources/new") if err != nil { t.Fatal(err) } mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) for _, want := range []string{ - "Add source", `hx-post="/ui/sources"`, `name="id"`, `name="interface"`, - `aria-label="Breadcrumb"`, `href="/ui/sources"`, `aria-current="page">Add source`, + "Add source", `hx-post="/sources"`, `name="id"`, `name="interface"`, + `aria-label="Breadcrumb"`, `href="/sources"`, `aria-current="page">Add source`, } { if !strings.Contains(body, want) { t.Fatalf("source new page missing %q:\n%s", want, body) } } - for _, notWant := range []string{`href="/ui/sources/new"`, `id="source-panel"`} { + for _, notWant := range []string{`href="/sources/new"`, `id="source-panel"`} { if strings.Contains(body, notWant) { t.Fatalf("source new page should not include %q:\n%s", notWant, body) } @@ -160,7 +161,7 @@ func TestSourceEditPageOpensEditForm(t *testing.T) { ID: "can0", Name: "Engine CAN", Type: model.SourceSocketCAN, Enabled: true, Interface: "can0", }, true)) - resp, err := http.Get(srv.URL + "/ui/sources/can0/edit") + resp, err := http.Get(srv.URL + "/sources/can0/edit") if err != nil { t.Fatal(err) } @@ -173,20 +174,20 @@ func TestSourceEditPageOpensEditForm(t *testing.T) { `name="interface"`, `value="can0"`, `aria-label="Breadcrumb"`, - `href="/ui/sources"`, + `href="/sources"`, `aria-current="page">Edit can0`, } { if !strings.Contains(body, want) { t.Fatalf("sources page edit form missing %q:\n%s", want, body) } } - for _, notWant := range []string{`href="/ui/sources/new"`, `id="source-panel"`} { + for _, notWant := range []string{`href="/sources/new"`, `id="source-panel"`} { if strings.Contains(body, notWant) { t.Fatalf("source edit page should not include %q:\n%s", notWant, body) } } - resp2, err := http.Get(srv.URL + "/ui/sources/missing/edit") + resp2, err := http.Get(srv.URL + "/sources/missing/edit") if err != nil { t.Fatal(err) } @@ -200,7 +201,7 @@ func TestSourceOverviewPageRendersConfigAndLiveFragment(t *testing.T) { URL: "mqtt://broker.local:1883", Topic: "vessels/main/#", }, true)) - resp, err := http.Get(srv.URL + "/ui/sources/mqtt-in/") + resp, err := http.Get(srv.URL + "/sources/mqtt-in/") if err != nil { t.Fatal(err) } @@ -209,9 +210,14 @@ func TestSourceOverviewPageRendersConfigAndLiveFragment(t *testing.T) { for _, want := range []string{ "source overview", "MQTT input", "mqtt", "mqtt://broker.local:1883", "vessels/main/#", - `href="/ui/sources/mqtt-in/edit"`, - `aria-label="Breadcrumb"`, `href="/ui/sources"`, `aria-current="page">MQTT input`, - `hx-get="/ui/frag/sources/mqtt-in/overview"`, `hx-trigger="load, every 2s"`, + `href="/sources/mqtt-in/edit"`, + `aria-label="Breadcrumb"`, `href="/sources"`, `aria-current="page">MQTT input`, + `hx-get="/frag/sources/mqtt-in/overview"`, `hx-trigger="load, every 500ms"`, + `data-stream-url="/ui/streams/sources/mqtt-in"`, "Stream contents", + `data-stream-filter`, `aria-label="CEL stream filter"`, + `data-stream-start`, `data-stream-stop`, "JSONL", "CAN bytes", + `data-stream-export="json"`, `data-stream-export="can"`, `data-stream-copy`, + `data-stream-cel-inspector`, `data-stream-cel-expression`, } { if !strings.Contains(body, want) { t.Fatalf("source overview page missing %q:\n%s", want, body) @@ -225,13 +231,13 @@ func TestSinksPageRendersConfiguredEntities(t *testing.T) { ID: "out1", Name: "NMEA Out", Type: model.SinkTCP, Enabled: true, Address: "0.0.0.0:2000", }, true)) - resp, err := http.Get(srv.URL + "/ui/sinks") + resp, err := http.Get(srv.URL + "/sinks") if err != nil { t.Fatal(err) } mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) - for _, want := range []string{"NMEA Out", "out1", "tcp", "0.0.0.0:2000", "badge-success", "component-status-row state-restarting", `href="/ui/sinks/new"`, `href="/ui/sinks/out1/"`, `href="/ui/sinks/out1/edit"`} { + for _, want := range []string{"NMEA Out", "out1", "tcp", "0.0.0.0:2000", "badge-success", "component-status-row state-restarting", `href="/sinks/new"`, `href="/sinks/out1/"`, `href="/sinks/out1/edit"`} { if !strings.Contains(body, want) { t.Fatalf("sinks page missing %q:\n%s", want, body) } @@ -241,21 +247,21 @@ func TestSinksPageRendersConfiguredEntities(t *testing.T) { func TestSinkNewPageOpensCreateForm(t *testing.T) { srv, _ := newUIServerWithService(t) - resp, err := http.Get(srv.URL + "/ui/sinks/new") + resp, err := http.Get(srv.URL + "/sinks/new") if err != nil { t.Fatal(err) } mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) for _, want := range []string{ - "Add sink", `hx-post="/ui/sinks"`, `name="id"`, `name="interface"`, - `aria-label="Breadcrumb"`, `href="/ui/sinks"`, `aria-current="page">Add sink`, + "Add sink", `hx-post="/sinks"`, `name="id"`, `name="interface"`, + `aria-label="Breadcrumb"`, `href="/sinks"`, `aria-current="page">Add sink`, } { if !strings.Contains(body, want) { t.Fatalf("sink new page missing %q:\n%s", want, body) } } - for _, notWant := range []string{`href="/ui/sinks/new"`, `id="sink-panel"`} { + for _, notWant := range []string{`href="/sinks/new"`, `id="sink-panel"`} { if strings.Contains(body, notWant) { t.Fatalf("sink new page should not include %q:\n%s", notWant, body) } @@ -268,7 +274,7 @@ func TestSinkEditPageOpensEditForm(t *testing.T) { ID: "out1", Name: "NMEA Out", Type: model.SinkTCP, Enabled: true, Address: "0.0.0.0:2000", }, true)) - resp, err := http.Get(srv.URL + "/ui/sinks/out1/edit") + resp, err := http.Get(srv.URL + "/sinks/out1/edit") if err != nil { t.Fatal(err) } @@ -281,20 +287,20 @@ func TestSinkEditPageOpensEditForm(t *testing.T) { `name="address"`, `value="0.0.0.0:2000"`, `aria-label="Breadcrumb"`, - `href="/ui/sinks"`, + `href="/sinks"`, `aria-current="page">Edit out1`, } { if !strings.Contains(body, want) { t.Fatalf("sinks page edit form missing %q:\n%s", want, body) } } - for _, notWant := range []string{`href="/ui/sinks/new"`, `id="sink-panel"`} { + for _, notWant := range []string{`href="/sinks/new"`, `id="sink-panel"`} { if strings.Contains(body, notWant) { t.Fatalf("sink edit page should not include %q:\n%s", notWant, body) } } - resp2, err := http.Get(srv.URL + "/ui/sinks/missing/edit") + resp2, err := http.Get(srv.URL + "/sinks/missing/edit") if err != nil { t.Fatal(err) } @@ -308,7 +314,7 @@ func TestSinkOverviewPageRendersConfigAndLiveFragment(t *testing.T) { URL: "mqtt://broker.local:1883", Topic: "vessels/main/json", }, true)) - resp, err := http.Get(srv.URL + "/ui/sinks/mqtt-out/") + resp, err := http.Get(srv.URL + "/sinks/mqtt-out/") if err != nil { t.Fatal(err) } @@ -317,9 +323,14 @@ func TestSinkOverviewPageRendersConfigAndLiveFragment(t *testing.T) { for _, want := range []string{ "sink overview", "MQTT output", "mqtt", "mqtt://broker.local:1883", "vessels/main/json", - `href="/ui/sinks/mqtt-out/edit"`, - `aria-label="Breadcrumb"`, `href="/ui/sinks"`, `aria-current="page">MQTT output`, - `hx-get="/ui/frag/sinks/mqtt-out/overview"`, `hx-trigger="load, every 2s"`, + `href="/sinks/mqtt-out/edit"`, + `aria-label="Breadcrumb"`, `href="/sinks"`, `aria-current="page">MQTT output`, + `hx-get="/frag/sinks/mqtt-out/overview"`, `hx-trigger="load, every 500ms"`, + `data-stream-url="/ui/streams/sinks/mqtt-out"`, "Stream contents", + `data-stream-filter`, `aria-label="CEL stream filter"`, + `data-stream-start`, `data-stream-stop`, "JSONL", "CAN bytes", + `data-stream-export="json"`, `data-stream-export="can"`, `data-stream-copy`, + `data-stream-cel-inspector`, `data-stream-cel-expression`, } { if !strings.Contains(body, want) { t.Fatalf("sink overview page missing %q:\n%s", want, body) @@ -327,6 +338,140 @@ func TestSinkOverviewPageRendersConfigAndLiveFragment(t *testing.T) { } } +func TestComponentStreamSendsFutureCanonicalEnvelope(t *testing.T) { + srv, svc, reg := newUIServerWithServiceAndRegistry(t) + must(t, svc.PutSource(context.Background(), model.Source{ + ID: "src1", Name: "Source One", Type: model.SourceSocketCAN, Enabled: true, Interface: "can0", + }, true)) + + streamContext, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(streamContext, http.MethodGet, srv.URL+"/ui/streams/sources/src1", nil) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + mustStatus(t, resp, http.StatusOK) + if got := resp.Header.Get("Content-Type"); got != "text/event-stream" { + t.Fatalf("Content-Type = %q, want text/event-stream", got) + } + + payload := json.RawMessage(`{"info":{"timestamp":"2026-06-29T10:10:17Z","priority":5,"pgn":130314,"sourceId":6,"targetId":null},"instance":0,"source":0,"pressure":1020690}`) + reg.RecordSource("src1", &msg.Envelope{ + PGN: 130314, Source: 6, Dest: 255, Priority: 5, + Payload: payload, + Raw: []byte{0xff, 0x00, 0x00, 0x12}, + }) + + reader := bufio.NewReader(resp.Body) + var document string + for document == "" { + line, readErr := reader.ReadString('\n') + if readErr != nil { + t.Fatal(readErr) + } + if strings.HasPrefix(line, "data: ") { + document = strings.TrimSpace(strings.TrimPrefix(line, "data: ")) + } + } + var wire struct { + Payload json.RawMessage `json:"payload"` + Raw []byte `json:"raw"` + } + if err := json.Unmarshal([]byte(document), &wire); err != nil { + t.Fatal(err) + } + if string(wire.Payload) != string(payload) { + t.Fatalf("stream payload = %s, want verbatim %s", wire.Payload, payload) + } + if got, want := base64.StdEncoding.EncodeToString(wire.Raw), "/wAAEg=="; got != want { + t.Fatalf("stream raw = %s, want %s", got, want) + } +} + +func TestComponentStreamAppliesCELFilter(t *testing.T) { + srv, svc, reg := newUIServerWithServiceAndRegistry(t) + must(t, svc.PutSource(context.Background(), model.Source{ + ID: "src1", Name: "Source One", Type: model.SourceSocketCAN, Enabled: true, Interface: "can0", + }, true)) + + expression := `msg.pgn == 130314 && msg.payload.pressure == 1020690` + streamContext, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + req, err := http.NewRequestWithContext( + streamContext, + http.MethodGet, + srv.URL+"/ui/streams/sources/src1?filter="+url.QueryEscape(expression), + nil, + ) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + mustStatus(t, resp, http.StatusOK) + + reg.RecordSource("src1", &msg.Envelope{ + PGN: 127250, Source: 6, Dest: 255, Priority: 5, + Payload: json.RawMessage(`{"info":{"timestamp":"2026-06-29T10:10:17Z","priority":5,"pgn":127250,"sourceId":6,"targetId":null},"pressure":1}`), + Raw: []byte{0x01}, + }) + reg.RecordSource("src1", &msg.Envelope{ + PGN: 130314, Source: 6, Dest: 255, Priority: 5, + Payload: json.RawMessage(`{"info":{"timestamp":"2026-06-29T10:10:17Z","priority":5,"pgn":130314,"sourceId":6,"targetId":null},"pressure":1020690}`), + Raw: []byte{0xff, 0x00, 0x00, 0x12}, + }) + + reader := bufio.NewReader(resp.Body) + var document string + for document == "" { + line, readErr := reader.ReadString('\n') + if readErr != nil { + t.Fatal(readErr) + } + if strings.HasPrefix(line, "data: ") { + document = strings.TrimSpace(strings.TrimPrefix(line, "data: ")) + } + } + if !strings.Contains(document, `"pgn":130314`) || !strings.Contains(document, `"pressure":1020690`) { + t.Fatalf("filtered stream document = %s, want matching envelope", document) + } + if strings.Contains(document, `"pgn":127250`) { + t.Fatalf("filtered stream document contains non-matching envelope: %s", document) + } +} + +func TestComponentStreamRejectsInvalidCELFilter(t *testing.T) { + srv, svc, _ := newUIServerWithServiceAndRegistry(t) + must(t, svc.PutSource(context.Background(), model.Source{ + ID: "src1", Name: "Source One", Type: model.SourceSocketCAN, Enabled: true, Interface: "can0", + }, true)) + + resp, err := http.Get(srv.URL + "/ui/streams/sources/src1?filter=" + url.QueryEscape("msg.pgn == @")) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + mustStatus(t, resp, http.StatusBadRequest) +} + +func TestComponentStreamRejectsUnknownEntity(t *testing.T) { + srv, _, _ := newUIServerWithServiceAndRegistry(t) + resp, err := http.Get(srv.URL + "/ui/streams/sinks/missing") + if err != nil { + t.Fatal(err) + } + mustStatus(t, resp, http.StatusNotFound) + _ = resp.Body.Close() +} + // --- Type-fields fragment per type --- func TestSourceTypeFieldsFragmentPerType(t *testing.T) { @@ -346,7 +491,7 @@ func TestSourceTypeFieldsFragmentPerType(t *testing.T) { } for _, tc := range cases { t.Run(tc.typ, func(t *testing.T) { - resp, err := http.Get(srv.URL + "/ui/frag/source-type-fields?type=" + tc.typ) + resp, err := http.Get(srv.URL + "/frag/source-type-fields?type=" + tc.typ) if err != nil { t.Fatal(err) } @@ -366,7 +511,7 @@ func TestSourceTypeFieldsFragmentPerType(t *testing.T) { // type: another type's fields aren't in the DOM to be resent, so their // values do NOT survive an A→B→A type switch — deliberate, see // sourceTypeFieldsData's doc comment in forms.go. - resp, err := http.Get(srv.URL + "/ui/frag/source-type-fields?type=socketcan&interface=can5") + resp, err := http.Get(srv.URL + "/frag/source-type-fields?type=socketcan&interface=can5") if err != nil { t.Fatal(err) } @@ -375,7 +520,7 @@ func TestSourceTypeFieldsFragmentPerType(t *testing.T) { t.Fatalf("type-fields fragment did not preserve interface value:\n%s", body) } - resp, err = http.Get(srv.URL + "/ui/frag/source-type-fields?type=udp&address=0.0.0.0%3A1457&format=actisense") + resp, err = http.Get(srv.URL + "/frag/source-type-fields?type=udp&address=0.0.0.0%3A1457&format=actisense") if err != nil { t.Fatal(err) } @@ -394,7 +539,7 @@ func TestSourceOverviewShowsGatewayConfiguration(t *testing.T) { Address: "0.0.0.0:1457", Format: model.StreamFormatActisense, }, true)) - resp, err := http.Get(srv.URL + "/ui/sources/gateway-in/") + resp, err := http.Get(srv.URL + "/sources/gateway-in/") if err != nil { t.Fatal(err) } @@ -423,7 +568,7 @@ func TestSinkTypeFieldsFragmentPerType(t *testing.T) { } for _, tc := range cases { t.Run(tc.typ, func(t *testing.T) { - resp, err := http.Get(srv.URL + "/ui/frag/sink-type-fields?type=" + tc.typ) + resp, err := http.Get(srv.URL + "/frag/sink-type-fields?type=" + tc.typ) if err != nil { t.Fatal(err) } @@ -446,7 +591,7 @@ func TestSourceEditPagePreFillsAndLocksID(t *testing.T) { ID: "can0", Name: "Engine", Type: model.SourceSocketCAN, Interface: "can0", Enabled: true, }, true)) - resp, err := http.Get(srv.URL + "/ui/sources/can0/edit") + resp, err := http.Get(srv.URL + "/sources/can0/edit") if err != nil { t.Fatal(err) } @@ -458,7 +603,7 @@ func TestSourceEditPagePreFillsAndLocksID(t *testing.T) { } } - resp2, err := http.Get(srv.URL + "/ui/sources/doesnotexist/edit") + resp2, err := http.Get(srv.URL + "/sources/doesnotexist/edit") if err != nil { t.Fatal(err) } @@ -480,8 +625,8 @@ func assertCreateRedirect(t *testing.T, resp *http.Response, wantMsg string) { if resp.StatusCode != http.StatusOK { t.Fatalf("create status = %d, want 200", resp.StatusCode) } - if got := resp.Header.Get("HX-Redirect"); got != "/ui/dashboard" { - t.Fatalf("HX-Redirect = %q, want /ui/dashboard", got) + if got := resp.Header.Get("HX-Redirect"); got != "/dashboard" { + t.Fatalf("HX-Redirect = %q, want /dashboard", got) } for _, c := range resp.Cookies() { if c.Name != flashCookieName { @@ -501,7 +646,7 @@ func assertCreateRedirect(t *testing.T, resp *http.Response, wantMsg string) { func TestSourceCreateRoundTrip(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sources", url.Values{ + resp := postForm(t, srv, "/sources", url.Values{ "id": {"can0"}, "name": {"Engine CAN"}, "type": {"socketcan"}, "enabled": {"1"}, "interface": {"can0"}, }) @@ -517,12 +662,12 @@ func TestSourceCreateRoundTrip(t *testing.T) { t.Fatalf("persisted source = %+v", got) } - resp2, err := http.Get(srv.URL + "/ui/sources") + resp2, err := http.Get(srv.URL + "/sources") if err != nil { t.Fatal(err) } if body2 := mustBody(t, resp2); !strings.Contains(body2, "Engine CAN") { - t.Fatalf("GET /ui/sources does not reflect created source:\n%s", body2) + t.Fatalf("GET /sources does not reflect created source:\n%s", body2) } } @@ -532,7 +677,7 @@ func TestSourceUpdateRoundTrip(t *testing.T) { ID: "can0", Name: "Old Name", Type: model.SourceSocketCAN, Interface: "can0", }, true)) - resp := postForm(t, srv, "/ui/sources/can0", url.Values{ + resp := postForm(t, srv, "/sources/can0", url.Values{ "id": {"can0"}, "name": {"New Name"}, "type": {"socketcan"}, "interface": {"can1"}, }) mustStatus(t, resp, http.StatusOK) @@ -552,7 +697,7 @@ func TestSourceUpdateRoundTrip(t *testing.T) { func TestSourceHeadersRoundTrip(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sources", url.Values{ + resp := postForm(t, srv, "/sources", url.Values{ "id": {"sse1"}, "name": {"SSE"}, "type": {"http_sse"}, "url": {"https://example.com/stream"}, "headers": {"Authorization: Bearer tok\nX-Custom: value"}, @@ -567,7 +712,7 @@ func TestSourceHeadersRoundTrip(t *testing.T) { t.Fatalf("parsed headers = %+v", got.Headers) } - resp2, err := http.Get(srv.URL + "/ui/sources/sse1/edit") + resp2, err := http.Get(srv.URL + "/sources/sse1/edit") if err != nil { t.Fatal(err) } @@ -579,7 +724,7 @@ func TestSourceHeadersRoundTrip(t *testing.T) { func TestMQTTSourceCreateRoundTrip(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sources", url.Values{ + resp := postForm(t, srv, "/sources", url.Values{ "id": {"mqtt-in"}, "name": {"MQTT input"}, "type": {"mqtt"}, "enabled": {"1"}, "url": {"mqtt://broker.local:1883"}, "topic": {"vessels/main/engine/#"}, }) @@ -593,7 +738,7 @@ func TestMQTTSourceCreateRoundTrip(t *testing.T) { t.Fatalf("persisted source = %+v", got) } - resp2, err := http.Get(srv.URL + "/ui/sources/mqtt-in/edit") + resp2, err := http.Get(srv.URL + "/sources/mqtt-in/edit") if err != nil { t.Fatal(err) } @@ -607,7 +752,7 @@ func TestMQTTSourceCreateRoundTrip(t *testing.T) { func TestSinkCreateRoundTrip(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"out1"}, "name": {"NMEA Out"}, "type": {"tcp"}, "enabled": {"1"}, "address": {"0.0.0.0:2000"}, }) @@ -624,7 +769,7 @@ func TestSinkCreateRoundTrip(t *testing.T) { func TestMQTTSinkCreateRoundTrip(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"mqtt-out"}, "name": {"MQTT output"}, "type": {"mqtt"}, "enabled": {"1"}, "url": {"mqtt://broker.local:1883"}, "topic": {"vessels/main/engine/json"}, }) @@ -638,7 +783,7 @@ func TestMQTTSinkCreateRoundTrip(t *testing.T) { t.Fatalf("persisted sink = %+v", got) } - resp2, err := http.Get(srv.URL + "/ui/sinks/mqtt-out/edit") + resp2, err := http.Get(srv.URL + "/sinks/mqtt-out/edit") if err != nil { t.Fatal(err) } @@ -652,7 +797,7 @@ func TestMQTTSinkCreateRoundTrip(t *testing.T) { func TestTCPGatewaySinkCreateRoundTrip(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"gw-out"}, "name": {"YD gateway out"}, "type": {"tcp_gateway"}, "enabled": {"1"}, "address": {"192.168.4.1:1457"}, "format": {"ydraw"}, }) @@ -666,7 +811,7 @@ func TestTCPGatewaySinkCreateRoundTrip(t *testing.T) { t.Fatalf("persisted sink = %+v", got) } - resp2, err := http.Get(srv.URL + "/ui/sinks/gw-out/edit") + resp2, err := http.Get(srv.URL + "/sinks/gw-out/edit") if err != nil { t.Fatal(err) } @@ -689,7 +834,7 @@ func TestTCPGatewaySinkCreateRoundTrip(t *testing.T) { // parseOptionalInt turn into 0, not an error. func TestFileSinkCreateRoundTrip(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"navlog"}, "name": {"Nav log"}, "type": {"file"}, "enabled": {"1"}, "file_path": {"/data/nav.log"}, "format": {"ndjson"}, }) @@ -706,7 +851,7 @@ func TestFileSinkCreateRoundTrip(t *testing.T) { // The edit form round-trips file_path/format and shows the defaults as // placeholders (blank stored value -> blank input, default shown only // as a placeholder, not baked into the value). - resp2, err := http.Get(srv.URL + "/ui/sinks/navlog/edit") + resp2, err := http.Get(srv.URL + "/sinks/navlog/edit") if err != nil { t.Fatal(err) } @@ -729,7 +874,7 @@ func TestFileSinkCreateRoundTrip(t *testing.T) { // the edit form as their literal values, not the defaults. func TestFileSinkMaxFileBytesAndMaxFilesRoundTrip(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"navlog"}, "name": {"Nav log"}, "type": {"file"}, "file_path": {"/data/nav.candump"}, "format": {"candump"}, "max_file_bytes": {"52428800"}, "max_files": {"3"}, @@ -744,7 +889,7 @@ func TestFileSinkMaxFileBytesAndMaxFilesRoundTrip(t *testing.T) { t.Fatalf("persisted sink = %+v", got) } - resp2, err := http.Get(srv.URL + "/ui/sinks/navlog/edit") + resp2, err := http.Get(srv.URL + "/sinks/navlog/edit") if err != nil { t.Fatal(err) } @@ -766,7 +911,7 @@ func TestFileSinkMaxFileBytesAndMaxFilesRoundTrip(t *testing.T) { // validation failure follows (see TestSinkCreateValidationErrorRendersFormNot500). func TestFileSinkCreateRelativePathRendersFormNot500(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"navlog"}, "name": {"Nav log"}, "type": {"file"}, "file_path": {"relative/nav.log"}, "format": {"ndjson"}, }) @@ -790,7 +935,7 @@ func TestFileSinkCreateRelativePathRendersFormNot500(t *testing.T) { // for the connector form's buffer fields. func TestFileSinkNonNumericMaxFileBytesRendersFormNot500(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"navlog"}, "name": {"Nav log"}, "type": {"file"}, "file_path": {"/data/nav.log"}, "format": {"ndjson"}, "max_file_bytes": {"not-a-number"}, @@ -814,7 +959,7 @@ func TestFileSinkNonNumericMaxFileBytesRendersFormNot500(t *testing.T) { // — see forms.go). func TestFileSinkNonNumericMaxFilesRendersFormNot500(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"navlog"}, "name": {"Nav log"}, "type": {"file"}, "file_path": {"/data/nav.log"}, "format": {"ndjson"}, "max_files": {"not-a-number"}, @@ -843,7 +988,7 @@ func TestSinksPageShowsFilePathDetailForFileSink(t *testing.T) { FilePath: "/data/nav.log", Format: model.FileFormatNDJSON, }, true)) - resp, err := http.Get(srv.URL + "/ui/sinks") + resp, err := http.Get(srv.URL + "/sinks") if err != nil { t.Fatal(err) } @@ -861,7 +1006,7 @@ func TestSinksPageShowsFilePathDetailForFileSink(t *testing.T) { func TestSourceCreateValidationErrorRendersFormNot500(t *testing.T) { srv, svc := newUIServerWithService(t) // socketcan requires an interface; omit it. - resp := postForm(t, srv, "/ui/sources", url.Values{ + resp := postForm(t, srv, "/sources", url.Values{ "id": {"bad"}, "name": {"Bad Source"}, "type": {"socketcan"}, }) mustStatus(t, resp, http.StatusOK) @@ -889,7 +1034,7 @@ func TestSourceCreateValidationErrorRendersFormNot500(t *testing.T) { // sink logic (see forms.go's package doc comment). func TestSourceCreateParseFormErrorRendersAlertNot500(t *testing.T) { srv, svc := newUIServerWithService(t) - req, err := http.NewRequest(http.MethodPost, srv.URL+"/ui/sources?%zz", + req, err := http.NewRequest(http.MethodPost, srv.URL+"/sources?%zz", strings.NewReader(url.Values{"id": {"bad"}, "name": {"Bad"}, "type": {"socketcan"}}.Encode())) if err != nil { t.Fatal(err) @@ -911,7 +1056,7 @@ func TestSourceCreateParseFormErrorRendersAlertNot500(t *testing.T) { func TestSourceCreateMalformedHeadersRendersFormNot500(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sources", url.Values{ + resp := postForm(t, srv, "/sources", url.Values{ "id": {"sse1"}, "name": {"SSE Source"}, "type": {"http_sse"}, "url": {"https://example.com/stream"}, "headers": {"not-a-valid-header-line"}, @@ -935,7 +1080,7 @@ func TestSourceCreateExistingIDRendersFormNot500(t *testing.T) { ID: "can0", Name: "Engine", Type: model.SourceSocketCAN, Interface: "can0", }, true)) - resp := postForm(t, srv, "/ui/sources", url.Values{ + resp := postForm(t, srv, "/sources", url.Values{ "id": {"can0"}, "name": {"Dup"}, "type": {"socketcan"}, "interface": {"can1"}, }) mustStatus(t, resp, http.StatusOK) @@ -947,7 +1092,7 @@ func TestSourceCreateExistingIDRendersFormNot500(t *testing.T) { func TestSinkCreateValidationErrorRendersFormNot500(t *testing.T) { srv, svc := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"bad"}, "name": {"Bad Sink"}, "type": {"tcp"}, "address": {"not-a-valid-address"}, }) mustStatus(t, resp, http.StatusOK) @@ -972,7 +1117,7 @@ func TestSourceDeleteAndDeleteInUse(t *testing.T) { must(t, svc.PutSink(ctx, model.Sink{ID: "sink1", Name: "Sink One", Type: model.SinkTCP, Address: "127.0.0.1:9000"}, true)) must(t, svc.PutConnector(ctx, model.Connector{ID: "conn1", Name: "Conn One", SourceID: "src1", SinkID: "sink1"}, true)) - resp := postForm(t, srv, "/ui/sources/src1/delete", nil) + resp := postForm(t, srv, "/sources/src1/delete", nil) mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) if !strings.Contains(body, "conn1") || !strings.Contains(body, "alert-error") { @@ -988,7 +1133,7 @@ func TestSourceDeleteAndDeleteInUse(t *testing.T) { } must(t, svc.DeleteConnector(ctx, "conn1")) - resp2 := postForm(t, srv, "/ui/sources/src1/delete", nil) + resp2 := postForm(t, srv, "/sources/src1/delete", nil) mustStatus(t, resp2, http.StatusOK) body2 := mustBody(t, resp2) if !strings.Contains(body2, "alert-success") { @@ -1012,7 +1157,7 @@ func TestSinkDeleteAndDeleteInUse(t *testing.T) { must(t, svc.PutSink(ctx, model.Sink{ID: "sink1", Name: "Sink One", Type: model.SinkTCP, Address: "127.0.0.1:9000"}, true)) must(t, svc.PutConnector(ctx, model.Connector{ID: "conn1", Name: "Conn One", SourceID: "src1", SinkID: "sink1"}, true)) - resp := postForm(t, srv, "/ui/sinks/sink1/delete", nil) + resp := postForm(t, srv, "/sinks/sink1/delete", nil) mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) if !strings.Contains(body, "conn1") || !strings.Contains(body, "alert-error") { @@ -1020,7 +1165,7 @@ func TestSinkDeleteAndDeleteInUse(t *testing.T) { } must(t, svc.DeleteConnector(ctx, "conn1")) - resp2 := postForm(t, srv, "/ui/sinks/sink1/delete", nil) + resp2 := postForm(t, srv, "/sinks/sink1/delete", nil) mustStatus(t, resp2, http.StatusOK) body2 := mustBody(t, resp2) if !strings.Contains(body2, "alert-success") { @@ -1033,7 +1178,7 @@ func TestSinkDeleteAndDeleteInUse(t *testing.T) { func TestSourceDeleteUnknownID(t *testing.T) { srv, _ := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sources/nope/delete", nil) + resp := postForm(t, srv, "/sources/nope/delete", nil) mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) if !strings.Contains(body, "alert-error") || !strings.Contains(body, "not found") { @@ -1051,7 +1196,7 @@ func TestConnectorsPageRendersConfiguredEntities(t *testing.T) { Filters: []string{"msg.pgn == 127250"}, Enabled: true, }, true)) - resp, err := http.Get(srv.URL + "/ui/connectors") + resp, err := http.Get(srv.URL + "/connectors") if err != nil { t.Fatal(err) } @@ -1061,7 +1206,7 @@ func TestConnectorsPageRendersConfiguredEntities(t *testing.T) { // One" — seedSourceSink's names), not their raw ids: see // TestConnectorsPageNameFallsBackToIDWhenEmpty for the fallback case. for _, want := range []string{ - `href="/ui/connectors/new"`, `href="/ui/connectors/conn1/"`, `href="/ui/connectors/conn1/edit"`, "NMEA Bridge", + `href="/connectors/new"`, `href="/connectors/conn1/"`, `href="/connectors/conn1/edit"`, "NMEA Bridge", "Source One", "Sink One", "badge-success", "component-status-row state-restarting", ">restarting<", } { @@ -1089,7 +1234,7 @@ func TestConnectorsPageNameFallsBackToIDWhenEmpty(t *testing.T) { must(t, svc.PutSink(ctx, model.Sink{ID: "sink1", Type: model.SinkTCP, Address: "127.0.0.1:9000"}, true)) must(t, svc.PutConnector(ctx, model.Connector{ID: "conn1", Name: "Bridge", SourceID: "src1", SinkID: "sink1"}, true)) - resp, err := http.Get(srv.URL + "/ui/connectors") + resp, err := http.Get(srv.URL + "/connectors") if err != nil { t.Fatal(err) } @@ -1104,7 +1249,7 @@ func TestConnectorNewPageOpensCreateForm(t *testing.T) { srv, svc := newUIServerWithService(t) seedSourceSink(t, svc) - resp, err := http.Get(srv.URL + "/ui/connectors/new") + resp, err := http.Get(srv.URL + "/connectors/new") if err != nil { t.Fatal(err) } @@ -1112,7 +1257,7 @@ func TestConnectorNewPageOpensCreateForm(t *testing.T) { body := mustBody(t, resp) for _, want := range []string{ "Add connector", - `hx-post="/ui/connectors"`, + `hx-post="/connectors"`, `name="id"`, `value="src1"`, `value="sink1"`, @@ -1122,7 +1267,7 @@ func TestConnectorNewPageOpensCreateForm(t *testing.T) { `role="listbox"`, `aria-keyshortcuts="Control+Space"`, `aria-label="Breadcrumb"`, - `href="/ui/connectors"`, + `href="/connectors"`, `aria-current="page">Add connector`, } { if !strings.Contains(body, want) { @@ -1134,7 +1279,7 @@ func TestConnectorNewPageOpensCreateForm(t *testing.T) { func TestCELCompletionCatalogIncludesSchemaFieldsAndPGNs(t *testing.T) { srv, _ := newUIServerWithService(t) - resp, err := http.Get(srv.URL + "/ui/cel-completions") + resp, err := http.Get(srv.URL + "/cel-completions") if err != nil { t.Fatal(err) } @@ -1163,7 +1308,7 @@ func TestConnectorEditPageOpensEditForm(t *testing.T) { Enabled: true, }, true)) - resp, err := http.Get(srv.URL + "/ui/connectors/conn1/edit") + resp, err := http.Get(srv.URL + "/connectors/conn1/edit") if err != nil { t.Fatal(err) } @@ -1181,7 +1326,7 @@ func TestConnectorEditPageOpensEditForm(t *testing.T) { `value="24h0m0s"`, `value="1048576"`, `aria-label="Breadcrumb"`, - `href="/ui/connectors"`, + `href="/connectors"`, `aria-current="page">Edit conn1`, } { if !strings.Contains(body, want) { @@ -1189,7 +1334,7 @@ func TestConnectorEditPageOpensEditForm(t *testing.T) { } } - resp2, err := http.Get(srv.URL + "/ui/connectors/doesnotexist/edit") + resp2, err := http.Get(srv.URL + "/connectors/doesnotexist/edit") if err != nil { t.Fatal(err) } @@ -1206,7 +1351,7 @@ func TestConnectorEditPagePreFillsAndLocksID(t *testing.T) { Enabled: true, }, true)) - resp, err := http.Get(srv.URL + "/ui/connectors/conn1/edit") + resp, err := http.Get(srv.URL + "/connectors/conn1/edit") if err != nil { t.Fatal(err) } @@ -1223,7 +1368,7 @@ func TestConnectorEditPagePreFillsAndLocksID(t *testing.T) { } } - resp2, err := http.Get(srv.URL + "/ui/connectors/doesnotexist/edit") + resp2, err := http.Get(srv.URL + "/connectors/doesnotexist/edit") if err != nil { t.Fatal(err) } @@ -1234,7 +1379,7 @@ func TestConnectorCreateRoundTrip(t *testing.T) { srv, svc := newUIServerWithService(t) seedSourceSink(t, svc) - resp := postForm(t, srv, "/ui/connectors", url.Values{ + resp := postForm(t, srv, "/connectors", url.Values{ "id": {"conn1"}, "name": {"NMEA Bridge"}, "source_id": {"src1"}, "sink_id": {"sink1"}, "enabled": {"1"}, "filters": {"msg.pgn == 127250\n\n msg.priority < 4 \n"}, @@ -1256,12 +1401,12 @@ func TestConnectorCreateRoundTrip(t *testing.T) { t.Fatalf("persisted connector = %+v", got) } - resp2, err := http.Get(srv.URL + "/ui/connectors") + resp2, err := http.Get(srv.URL + "/connectors") if err != nil { t.Fatal(err) } if body2 := mustBody(t, resp2); !strings.Contains(body2, "NMEA Bridge") { - t.Fatalf("GET /ui/connectors does not reflect created connector:\n%s", body2) + t.Fatalf("GET /connectors does not reflect created connector:\n%s", body2) } } @@ -1270,7 +1415,7 @@ func TestConnectorCreateRoundTrip(t *testing.T) { // cookie, so a subsequent load never shows it again. func TestCreateFlashShownOnDashboardOnce(t *testing.T) { srv, _ := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/sinks", url.Values{ + resp := postForm(t, srv, "/sinks", url.Values{ "id": {"out1"}, "name": {"NMEA Out"}, "type": {"tcp"}, "enabled": {"1"}, "address": {"0.0.0.0:2000"}, }) @@ -1286,7 +1431,7 @@ func TestCreateFlashShownOnDashboardOnce(t *testing.T) { } // The dashboard (the redirect target) renders the flash once... - req, _ := http.NewRequest(http.MethodGet, srv.URL+"/ui/dashboard", nil) + req, _ := http.NewRequest(http.MethodGet, srv.URL+"/dashboard", nil) req.AddCookie(flash) resp2, err := http.DefaultClient.Do(req) if err != nil { @@ -1309,7 +1454,7 @@ func TestCreateFlashShownOnDashboardOnce(t *testing.T) { } // A dashboard load without the cookie shows no flash at all. - resp3, err := http.Get(srv.URL + "/ui/dashboard") + resp3, err := http.Get(srv.URL + "/dashboard") if err != nil { t.Fatal(err) } @@ -1325,7 +1470,7 @@ func TestConnectorUpdateRoundTrip(t *testing.T) { ID: "conn1", Name: "Old Name", SourceID: "src1", SinkID: "sink1", }, true)) - resp := postForm(t, srv, "/ui/connectors/conn1", url.Values{ + resp := postForm(t, srv, "/connectors/conn1", url.Values{ "id": {"conn1"}, "name": {"New Name"}, "source_id": {"src1"}, "sink_id": {"sink1"}, "max_age": {"90s"}, }) @@ -1348,7 +1493,7 @@ func TestConnectorMaxAgeParseErrorRendersFormNot500(t *testing.T) { srv, svc := newUIServerWithService(t) seedSourceSink(t, svc) - resp := postForm(t, srv, "/ui/connectors", url.Values{ + resp := postForm(t, srv, "/connectors", url.Values{ "id": {"conn1"}, "name": {"Bad Connector"}, "source_id": {"src1"}, "sink_id": {"sink1"}, "max_age": {"notaduration"}, }) @@ -1377,7 +1522,7 @@ func TestConnectorMaxAgeParseErrorPreservesSelectionsAndFilters(t *testing.T) { srv, svc := newUIServerWithService(t) seedSourceSink(t, svc) - resp := postForm(t, srv, "/ui/connectors", url.Values{ + resp := postForm(t, srv, "/connectors", url.Values{ "id": {"conn1"}, "name": {"Bad Connector"}, "source_id": {"src1"}, "sink_id": {"sink1"}, "filters": {"msg.pgn == 127250\nmsg.priority == 4"}, "max_age": {"notaduration"}, @@ -1404,7 +1549,7 @@ func TestConnectorCreateValidationErrorRendersFormNot500(t *testing.T) { srv, svc := newUIServerWithService(t) seedSourceSink(t, svc) // source_id is required (model.Connector.Validate); omit it. - resp := postForm(t, srv, "/ui/connectors", url.Values{ + resp := postForm(t, srv, "/connectors", url.Values{ "id": {"conn1"}, "name": {"No Source"}, "sink_id": {"sink1"}, }) mustStatus(t, resp, http.StatusOK) @@ -1427,7 +1572,7 @@ func TestConnectorCreateExistingIDRendersFormNot500(t *testing.T) { ID: "conn1", Name: "Existing", SourceID: "src1", SinkID: "sink1", }, true)) - resp := postForm(t, srv, "/ui/connectors", url.Values{ + resp := postForm(t, srv, "/connectors", url.Values{ "id": {"conn1"}, "name": {"Dup"}, "source_id": {"src1"}, "sink_id": {"sink1"}, }) mustStatus(t, resp, http.StatusOK) @@ -1443,7 +1588,7 @@ func TestConnectorDeleteRoundTripAndUnknownID(t *testing.T) { ctx := context.Background() must(t, svc.PutConnector(ctx, model.Connector{ID: "conn1", Name: "Conn One", SourceID: "src1", SinkID: "sink1"}, true)) - resp := postForm(t, srv, "/ui/connectors/conn1/delete", nil) + resp := postForm(t, srv, "/connectors/conn1/delete", nil) mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) if !strings.Contains(body, "alert-success") { @@ -1452,7 +1597,7 @@ func TestConnectorDeleteRoundTripAndUnknownID(t *testing.T) { if !strings.Contains(body, `role="alert" data-autodismiss`) { t.Fatalf("delete success alert missing the auto-dismiss marker:\n%s", body) } - if strings.Contains(body, "Conn One") || strings.Contains(body, "/ui/connectors/conn1") { + if strings.Contains(body, "Conn One") || strings.Contains(body, "/connectors/conn1") { t.Fatalf("connector row should be gone after delete:\n%s", body) } if _, err := svc.GetConnector(ctx, "conn1"); !errors.Is(err, config.ErrNotFound) { @@ -1462,7 +1607,7 @@ func TestConnectorDeleteRoundTripAndUnknownID(t *testing.T) { // No ErrInUse case for connector delete (config.Service: // "nothing else references a connector") — the only failure mode left // to exercise is unknown id. - resp2 := postForm(t, srv, "/ui/connectors/nope/delete", nil) + resp2 := postForm(t, srv, "/connectors/nope/delete", nil) mustStatus(t, resp2, http.StatusOK) body2 := mustBody(t, resp2) if !strings.Contains(body2, "alert-error") || !strings.Contains(body2, "not found") { @@ -1475,14 +1620,14 @@ func TestConnectorDeleteRoundTripAndUnknownID(t *testing.T) { func TestValidateFiltersFragmentHappyAndError(t *testing.T) { srv, _ := newUIServerWithService(t) - resp := postForm(t, srv, "/ui/frag/validate-filters", url.Values{"filters": {"msg.pgn == 127250"}}) + resp := postForm(t, srv, "/frag/validate-filters", url.Values{"filters": {"msg.pgn == 127250"}}) mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) if !strings.Contains(body, "filters OK") { t.Fatalf("expected a filters-OK line for a valid expression:\n%s", body) } - resp2 := postForm(t, srv, "/ui/frag/validate-filters", url.Values{"filters": {"not a valid ((( expr"}}) + resp2 := postForm(t, srv, "/frag/validate-filters", url.Values{"filters": {"not a valid ((( expr"}}) mustStatus(t, resp2, http.StatusOK) body2 := mustBody(t, resp2) if !strings.Contains(body2, "alert-error") { @@ -1493,7 +1638,7 @@ func TestValidateFiltersFragmentHappyAndError(t *testing.T) { func TestValidateFiltersJSONReturnsEditorRanges(t *testing.T) { srv, _ := newUIServerWithService(t) text := "msg.timestamp == \"😁\"\n msg.source == @" - req, err := http.NewRequest(http.MethodPost, srv.URL+"/ui/frag/validate-filters", strings.NewReader(url.Values{"filters": {text}}.Encode())) + req, err := http.NewRequest(http.MethodPost, srv.URL+"/frag/validate-filters", strings.NewReader(url.Values{"filters": {text}}.Encode())) if err != nil { t.Fatal(err) } @@ -1542,7 +1687,7 @@ func TestConnectorStatsFragmentRendersSnapshotNumbers(t *testing.T) { reg.Record("conn1", 42, 20480) reg.SetQueue("conn1", 7, 4096) - resp, err := http.Get(srv.URL + "/ui/frag/connectors/conn1/stats") + resp, err := http.Get(srv.URL + "/frag/connectors/conn1/stats") if err != nil { t.Fatal(err) } @@ -1561,7 +1706,7 @@ func TestConnectorStatsFragmentRendersSnapshotNumbers(t *testing.T) { // (swapped outerHTML into connector_detail.html's #connector-stats), // not on a stable parent — see TestConnectorStatsFragmentUnknownID // DeletedNoticeHaltsPolling for why that matters. - if !strings.Contains(body, `hx-get="/ui/frag/connectors/conn1/stats"`) || !strings.Contains(body, `hx-trigger="load, every 2s"`) { + if !strings.Contains(body, `hx-get="/frag/connectors/conn1/stats"`) || !strings.Contains(body, `hx-trigger="load, every 2s"`) { t.Fatalf("stats fragment missing its own polling attributes:\n%s", body) } } @@ -1578,7 +1723,7 @@ func TestConnectorStatsFragmentRendersSparkline(t *testing.T) { reg.SetQueue("conn1", 2, 200) reg.SetQueue("conn1", 9, 900) - resp, err := http.Get(srv.URL + "/ui/frag/connectors/conn1/stats") + resp, err := http.Get(srv.URL + "/frag/connectors/conn1/stats") if err != nil { t.Fatal(err) } @@ -1610,7 +1755,7 @@ func TestConnectorStatsFragmentRendersSparkline(t *testing.T) { func TestConnectorStatsFragmentUnknownIDDeletedNoticeHaltsPolling(t *testing.T) { srv, _ := newUIServerWithService(t) - resp, err := http.Get(srv.URL + "/ui/frag/connectors/doesnotexist/stats") + resp, err := http.Get(srv.URL + "/frag/connectors/doesnotexist/stats") if err != nil { t.Fatal(err) } @@ -1627,7 +1772,7 @@ func TestConnectorStatsFragmentUnknownIDDeletedNoticeHaltsPolling(t *testing.T) } } -func TestOverviewFragmentsRenderStatsAndMessageStreams(t *testing.T) { +func TestOverviewFragmentsRenderConsistentLiveMetrics(t *testing.T) { srv, svc, reg := newUIServerWithServiceAndRegistry(t) ctx := context.Background() must(t, svc.PutSource(ctx, model.Source{ID: "src1", Name: "Source One", Type: model.SourceSocketCAN, Enabled: true, Interface: "can0"}, true)) @@ -1643,13 +1788,64 @@ func TestOverviewFragmentsRenderStatsAndMessageStreams(t *testing.T) { reg.RecordConnectorEvent("conn1", "received", env) reg.Record("conn1", 1, int64(env.SizeBytes())) + // Status and metrics share one card, so there is no separate "Metrics" + // heading. The queue tiles are asserted per-kind below: only connectors own + // a queue, so only their fragment carries pending/retained numbers. + commonWants := []string{ + "Status", + "Msg/s", "Bytes/s", + `hx-trigger="every 500ms"`, + } + commonNotWants := []string{"Total messages", "Total bytes", "Message stream", `every 2s`} + for _, tc := range []struct { - path string - wants []string + path string + wants []string + notWants []string }{ - {"/ui/frag/sources/src1/overview", []string{"source-overview-live", "Status", "Statistics", "PGN traffic", "Rate / jitter", "Last seen / gaps", "Values / raw wire", "decoded fields", "Traffic changes", "Set expected traffic baseline", "Message stream", "Total messages", "127250", "heading", "received"}}, - {"/ui/frag/sinks/sink1/overview", []string{"sink-overview-live", "Status", "Statistics", "Message stream", "Total messages", "127250", "heading", "sent", "conn1"}}, - {"/ui/frag/connectors/conn1/overview", []string{"connector-overview-live", "Status", "Statistics", "Message stream", "Total messages", "127250", "heading", "received", "conn1"}}, + { + "/frag/sources/src1/overview", + []string{ + "source-overview-live", "Source devices", + "Msg/s", "Bytes/s", "Traffic share", + `id="source-overview-summary"`, + `hx-select="#source-overview-summary"`, + `id="source-device-table-frame"`, `id="source-device-rows"`, + `data-source-device-row-refresh`, + `hx-get="/frag/sources/src1/device-rows?dir=asc&sort=address"`, + `hx-swap="none"`, + `aria-label="Sort by address descending"`, + `aria-label="Sort by messages per second descending"`, + `aria-label="Sort by last seen descending"`, + }, + []string{ + `aria-label="PGN traffic"`, `aria-label="Traffic changes"`, + "Set expected traffic baseline", + "Model / product", "Instances", "Software / serial", + `hx-select="#source-device-rows"`, + "Pending delivery", "Pending bytes", "Retained history", "Retained bytes", + }, + }, + { + "/frag/sinks/sink1/overview", + []string{ + "sink-overview-live", `id="sink-overview-summary"`, + `hx-select="#sink-overview-summary"`, + }, + []string{ + "127250", "sent", + "Pending delivery", "Pending bytes", "Retained history", "Retained bytes", + }, + }, + { + "/frag/connectors/conn1/overview", + []string{ + "connector-overview-live", `id="connector-overview-summary"`, + `hx-select="#connector-overview-summary"`, + "Pending delivery", "Pending bytes", "Retained history", "Retained bytes", + }, + []string{"127250", "received"}, + }, } { t.Run(tc.path, func(t *testing.T) { resp, err := http.Get(srv.URL + tc.path) @@ -1658,41 +1854,206 @@ func TestOverviewFragmentsRenderStatsAndMessageStreams(t *testing.T) { } mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) - for _, want := range tc.wants { - if !strings.Contains(body, want) { - t.Fatalf("overview fragment missing %q:\n%s", want, body) + for _, wants := range [][]string{commonWants, tc.wants} { + for _, want := range wants { + if !strings.Contains(body, want) { + t.Fatalf("overview fragment missing %q:\n%s", want, body) + } + } + } + for _, notWants := range [][]string{commonNotWants, tc.notWants} { + for _, notWant := range notWants { + if strings.Contains(body, notWant) { + t.Fatalf("overview fragment unexpectedly contains %q:\n%s", notWant, body) + } + } + } + if tc.path == "/frag/sources/src1/overview" { + start := strings.Index(body, `") + if end < 0 { + t.Fatalf("source device tbody tag is unterminated:\n%s", body) + } + tbodyTag := body[start : start+end+1] + if strings.Contains(tbodyTag, "hx-") { + t.Fatalf("source device tbody must remain stable, got %s", tbodyTag) } } }) } } -func TestSourceTrafficBaselineFormActions(t *testing.T) { +func TestSourceDeviceSortStatePersistsInLiveRefresh(t *testing.T) { + srv, svc, _ := newUIServerWithServiceAndRegistry(t) + must(t, svc.PutSource(context.Background(), model.Source{ + ID: "src1", Name: "Source One", Type: model.SourceSocketCAN, Enabled: true, Interface: "can0", + }, true)) + + resp, err := http.Get(srv.URL + "/frag/sources/src1/overview?sort=bytes&dir=desc") + if err != nil { + t.Fatal(err) + } + mustStatus(t, resp, http.StatusOK) + body := mustBody(t, resp) + + for _, want := range []string{ + `aria-label="Sort by bytes per second ascending"`, + `hx-get="/frag/sources/src1/overview?dir=desc&sort=bytes"`, + `hx-get="/frag/sources/src1/device-rows?dir=desc&sort=bytes"`, + `data-source-device-row-refresh`, + } { + if !strings.Contains(body, want) { + t.Fatalf("sorted source fragment missing %q:\n%s", want, body) + } + } + if got := strings.Count(body, `aria-sort="descending"`); got != 1 { + t.Fatalf("descending sort markers = %d, want 1:\n%s", got, body) + } +} + +func TestSourceDeviceRowSnapshotFragmentContainsRowsOnly(t *testing.T) { srv, svc, reg := newUIServerWithServiceAndRegistry(t) must(t, svc.PutSource(context.Background(), model.Source{ - ID: "src1", Name: "Source One", Type: model.SourceSocketCAN, Interface: "can0", + ID: "src1", Name: "Source One", Type: model.SourceSocketCAN, Enabled: true, Interface: "can0", }, true)) reg.RecordSource("src1", &msg.Envelope{ - PGN: 127250, Source: 12, Dest: 255, Priority: 2, Raw: []byte{1, 2, 3, 4}, + PGN: 127250, PGNName: "Vessel Heading", Source: 12, + Raw: []byte{1, 2, 3, 4, 5, 6, 7, 8}, }) - resp := postForm(t, srv, "/ui/sources/src1/traffic-baseline", url.Values{}) + resp, err := http.Get(srv.URL + "/frag/sources/src1/device-rows?sort=bytes&dir=desc") + if err != nil { + t.Fatal(err) + } mustStatus(t, resp, http.StatusOK) body := mustBody(t, resp) - for _, want := range []string{"baseline matched", "Clear baseline", "baseline_committed"} { + + for _, want := range []string{ + ``, + `id="source-device-row-12"`, + `data-device-address="12"`, + `hx-get="/frag/sources/src1/overview?device=12&dir=desc&pgn_dir=asc&pgn_sort=pgn&sort=bytes"`, + `class="source-device-stat"`, + } { if !strings.Contains(body, want) { - t.Fatalf("baseline commit response missing %q:\n%s", want, body) + t.Fatalf("source device row snapshot missing %q:\n%s", want, body) } } - if len(reg.SourceTrafficBaselines("src1")) != 1 { - t.Fatalf("baseline was not stored: %+v", reg.SourceTrafficBaselines("src1")) + for _, notWant := range []string{"src1", "sink1", "msg.pgn == 127250", "500", "24h0m0s", "1048576", - `href="/ui/connectors/conn1/edit"`, - `aria-label="Breadcrumb"`, `href="/ui/connectors"`, `aria-current="page">NMEA Bridge`, - `hx-get="/ui/frag/connectors/conn1/overview"`, `hx-trigger="load, every 2s"`, + `href="/connectors/conn1/edit"`, + `aria-label="Breadcrumb"`, `href="/connectors"`, `aria-current="page">NMEA Bridge`, + `hx-get="/frag/connectors/conn1/overview"`, `hx-trigger="load, every 500ms"`, } { if !strings.Contains(body, want) { t.Fatalf("connector detail page missing %q:\n%s", want, body) @@ -1727,7 +2088,7 @@ func TestConnectorOverviewPageRendersConfigSummary(t *testing.T) { func TestConnectorDetailPage404ForUnknownID(t *testing.T) { srv, _ := newUIServerWithService(t) - resp, err := http.Get(srv.URL + "/ui/connectors/doesnotexist") + resp, err := http.Get(srv.URL + "/connectors/doesnotexist") if err != nil { t.Fatal(err) } diff --git a/internal/ui/overview.go b/internal/ui/overview.go index 4ad5269..fc45482 100644 --- a/internal/ui/overview.go +++ b/internal/ui/overview.go @@ -3,11 +3,8 @@ package ui import ( "encoding/json" "errors" - "fmt" "log/slog" - "math" "net/http" - "sort" "strconv" "strings" "time" @@ -18,8 +15,6 @@ import ( "github.com/open-ships/beacon/internal/supervisor" ) -const overviewEventLimit = 24 - type configRow struct { Label string Value string @@ -34,6 +29,8 @@ type overviewPageData struct { ConfigRows []configRow LiveDOMID string LiveHref string + StreamHref string + StreamID string Description string } @@ -42,128 +39,39 @@ type overviewLogRow struct { Message string } -type overviewEventRow struct { - TimeText string - Stage string - ConnectorID string - PGN uint32 - PGNName string - Source uint8 - Dest uint8 - Priority uint8 - MessageTime string - Payload string - SizeBytesText string -} - type overviewLiveData struct { - DOMID string - RefreshHref string - Kind string - ID string - State string - Err string - Logs []overviewLogRow - Snapshot stats.Snapshot - BytesPerSecText string - TotalBytesText string - QueueBytesText string - RetainedBytesText string - Events []overviewEventRow - SourcePGNs []sourcePGNRow - SourceEvents []sourceMetricEventRow - BaselineCount int - EmptyStream string -} - -type sourcePGNRow struct { - Observed bool - Status string - PGN uint32 - PGNName string - SourceAddress uint8 - DeviceNameHex string - FrequencyText string - PeriodText string - LastSeenText string - LastSeenTitle string - GapText string - PayloadText string - PayloadDetail string - Messages int64 - AnomalyText string - Fields []sourceFieldRow - DecodeStatus string - DecodeDetail string - DecodeMetadata string - Destinations string - Priorities string - RateDetail string - TrafficText string - TrafficDetail string - BaselineStatus string - BaselineIssues []string - Raw *stats.RawPayloadDiagnostics - RawFingerprints []sourceRawFingerprintRow - RawBytes []sourceRawByteRow - RawSamples []sourceRawSampleRow -} - -type sourceFieldRow struct { - Name string - Summary string - Samples int64 - Anomalous bool - Anomalies int64 - AvailabilityText string - QualityText string -} - -type sourceRawByteRow struct { - Offset int - Range string - Mode string - Entropy string - Changed string - BitMask string -} - -type sourceRawFingerprintRow struct { - Fingerprint string - Count int64 - Share string - Length int -} - -type sourceRawSampleRow struct { - Time string - Hex string - Fingerprint string - Length int -} - -type sourceMetricEventRow struct { - Time string - Kind string - Severity string - Summary string - PGN uint32 - SourceAddress uint8 + DOMID string + RefreshHref string + Kind string + ID string + State string + Err string + Logs []overviewLogRow + Snapshot stats.Snapshot + BytesPerSecText string + QueueBytesText string + RetainedBytesText string + SourceDevices []sourceDeviceRow + SourceDeviceSort sourceDeviceSortView + SourceDeviceDetail *sourceDeviceDetailView + SourceDeviceRowsRefreshHref string } func sourceOverviewPageData(version string, s model.Source) overviewPageData { title := displayName(s.Name, s.ID) return overviewPageData{ pageData: newPageData(title, version, "sources").withBreadcrumbs( - breadcrumbItem{Label: "Sources", Href: "/ui/sources"}, + breadcrumbItem{Label: "Sources", Href: "/sources"}, breadcrumbItem{Label: title}, ), Kind: "source", Heading: title, - EditHref: "/ui/sources/" + s.ID + "/edit", + EditHref: "/sources/" + s.ID + "/edit", ConfigRows: sourceConfigRows(s), LiveDOMID: "source-overview-live", - LiveHref: "/ui/frag/sources/" + s.ID + "/overview", + LiveHref: "/frag/sources/" + s.ID + "/overview", + StreamHref: "/ui/streams/sources/" + s.ID, + StreamID: s.ID, Description: "Received message activity and runtime health for this source.", } } @@ -172,15 +80,17 @@ func sinkOverviewPageData(version string, s model.Sink) overviewPageData { title := displayName(s.Name, s.ID) return overviewPageData{ pageData: newPageData(title, version, "sinks").withBreadcrumbs( - breadcrumbItem{Label: "Sinks", Href: "/ui/sinks"}, + breadcrumbItem{Label: "Sinks", Href: "/sinks"}, breadcrumbItem{Label: title}, ), Kind: "sink", Heading: title, - EditHref: "/ui/sinks/" + s.ID + "/edit", + EditHref: "/sinks/" + s.ID + "/edit", ConfigRows: sinkConfigRows(s), LiveDOMID: "sink-overview-live", - LiveHref: "/ui/frag/sinks/" + s.ID + "/overview", + LiveHref: "/frag/sinks/" + s.ID + "/overview", + StreamHref: "/ui/streams/sinks/" + s.ID, + StreamID: s.ID, Description: "Sent message activity and runtime health for this sink.", } } @@ -189,15 +99,15 @@ func connectorOverviewPageData(version string, c model.Connector) overviewPageDa title := displayName(c.Name, c.ID) return overviewPageData{ pageData: newPageData(title, version, "connectors").withBreadcrumbs( - breadcrumbItem{Label: "Connectors", Href: "/ui/connectors"}, + breadcrumbItem{Label: "Connectors", Href: "/connectors"}, breadcrumbItem{Label: title}, ), Kind: "connector", Heading: title, - EditHref: "/ui/connectors/" + c.ID + "/edit", + EditHref: "/connectors/" + c.ID + "/edit", ConfigRows: connectorConfigRows(c), LiveDOMID: "connector-overview-live", - LiveHref: "/ui/frag/connectors/" + c.ID + "/overview", + LiveHref: "/frag/connectors/" + c.ID + "/overview", Description: "Matched and delivered message activity for this connector.", } } @@ -317,205 +227,42 @@ func overviewLogs(kind, state, errText string) []overviewLogRow { } } -func overviewEvents(events []stats.Event) []overviewEventRow { - rows := make([]overviewEventRow, 0, len(events)) - for _, e := range events { - row := overviewEventRow{ - TimeText: formatEventTime(e.Time), - Stage: e.Stage, - ConnectorID: e.ConnectorID, - PGN: e.PGN, - PGNName: e.PGNName, - Source: e.Source, - Dest: e.Dest, - Priority: e.Priority, - MessageTime: formatEventTime(e.Timestamp), - Payload: e.Payload, - SizeBytesText: humanizeBytes(float64(e.SizeBytes), ""), - } - rows = append(rows, row) - } - return rows -} - -func formatEventTime(t time.Time) string { - if t.IsZero() { - return "-" - } - return t.Format("15:04:05") -} - -func sourceOverviewLiveData(s model.Source, reg *stats.Registry, statuses []supervisor.Status) overviewLiveData { +func sourceOverviewLiveData( + s model.Source, + reg *stats.Registry, + statuses []supervisor.Status, + deviceSorting sourceDeviceSort, + selectedAddress *uint8, + pgnSorting sourceDevicePGNSort, +) overviewLiveData { snap, _ := reg.SourceSnapshot(s.ID) + sourceMetrics := reg.SourcePGNMetrics(s.ID) state, errText := overviewStatus(statuses, "source", s.ID, s.Enabled) + refreshBase := "/frag/sources/" + s.ID + "/overview" + sourceDevices := sourceDeviceRows(sourceMetrics, deviceSorting) + decorateSourceDeviceRows(sourceDevices, refreshBase, deviceSorting, selectedAddress, pgnSorting) return overviewLiveData{ - DOMID: "source-overview-live", - RefreshHref: "/ui/frag/sources/" + s.ID + "/overview", - Kind: "source", - ID: s.ID, - State: state, - Err: errText, - Logs: overviewLogs("source", state, errText), - Snapshot: snap, - BytesPerSecText: humanizeBytes(snap.BytesPerSec, "/s"), - TotalBytesText: humanizeBytes(float64(snap.TotalBytes), ""), - QueueBytesText: humanizeBytes(float64(snap.QueueBytes), ""), - RetainedBytesText: humanizeBytes(float64(snap.RetainedBytes), ""), - Events: overviewEvents(reg.Recent("source", s.ID, overviewEventLimit)), - SourcePGNs: sourcePGNRows(reg.SourcePGNMetrics(s.ID)), - SourceEvents: sourceMetricEventRows(reg.SourceMetricEvents(s.ID, 30)), - BaselineCount: len(reg.SourceTrafficBaselines(s.ID)), - EmptyStream: "No decoded messages have been received from this source in this process.", - } -} - -func sourcePGNRows(metrics []stats.SourcePGNMetric) []sourcePGNRow { - rows := make([]sourcePGNRow, 0, len(metrics)) - for _, metric := range metrics { - row := sourcePGNRow{ - Observed: metric.Observed, Status: metric.Status, PGN: metric.PGN, PGNName: metric.PGNName, - SourceAddress: metric.SourceAddress, DeviceNameHex: metric.DeviceNameHex, - FrequencyText: formatFrequency(metric.FrequencyHz), - PeriodText: formatMetricDuration(metric.ExpectedPeriodSeconds), - LastSeenText: formatAge(metric.AgeSeconds), - LastSeenTitle: metric.LastSeen.UTC().Format(time.RFC3339Nano), - GapText: sourceGapText(metric), - PayloadText: strconv.FormatInt(metric.PayloadBytesLast, 10) + " B", - PayloadDetail: fmt.Sprintf("mean %s B · range %d–%d B", - formatMetricFloat(metric.PayloadBytesMean), metric.PayloadBytesMin, metric.PayloadBytesMax), - Messages: metric.Messages, - DecodeStatus: metric.DecodeStatus, - DecodeDetail: sourceDecodeDetail(metric), - DecodeMetadata: sourceDecodeMetadata(metric), - Destinations: sortedMetricKeys(metric.DestinationCounts), - Priorities: sortedMetricKeys(metric.PriorityCounts), - RateDetail: fmt.Sprintf("p95 %s · p99 %s · jitter %s (%.1f%%) · %d bursts", - formatDuration(metric.PeriodP95Seconds), formatDuration(metric.PeriodP99Seconds), - formatDuration(metric.JitterMADSeconds), metric.JitterPercent, metric.BurstCount), - TrafficText: fmt.Sprintf("%s/s · %.1f%% of source", humanizeBytes(metric.RecentBytesPerSec, ""), metric.TrafficSharePercent), - TrafficDetail: fmt.Sprintf("%.2f msg/s · ~%.3f%% bus load · %d messages", - metric.RecentMessagesPerSec, metric.EstimatedBusLoadPercent, metric.Messages), - BaselineStatus: metric.BaselineStatus, BaselineIssues: metric.BaselineIssues, - Raw: metric.Raw, - } - if !metric.Observed { - row.LastSeenText = "not observed" - row.LastSeenTitle = "" - row.GapText = "expected by approved baseline" - row.PayloadText = "-" - row.PayloadDetail = "awaiting traffic" - row.TrafficText = "-" - row.TrafficDetail = "no observations in this process" - row.DecodeDetail = "approved expectation" - } - if metric.RecentAnomaly { - row.AnomalyText = metric.AnomalyField - if metric.AnomalyReason != "" { - row.AnomalyText += ": " + metric.AnomalyReason - } - } - for _, field := range metric.Fields { - row.Fields = append(row.Fields, sourceFieldDistributionRow(field)) - } - if metric.Raw != nil { - for _, fingerprint := range metric.Raw.Fingerprints { - row.RawFingerprints = append(row.RawFingerprints, sourceRawFingerprintRow{ - Fingerprint: fingerprint.Fingerprint, Count: fingerprint.Count, - Share: fmt.Sprintf("%.1f%%", fingerprint.Share*100), Length: fingerprint.Length, - }) - } - for _, rawByte := range metric.Raw.Bytes { - row.RawBytes = append(row.RawBytes, sourceRawByteRow{ - Offset: rawByte.Offset, - Range: fmt.Sprintf("%02x–%02x", rawByte.Minimum, rawByte.Maximum), - Mode: fmt.Sprintf("%02x (%.0f%%)", rawByte.MostCommon, rawByte.MostCommonShare*100), - Entropy: fmt.Sprintf("%.2f bits", rawByte.EntropyBits), - Changed: fmt.Sprintf("%.0f%%", rawByte.ChangedShare*100), BitMask: rawByte.ChangedBitMaskHex, - }) - } - for _, sample := range metric.Raw.Samples { - row.RawSamples = append(row.RawSamples, sourceRawSampleRow{ - Time: sample.ObservedAt.Format("15:04:05"), Hex: sample.Hex, - Fingerprint: sample.Fingerprint, Length: sample.Length, - }) - } - } - rows = append(rows, row) - } - return rows -} - -func sourceDecodeDetail(metric stats.SourcePGNMetric) string { - parts := []string{fmt.Sprintf("%d complete", metric.DecodeComplete)} - if metric.DecodeIncomplete > 0 { - parts = append(parts, fmt.Sprintf("%d incomplete", metric.DecodeIncomplete)) - } - if metric.DecodeFallback > 0 { - parts = append(parts, fmt.Sprintf("%d fallback", metric.DecodeFallback)) - } - if metric.UnknownMessages > 0 { - parts = append(parts, fmt.Sprintf("%d unknown", metric.UnknownMessages)) - } - if len(metric.MissingDecodedFields) > 0 { - parts = append(parts, "missing "+sortedMetricKeys(metric.MissingDecodedFields)) - } - return strings.Join(parts, " · ") -} - -func sourceDecodeMetadata(metric stats.SourcePGNMetric) string { - parts := make([]string, 0, 3) - if metric.Variant != "" { - parts = append(parts, metric.Variant) - } - if metric.Transport != "" { - parts = append(parts, metric.Transport) - } - if metric.ManufacturerCode != nil { - parts = append(parts, fmt.Sprintf("manufacturer %d", *metric.ManufacturerCode)) - } - return strings.Join(parts, " · ") -} - -func sortedMetricKeys(values map[string]int64) string { - keys := make([]string, 0, len(values)) - for key := range values { - keys = append(keys, key) - } - sort.Slice(keys, func(i, j int) bool { - left, leftErr := strconv.Atoi(keys[i]) - right, rightErr := strconv.Atoi(keys[j]) - if leftErr == nil && rightErr == nil && left != right { - return left < right - } - return keys[i] < keys[j] - }) - if len(keys) == 0 { - return "-" - } - return strings.Join(keys, ", ") -} - -func formatFrequency(hz float64) string { - if hz <= 0 { - return "learning" - } - if hz >= 100 { - return fmt.Sprintf("%.0f Hz", hz) - } - if hz >= 10 { - return fmt.Sprintf("%.1f Hz", hz) - } - if hz >= 1 { - return fmt.Sprintf("%.2f Hz", hz) - } - return fmt.Sprintf("%.3f Hz", hz) -} - -func formatMetricDuration(seconds float64) string { - if seconds <= 0 { - return "period learning" + DOMID: "source-overview-live", + RefreshHref: sourceDeviceSortURL(refreshBase, deviceSorting, selectedAddress, pgnSorting), + Kind: "source", + ID: s.ID, + State: state, + Err: errText, + Logs: overviewLogs("source", state, errText), + Snapshot: snap, + BytesPerSecText: humanizeBytes(snap.BytesPerSec, "/s"), + QueueBytesText: humanizeBytes(float64(snap.QueueBytes), ""), + RetainedBytesText: humanizeBytes(float64(snap.RetainedBytes), ""), + SourceDevices: sourceDevices, + SourceDeviceSort: sourceDeviceSortControls(refreshBase, deviceSorting, selectedAddress, pgnSorting), + SourceDeviceDetail: sourceDeviceDetail(sourceMetrics, sourceDevices, selectedAddress, refreshBase, deviceSorting, pgnSorting), + SourceDeviceRowsRefreshHref: sourceDeviceSortURL( + "/frag/sources/"+s.ID+"/device-rows", + deviceSorting, + selectedAddress, + pgnSorting, + ), } - return formatDuration(seconds) + " period" } func formatDuration(seconds float64) string { @@ -549,94 +296,12 @@ func formatAge(seconds float64) string { return d.Round(time.Hour).String() + " ago" } -func sourceGapText(metric stats.SourcePGNMetric) string { - if metric.GapActive { - return fmt.Sprintf("%.1f× expected now", metric.GapRatio) - } - if metric.GapCount == 0 { - return "none detected" - } - return fmt.Sprintf("%d prior · max %s", metric.GapCount, - formatDuration(metric.LongestGapSeconds)) -} - -func sourceFieldDistributionRow(field stats.FieldDistribution) sourceFieldRow { - row := sourceFieldRow{Name: field.Field, Samples: field.Samples, - Anomalous: field.Anomalous, Anomalies: field.AnomalyCount, - AvailabilityText: fmt.Sprintf("%.1f%% available · %s unchanged", field.AvailabilityPercent, formatDuration(field.StuckSeconds)), - QualityText: fmt.Sprintf("%d missing · %d invalid · %d out of range · %d novel values", - field.MissingMessages, field.InvalidCount, field.OutOfRangeCount, field.NovelValueCount)} - if field.Kind == "number" && field.LastNumeric != nil && field.Mean != nil && - field.Minimum != nil && field.Maximum != nil && field.StdDev != nil { - unit := "" - if field.Unit != "" { - unit = " " + field.Unit - } - row.Summary = fmt.Sprintf("last %s%s · mean %s ± %s%s · range %s–%s%s", - formatMetricFloat(*field.LastNumeric), unit, - formatMetricFloat(*field.Mean), formatMetricFloat(*field.StdDev), unit, - formatMetricFloat(*field.Minimum), formatMetricFloat(*field.Maximum), unit) - if field.LastChange != nil { - row.Summary += " · Δ " + formatMetricFloat(*field.LastChange) + unit - } - if field.P05 != nil && field.P50 != nil && field.P95 != nil && field.P99 != nil { - row.Summary += fmt.Sprintf(" · p05/p50/p95/p99 %s/%s/%s/%s%s", - formatMetricFloat(*field.P05), formatMetricFloat(*field.P50), - formatMetricFloat(*field.P95), formatMetricFloat(*field.P99), unit) - } - if field.LastRateOfChange != nil { - row.Summary += " · rate " + formatMetricFloat(*field.LastRateOfChange) + unit + "/s" - } - return row - } - type categoryCount struct { - value string - count int64 - } - counts := make([]categoryCount, 0, len(field.Values)) - for value, count := range field.Values { - counts = append(counts, categoryCount{value: value, count: count}) - } - sort.Slice(counts, func(i, j int) bool { - if counts[i].count != counts[j].count { - return counts[i].count > counts[j].count - } - return counts[i].value < counts[j].value - }) - parts := make([]string, 0, len(counts)+1) - for _, count := range counts { - parts = append(parts, fmt.Sprintf("%s × %d", count.value, count.count)) - } - if field.Other > 0 { - parts = append(parts, fmt.Sprintf("other × %d", field.Other)) - } - row.Summary = strings.Join(parts, " · ") - return row -} - -func sourceMetricEventRows(events []stats.SourceMetricEvent) []sourceMetricEventRow { - rows := make([]sourceMetricEventRow, 0, len(events)) - for _, event := range events { - rows = append(rows, sourceMetricEventRow{Time: event.Time.Format("2006-01-02 15:04:05"), - Kind: event.Kind, Severity: event.Severity, Summary: event.Summary, - PGN: event.PGN, SourceAddress: event.SourceAddress}) - } - return rows -} - -func formatMetricFloat(value float64) string { - if math.IsInf(value, 0) { - return "∞" - } - return strconv.FormatFloat(value, 'g', 4, 64) -} - func sinkOverviewLiveData(s model.Sink, reg *stats.Registry, statuses []supervisor.Status) overviewLiveData { snap, _ := reg.SinkSnapshot(s.ID) state, errText := overviewStatus(statuses, "sink", s.ID, s.Enabled) return overviewLiveData{ DOMID: "sink-overview-live", - RefreshHref: "/ui/frag/sinks/" + s.ID + "/overview", + RefreshHref: "/frag/sinks/" + s.ID + "/overview", Kind: "sink", ID: s.ID, State: state, @@ -644,11 +309,8 @@ func sinkOverviewLiveData(s model.Sink, reg *stats.Registry, statuses []supervis Logs: overviewLogs("sink", state, errText), Snapshot: snap, BytesPerSecText: humanizeBytes(snap.BytesPerSec, "/s"), - TotalBytesText: humanizeBytes(float64(snap.TotalBytes), ""), QueueBytesText: humanizeBytes(float64(snap.QueueBytes), ""), RetainedBytesText: humanizeBytes(float64(snap.RetainedBytes), ""), - Events: overviewEvents(reg.Recent("sink", s.ID, overviewEventLimit)), - EmptyStream: "No messages have been sent to this sink in this process.", } } @@ -657,7 +319,7 @@ func connectorOverviewLiveData(c model.Connector, reg *stats.Registry, statuses state, errText := overviewStatus(statuses, "connector", c.ID, c.Enabled) return overviewLiveData{ DOMID: "connector-overview-live", - RefreshHref: "/ui/frag/connectors/" + c.ID + "/overview", + RefreshHref: "/frag/connectors/" + c.ID + "/overview", Kind: "connector", ID: c.ID, State: state, @@ -665,11 +327,8 @@ func connectorOverviewLiveData(c model.Connector, reg *stats.Registry, statuses Logs: overviewLogs("connector", state, errText), Snapshot: snap, BytesPerSecText: humanizeBytes(snap.BytesPerSec, "/s"), - TotalBytesText: humanizeBytes(float64(snap.TotalBytes), ""), QueueBytesText: humanizeBytes(float64(snap.QueueBytes), ""), RetainedBytesText: humanizeBytes(float64(snap.RetainedBytes), ""), - Events: overviewEvents(reg.Recent("connector", c.ID, overviewEventLimit)), - EmptyStream: "No messages have moved through this connector in this process.", } } @@ -734,39 +393,25 @@ func handleSourceOverviewFrag(svc *config.Service, reg *stats.Registry, statuses renderOverviewFragErr(w, log, err) return } - renderFragment(w, log, "overview-live", sourceOverviewLiveData(source, reg, statuses())) + sorting := sourceDeviceSortFromQuery(r.URL.Query().Get("sort"), r.URL.Query().Get("dir")) + selectedAddress := sourceDeviceAddressFromQuery(r.URL.Query().Get("device")) + pgnSorting := sourceDevicePGNSortFromQuery(r.URL.Query().Get("pgn_sort"), r.URL.Query().Get("pgn_dir")) + renderFragment(w, log, "overview-live", sourceOverviewLiveData(source, reg, statuses(), sorting, selectedAddress, pgnSorting)) } } -func handleSourceTrafficBaselineCommit(svc *config.Service, reg *stats.Registry, statuses func() []supervisor.Status, log *slog.Logger) http.HandlerFunc { +func handleSourceDeviceRowsFrag(svc *config.Service, reg *stats.Registry, statuses func() []supervisor.Status, log *slog.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { source, err := svc.GetSource(r.Context(), r.PathValue("id")) if err != nil { renderOverviewFragErr(w, log, err) return } - if _, err := reg.CommitSourceTrafficBaseline(r.Context(), source.ID); err != nil { - log.Error("ui: commit source traffic baseline", "source", source.ID, "err", err) - http.Error(w, "traffic baseline failed", http.StatusInternalServerError) - return - } - renderFragment(w, log, "overview-live", sourceOverviewLiveData(source, reg, statuses())) - } -} - -func handleSourceTrafficBaselineClear(svc *config.Service, reg *stats.Registry, statuses func() []supervisor.Status, log *slog.Logger) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - source, err := svc.GetSource(r.Context(), r.PathValue("id")) - if err != nil { - renderOverviewFragErr(w, log, err) - return - } - if err := reg.ClearSourceTrafficBaseline(r.Context(), source.ID); err != nil { - log.Error("ui: clear source traffic baseline", "source", source.ID, "err", err) - http.Error(w, "traffic baseline clear failed", http.StatusInternalServerError) - return - } - renderFragment(w, log, "overview-live", sourceOverviewLiveData(source, reg, statuses())) + sorting := sourceDeviceSortFromQuery(r.URL.Query().Get("sort"), r.URL.Query().Get("dir")) + selectedAddress := sourceDeviceAddressFromQuery(r.URL.Query().Get("device")) + pgnSorting := sourceDevicePGNSortFromQuery(r.URL.Query().Get("pgn_sort"), r.URL.Query().Get("pgn_dir")) + data := sourceOverviewLiveData(source, reg, statuses(), sorting, selectedAddress, pgnSorting) + renderFragment(w, log, "source-device-row-snapshot", data) } } diff --git a/internal/ui/render.go b/internal/ui/render.go index 33b334b..d559035 100644 --- a/internal/ui/render.go +++ b/internal/ui/render.go @@ -25,7 +25,7 @@ func renderPage(w http.ResponseWriter, log *slog.Logger, name string, data any) t, ok := pages[name] if !ok { // Unreachable from HTTP: callers only pass names they've registered - // a "GET /ui/" route for, which by construction exist in + // a "GET /" route for, which by construction exist in // pages. Guarded anyway so a future routing/pages drift fails loud // instead of executing a nil template. log.Error("ui: render failed", "page", name, "err", "no template registered for page") @@ -44,7 +44,7 @@ func renderPage(w http.ResponseWriter, log *slog.Logger, name string, data any) // renderFragment executes the named template out of fragTemplates (an // htmx response fragment — no layout.html wrapper) and writes it to w. -// Used by every /ui/frag/* handler and by every source/sink write/delete +// Used by every /frag/* handler and by every source/sink write/delete // handler in forms.go to re-render the table or form fragment. Buffered // for the same reason renderPage is: a partial write on a mid-render // failure would leave htmx swapping in truncated HTML with no clean way to diff --git a/internal/ui/source_device_detail.go b/internal/ui/source_device_detail.go new file mode 100644 index 0000000..4fe775a --- /dev/null +++ b/internal/ui/source_device_detail.go @@ -0,0 +1,274 @@ +package ui + +import ( + "fmt" + "sort" + "time" + + "github.com/open-ships/beacon/internal/stats" +) + +type sourceDeviceDetailView struct { + Device sourceDeviceRow + PGNs []sourceDevicePGNRow + Sort sourceDevicePGNSortView + RefreshHref string + CloseHref string +} + +type sourceDevicePGNRow struct { + PGN uint32 + PGNName string + MessagesPerSec string + BytesPerSec string + DeviceTraffic string + MedianPeriod string + PeriodP90 string + PeriodP99 string + Payload string + LastSeen string + LastSeenTitle string + ActivityLevel string + ActivityLabel string + EstimatedBusLoad string + variant string + messagesPerSec float64 + trafficShare float64 + payloadBytesMean float64 + lastSeen time.Time +} + +type sourceDevicePGNSort struct { + key string + direction string +} + +type sourceDevicePGNSortView struct { + PGN sourceDeviceSortControl + Rates sourceDeviceSortControl + Traffic sourceDeviceSortControl + Payload sourceDeviceSortControl + Activity sourceDeviceSortControl +} + +const ( + sourceDevicePGNSortPGN = "pgn" + sourceDevicePGNSortRates = "rates" + sourceDevicePGNSortTraffic = "traffic" + sourceDevicePGNSortPayload = "payload" + sourceDevicePGNSortActivity = "activity" +) + +func sourceDeviceDetail( + metrics []stats.SourcePGNMetric, + devices []sourceDeviceRow, + selectedAddress *uint8, + base string, + deviceSorting sourceDeviceSort, + pgnSorting sourceDevicePGNSort, +) *sourceDeviceDetailView { + if selectedAddress == nil { + return nil + } + + var device *sourceDeviceRow + for i := range devices { + if devices[i].Address == *selectedAddress { + copy := devices[i] + device = © + break + } + } + if device == nil { + return nil + } + + deviceMetrics := make([]stats.SourcePGNMetric, 0) + totalObservedBytes := 0.0 + for _, metric := range metrics { + if !metric.Observed || metric.SourceAddress != *selectedAddress { + continue + } + deviceMetrics = append(deviceMetrics, metric) + totalObservedBytes += float64(metric.Messages) * metric.PayloadBytesMean + } + rows := make([]sourceDevicePGNRow, 0, len(deviceMetrics)) + for _, metric := range deviceMetrics { + trafficShare := 0.0 + if totalObservedBytes > 0 { + trafficShare = float64(metric.Messages) * metric.PayloadBytesMean / totalObservedBytes * 100 + } + learnedTiming := metric.ExpectedPeriodSeconds > 0 + activityLevel, activityLabel := pgnActivityGrade(metric.AgeSeconds, metric.PeriodP90Seconds) + rows = append(rows, sourceDevicePGNRow{ + PGN: metric.PGN, + PGNName: metric.PGNName, + MessagesPerSec: fmt.Sprintf("%.2f", metric.RecentMessagesPerSec), + BytesPerSec: humanizeBytes(metric.RecentBytesPerSec, "/s"), + DeviceTraffic: fmt.Sprintf("%.1f%%", trafficShare), + MedianPeriod: formatPanelDuration(metric.ExpectedPeriodSeconds, learnedTiming), + PeriodP90: formatPanelDuration(metric.PeriodP90Seconds, learnedTiming), + PeriodP99: formatPanelDuration(metric.PeriodP99Seconds, learnedTiming), + Payload: formatPayloadSize(metric.PayloadBytesLast, metric.PayloadBytesMean), + LastSeen: formatAge(metric.AgeSeconds), + LastSeenTitle: metric.LastSeen.UTC().Format(time.RFC3339Nano), + ActivityLevel: activityLevel, + ActivityLabel: activityLabel, + EstimatedBusLoad: fmt.Sprintf("%.3f%%", metric.EstimatedBusLoadPercent), + variant: metric.Variant, + messagesPerSec: metric.RecentMessagesPerSec, + trafficShare: trafficShare, + payloadBytesMean: metric.PayloadBytesMean, + lastSeen: metric.LastSeen, + }) + } + sortSourceDevicePGNRows(rows, pgnSorting) + + return &sourceDeviceDetailView{ + Device: *device, + PGNs: rows, + Sort: sourceDevicePGNSortControls(base, deviceSorting, *selectedAddress, pgnSorting), + RefreshHref: sourceDeviceSortURL(base, deviceSorting, selectedAddress, pgnSorting), + CloseHref: sourceDeviceSortURL(base, deviceSorting, nil, pgnSorting), + } +} + +func sourceDevicePGNSortFromQuery(key, direction string) sourceDevicePGNSort { + switch key { + case sourceDevicePGNSortRates, sourceDevicePGNSortTraffic, sourceDevicePGNSortPayload, sourceDevicePGNSortActivity: + default: + key = sourceDevicePGNSortPGN + } + if direction != sourceDeviceSortAsc && direction != sourceDeviceSortDesc { + direction = defaultSourceDevicePGNSortDirection(key) + } + return sourceDevicePGNSort{key: key, direction: direction} +} + +func defaultSourceDevicePGNSortDirection(key string) string { + if key == sourceDevicePGNSortPGN { + return sourceDeviceSortAsc + } + return sourceDeviceSortDesc +} + +func sourceDevicePGNSortControls( + base string, + deviceSorting sourceDeviceSort, + selectedAddress uint8, + current sourceDevicePGNSort, +) sourceDevicePGNSortView { + return sourceDevicePGNSortView{ + PGN: sourceDevicePGNSortControlFor(base, deviceSorting, selectedAddress, current, sourceDevicePGNSortPGN, "PGN number"), + Rates: sourceDevicePGNSortControlFor(base, deviceSorting, selectedAddress, current, sourceDevicePGNSortRates, "message rate"), + Traffic: sourceDevicePGNSortControlFor(base, deviceSorting, selectedAddress, current, sourceDevicePGNSortTraffic, "traffic percentage"), + Payload: sourceDevicePGNSortControlFor(base, deviceSorting, selectedAddress, current, sourceDevicePGNSortPayload, "mean payload size"), + Activity: sourceDevicePGNSortControlFor(base, deviceSorting, selectedAddress, current, sourceDevicePGNSortActivity, "last activity"), + } +} + +func sourceDevicePGNSortControlFor( + base string, + deviceSorting sourceDeviceSort, + selectedAddress uint8, + current sourceDevicePGNSort, + key string, + label string, +) sourceDeviceSortControl { + nextDirection := defaultSourceDevicePGNSortDirection(key) + control := sourceDeviceSortControl{AriaSort: "none", Indicator: "↕"} + if current.key == key { + control.AriaSort = "ascending" + control.Indicator = "↑" + nextDirection = sourceDeviceSortDesc + if current.direction == sourceDeviceSortDesc { + control.AriaSort = "descending" + control.Indicator = "↓" + nextDirection = sourceDeviceSortAsc + } + } + control.Href = sourceDeviceSortURL( + base, + deviceSorting, + &selectedAddress, + sourceDevicePGNSort{key: key, direction: nextDirection}, + ) + control.Accessible = fmt.Sprintf("Sort by %s %s", label, sortDirectionWord(nextDirection)) + return control +} + +func sortSourceDevicePGNRows(rows []sourceDevicePGNRow, sorting sourceDevicePGNSort) { + sort.SliceStable(rows, func(i, j int) bool { + comparison := 0 + switch sorting.key { + case sourceDevicePGNSortRates: + comparison = compareFloat(rows[i].messagesPerSec, rows[j].messagesPerSec) + case sourceDevicePGNSortTraffic: + comparison = compareFloat(rows[i].trafficShare, rows[j].trafficShare) + case sourceDevicePGNSortPayload: + comparison = compareFloat(rows[i].payloadBytesMean, rows[j].payloadBytesMean) + case sourceDevicePGNSortActivity: + comparison = compareTime(rows[i].lastSeen, rows[j].lastSeen) + default: + comparison = compareInt(int(rows[i].PGN), int(rows[j].PGN)) + } + if comparison == 0 { + if rows[i].PGN == rows[j].PGN { + return rows[i].variant < rows[j].variant + } + return rows[i].PGN < rows[j].PGN + } + if sorting.direction == sourceDeviceSortDesc { + return comparison > 0 + } + return comparison < 0 + }) +} + +func compareTime(left, right time.Time) int { + switch { + case left.Before(right): + return -1 + case left.After(right): + return 1 + default: + return 0 + } +} + +func formatPanelDuration(seconds float64, learned bool) string { + if !learned { + return "learning" + } + if seconds <= 0 { + return "0s" + } + return formatDuration(seconds) +} + +func formatPayloadSize(last int64, mean float64) string { + return fmt.Sprintf("%d B last · %.1f B mean", last, mean) +} + +// pgnActivityGrade rates how recently a PGN was seen against its own learned +// p90 period, so a 10 Hz PGN and a once-a-minute PGN are judged on their own +// cadence. The half-second slack keeps sub-second PGNs from flapping between +// colors on the panel's 500ms refresh. +func pgnActivityGrade(ageSeconds, periodP90Seconds float64) (level, label string) { + if periodP90Seconds <= 0 { + return "warming", "Cadence still being learned" + } + switch { + case ageSeconds <= periodP90Seconds+pgnActivitySlackSeconds: + return "fresh", "On cadence · seen within its p90 period" + case ageSeconds <= pgnActivityStaleFactor*periodP90Seconds+pgnActivitySlackSeconds: + return "late", "Late · overdue past its p90 period" + default: + return "stale", "Stale · missed several periods" + } +} + +const ( + pgnActivitySlackSeconds = 0.5 + pgnActivityStaleFactor = 3 +) diff --git a/internal/ui/source_devices.go b/internal/ui/source_devices.go new file mode 100644 index 0000000..e1faf38 --- /dev/null +++ b/internal/ui/source_devices.go @@ -0,0 +1,366 @@ +package ui + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "net/url" + "sort" + "strconv" + "strings" + "time" + + n2k "github.com/open-ships/n2k" + "github.com/open-ships/n2k/pgn" + + "github.com/open-ships/beacon/internal/stats" +) + +// sourceDeviceRow summarizes every observed PGN stream for one source address. +// Address and Device NAME are deliberately separate: the address orders the +// table and identifies the current sender slot, while NAME is the stable +// ISO 11783 device identity that survives address changes. +type sourceDeviceRow struct { + Address uint8 + DeviceName string + IdentityNumber string + Manufacturer string + ManufacturerCode string + ManufacturerInformation string + Model string + ProductCode string + Role string + Instances string + Software string + Serial string + TrafficShareText string + MessagesPerSecText string + BytesPerSecText string + BusLoadText string + LastSeenText string + LastSeenTitle string + messagesPerSec float64 + bytesPerSec float64 + trafficShare float64 + lastSeen time.Time + Selected bool + DetailHref string +} + +type sourceDeviceAggregate struct { + sourceDeviceRow + observedBytes float64 + messagesPerSec float64 + bytesPerSec float64 + busLoad float64 + lastSeen time.Time +} + +type sourceDeviceSort struct { + key string + direction string +} + +type sourceDeviceSortControl struct { + Href string + AriaSort string + Indicator string + Accessible string +} + +type sourceDeviceSortView struct { + Address sourceDeviceSortControl + Messages sourceDeviceSortControl + Bytes sourceDeviceSortControl + Share sourceDeviceSortControl + LastSeen sourceDeviceSortControl +} + +const ( + sourceDeviceSortAddress = "address" + sourceDeviceSortMessages = "messages" + sourceDeviceSortBytes = "bytes" + sourceDeviceSortShare = "share" + sourceDeviceSortLastSeen = "last_seen" + sourceDeviceSortAsc = "asc" + sourceDeviceSortDesc = "desc" +) + +// sourceDeviceRows groups source/PGN metrics into one device row per observed +// source address. It uses process-lifetime payload bytes for traffic share so +// the percentages remain meaningful after an idle source's rate window decays. +func sourceDeviceRows(metrics []stats.SourcePGNMetric, sorting sourceDeviceSort) []sourceDeviceRow { + byAddress := make(map[uint8]*sourceDeviceAggregate) + totalObservedBytes := 0.0 + for _, metric := range metrics { + if !metric.Observed { + continue + } + device := byAddress[metric.SourceAddress] + if device == nil { + device = &sourceDeviceAggregate{sourceDeviceRow: sourceDeviceRow{Address: metric.SourceAddress}} + byAddress[metric.SourceAddress] = device + } + + observedBytes := float64(metric.Messages) * metric.PayloadBytesMean + device.observedBytes += observedBytes + totalObservedBytes += observedBytes + device.messagesPerSec += metric.RecentMessagesPerSec + device.bytesPerSec += metric.RecentBytesPerSec + device.busLoad += metric.EstimatedBusLoadPercent + if metric.LastSeen.After(device.lastSeen) { + device.lastSeen = metric.LastSeen + } + + if name, ok := sourceMetricDeviceName(metric); ok { + applySourceDeviceName(device, name) + } + if metric.ManufacturerCode != nil && device.ManufacturerCode == "" { + applySourceManufacturer(device, *metric.ManufacturerCode) + } + switch metric.PGN { + case 126996: + applySourceProductInformation(device, metric.Fields) + case 126998: + applySourceConfigurationInformation(device, metric.Fields) + } + } + + addresses := make([]uint8, 0, len(byAddress)) + for address := range byAddress { + addresses = append(addresses, address) + } + sort.Slice(addresses, func(i, j int) bool { + return addresses[i] < addresses[j] + }) + + rows := make([]sourceDeviceRow, 0, len(addresses)) + for _, address := range addresses { + device := byAddress[address] + share := 0.0 + if totalObservedBytes > 0 { + share = device.observedBytes / totalObservedBytes * 100 + } + device.sourceDeviceRow.messagesPerSec = device.messagesPerSec + device.sourceDeviceRow.bytesPerSec = device.bytesPerSec + device.trafficShare = share + device.sourceDeviceRow.lastSeen = device.lastSeen + device.TrafficShareText = fmt.Sprintf("%.1f%%", share) + device.MessagesPerSecText = fmt.Sprintf("%.2f", device.messagesPerSec) + device.BytesPerSecText = humanizeBytes(device.bytesPerSec, "/s") + device.BusLoadText = fmt.Sprintf("%.3f%%", device.busLoad) + device.LastSeenText = formatAge(time.Since(device.lastSeen).Seconds()) + device.LastSeenTitle = device.lastSeen.UTC().Format(time.RFC3339Nano) + rows = append(rows, device.sourceDeviceRow) + } + sortSourceDeviceRows(rows, sorting) + return rows +} + +func sourceDeviceSortFromQuery(key, direction string) sourceDeviceSort { + switch key { + case sourceDeviceSortMessages, sourceDeviceSortBytes, sourceDeviceSortShare, sourceDeviceSortLastSeen: + default: + key = sourceDeviceSortAddress + } + if direction != sourceDeviceSortAsc && direction != sourceDeviceSortDesc { + direction = defaultSourceDeviceSortDirection(key) + } + return sourceDeviceSort{key: key, direction: direction} +} + +func defaultSourceDeviceSortDirection(key string) string { + if key == sourceDeviceSortAddress { + return sourceDeviceSortAsc + } + return sourceDeviceSortDesc +} + +func sourceDeviceSortURL(base string, sorting sourceDeviceSort, selectedAddress *uint8, pgnSorting sourceDevicePGNSort) string { + query := url.Values{} + query.Set("sort", sorting.key) + query.Set("dir", sorting.direction) + if selectedAddress != nil { + query.Set("device", strconv.FormatUint(uint64(*selectedAddress), 10)) + query.Set("pgn_sort", pgnSorting.key) + query.Set("pgn_dir", pgnSorting.direction) + } + return base + "?" + query.Encode() +} + +func sourceDeviceSortControls(base string, current sourceDeviceSort, selectedAddress *uint8, pgnSorting sourceDevicePGNSort) sourceDeviceSortView { + return sourceDeviceSortView{ + Address: sourceDeviceSortControlFor(base, current, sourceDeviceSortAddress, "address", selectedAddress, pgnSorting), + Messages: sourceDeviceSortControlFor(base, current, sourceDeviceSortMessages, "messages per second", selectedAddress, pgnSorting), + Bytes: sourceDeviceSortControlFor(base, current, sourceDeviceSortBytes, "bytes per second", selectedAddress, pgnSorting), + Share: sourceDeviceSortControlFor(base, current, sourceDeviceSortShare, "traffic share", selectedAddress, pgnSorting), + LastSeen: sourceDeviceSortControlFor(base, current, sourceDeviceSortLastSeen, "last seen", selectedAddress, pgnSorting), + } +} + +func sourceDeviceSortControlFor(base string, current sourceDeviceSort, key, label string, selectedAddress *uint8, pgnSorting sourceDevicePGNSort) sourceDeviceSortControl { + nextDirection := defaultSourceDeviceSortDirection(key) + control := sourceDeviceSortControl{AriaSort: "none", Indicator: "↕"} + if current.key == key { + control.AriaSort = "ascending" + control.Indicator = "↑" + nextDirection = sourceDeviceSortDesc + if current.direction == sourceDeviceSortDesc { + control.AriaSort = "descending" + control.Indicator = "↓" + nextDirection = sourceDeviceSortAsc + } + } + control.Href = sourceDeviceSortURL(base, sourceDeviceSort{key: key, direction: nextDirection}, selectedAddress, pgnSorting) + control.Accessible = fmt.Sprintf("Sort by %s %s", label, sortDirectionWord(nextDirection)) + return control +} + +func sourceDeviceAddressFromQuery(raw string) *uint8 { + address, err := strconv.ParseUint(raw, 10, 8) + if err != nil { + return nil + } + value := uint8(address) + return &value +} + +func decorateSourceDeviceRows(rows []sourceDeviceRow, base string, sorting sourceDeviceSort, selectedAddress *uint8, pgnSorting sourceDevicePGNSort) { + for i := range rows { + address := rows[i].Address + rows[i].DetailHref = sourceDeviceSortURL(base, sorting, &address, pgnSorting) + rows[i].Selected = selectedAddress != nil && address == *selectedAddress + } +} + +func sortDirectionWord(direction string) string { + if direction == sourceDeviceSortDesc { + return "descending" + } + return "ascending" +} + +func sortSourceDeviceRows(rows []sourceDeviceRow, sorting sourceDeviceSort) { + sort.SliceStable(rows, func(i, j int) bool { + comparison := 0 + switch sorting.key { + case sourceDeviceSortMessages: + comparison = compareFloat(rows[i].messagesPerSec, rows[j].messagesPerSec) + case sourceDeviceSortBytes: + comparison = compareFloat(rows[i].bytesPerSec, rows[j].bytesPerSec) + case sourceDeviceSortShare: + comparison = compareFloat(rows[i].trafficShare, rows[j].trafficShare) + case sourceDeviceSortLastSeen: + comparison = compareTime(rows[i].lastSeen, rows[j].lastSeen) + default: + comparison = compareInt(int(rows[i].Address), int(rows[j].Address)) + } + if comparison == 0 { + return rows[i].Address < rows[j].Address + } + if sorting.direction == sourceDeviceSortDesc { + return comparison > 0 + } + return comparison < 0 + }) +} + +func compareFloat(left, right float64) int { + switch { + case left < right: + return -1 + case left > right: + return 1 + default: + return 0 + } +} + +func compareInt(left, right int) int { + switch { + case left < right: + return -1 + case left > right: + return 1 + default: + return 0 + } +} + +func sourceMetricDeviceName(metric stats.SourcePGNMetric) (uint64, bool) { + if metric.DeviceName != nil { + return *metric.DeviceName, true + } + if metric.DeviceNameHex != "" { + if name, err := strconv.ParseUint(strings.TrimPrefix(metric.DeviceNameHex, "0x"), 16, 64); err == nil { + return name, true + } + } + if metric.PGN != 60928 || metric.Raw == nil { + return 0, false + } + raw, err := hex.DecodeString(metric.Raw.LastHex) + if err != nil || len(raw) != 8 { + return 0, false + } + return binary.LittleEndian.Uint64(raw), true +} + +func applySourceDeviceName(device *sourceDeviceAggregate, rawName uint64) { + name := n2k.UnpackDeviceName(rawName) + device.DeviceName = fmt.Sprintf("0x%016X", rawName) + device.IdentityNumber = strconv.FormatUint(uint64(name.IdentityNumber), 10) + applySourceManufacturer(device, name.ManufacturerCode) + + className := pgn.DeviceClassConst(name.DeviceClass).String() + functionName := "" + if functions := pgn.DeviceFunctionConstMap[int(name.DeviceClass)]; functions != nil { + functionName = functions[int(name.DeviceFunction)] + } + if functionName == "" { + functionName = fmt.Sprintf("function %d", name.DeviceFunction) + } + device.Role = strings.TrimSpace(className + " / " + functionName) + device.Instances = fmt.Sprintf("device %d / system %d", name.DeviceInstance, name.SystemInstance) +} + +func applySourceManufacturer(device *sourceDeviceAggregate, code uint16) { + device.ManufacturerCode = strconv.FormatUint(uint64(code), 10) + device.Manufacturer = pgn.ManufacturerCodeConst(code).String() +} + +func applySourceProductInformation(device *sourceDeviceAggregate, fields []stats.FieldDistribution) { + values := sourceDeviceFieldValues(fields) + device.Model = values["modelId"] + device.ProductCode = values["productCode"] + device.Serial = values["modelSerialCode"] + device.Software = strings.TrimSpace(strings.Join(nonemptyStrings( + values["softwareVersionCode"], + values["modelVersion"], + ), " · ")) +} + +func applySourceConfigurationInformation(device *sourceDeviceAggregate, fields []stats.FieldDistribution) { + values := sourceDeviceFieldValues(fields) + device.ManufacturerInformation = values["manufacturerInformation"] +} + +func sourceDeviceFieldValues(fields []stats.FieldDistribution) map[string]string { + values := make(map[string]string, len(fields)) + for _, field := range fields { + if field.Last != "" { + values[field.Field] = strings.TrimSpace(field.Last) + } + } + return values +} + +func nonemptyStrings(values ...string) []string { + out := make([]string, 0, len(values)) + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + out = append(out, value) + } + } + return out +} diff --git a/internal/ui/source_devices_test.go b/internal/ui/source_devices_test.go new file mode 100644 index 0000000..0273187 --- /dev/null +++ b/internal/ui/source_devices_test.go @@ -0,0 +1,325 @@ +package ui + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "testing" + "time" + + n2k "github.com/open-ships/n2k" + "github.com/open-ships/n2k/pgn" + + "github.com/open-ships/beacon/internal/stats" +) + +func TestSourceDeviceRowsOrderIdentityAndTraffic(t *testing.T) { + now := time.Now().UTC() + garminName := n2k.DeviceName{ + IdentityNumber: 301, ManufacturerCode: uint16(pgn.Garmin), + DeviceClass: 25, DeviceFunction: 130, + }.Pack(true) + raymarineName := n2k.DeviceName{ + IdentityNumber: 1201, ManufacturerCode: uint16(pgn.Raymarine), + DeviceClass: 25, DeviceFunction: 130, DeviceInstance: 2, SystemInstance: 1, + }.Pack(true) + claimPayload := make([]byte, 8) + binary.LittleEndian.PutUint64(claimPayload, garminName) + + metrics := []stats.SourcePGNMetric{ + { + Observed: true, SourceAddress: 12, PGN: 126996, DeviceName: &raymarineName, + Messages: 3, PayloadBytesMean: 8, RecentMessagesPerSec: 1.25, RecentBytesPerSec: 6, + EstimatedBusLoadPercent: 0.25, LastSeen: now, + Fields: []stats.FieldDistribution{ + {Field: "productCode", Last: "2345"}, + {Field: "modelId", Last: "Axiom Pro"}, + {Field: "softwareVersionCode", Last: "4.8.1"}, + {Field: "modelVersion", Last: "Mk II"}, + {Field: "modelSerialCode", Last: "RAY-42"}, + }, + }, + { + Observed: true, SourceAddress: 3, PGN: 60928, + Messages: 1, PayloadBytesMean: 8, RecentMessagesPerSec: 0.5, RecentBytesPerSec: 2, + EstimatedBusLoadPercent: 0.05, LastSeen: now.Add(-time.Second), + Raw: &stats.RawPayloadDiagnostics{LastHex: hex.EncodeToString(claimPayload)}, + }, + { + Observed: true, SourceAddress: 12, PGN: 126998, DeviceName: &raymarineName, + RecentMessagesPerSec: 0.75, RecentBytesPerSec: 2, LastSeen: now, + Fields: []stats.FieldDistribution{ + {Field: "manufacturerInformation", Last: "Raymarine UK"}, + }, + }, + { + Observed: false, SourceAddress: 1, PGN: 127250, + Messages: 100, PayloadBytesMean: 8, LastSeen: now, + }, + } + + rows := sourceDeviceRows(metrics, sourceDeviceSortFromQuery("", "")) + if len(rows) != 2 { + t.Fatalf("device rows = %d, want 2: %+v", len(rows), rows) + } + if rows[0].Address != 3 || rows[1].Address != 12 { + t.Fatalf("addresses = [%d, %d], want [3, 12]", rows[0].Address, rows[1].Address) + } + if rows[0].DeviceName != fmt.Sprintf("0x%016X", garminName) { + t.Fatalf("claim Device NAME = %q, want packed Garmin NAME", rows[0].DeviceName) + } + if rows[0].Manufacturer != "Garmin" || rows[0].IdentityNumber != "301" { + t.Fatalf("claim identity = manufacturer %q / identity %q", rows[0].Manufacturer, rows[0].IdentityNumber) + } + if rows[1].Manufacturer != "Raymarine" || rows[1].ManufacturerCode != "1851" { + t.Fatalf("product manufacturer = %q (%s)", rows[1].Manufacturer, rows[1].ManufacturerCode) + } + if rows[1].ManufacturerInformation != "Raymarine UK" { + t.Fatalf("manufacturer information = %q", rows[1].ManufacturerInformation) + } + if rows[1].Model != "Axiom Pro" || rows[1].ProductCode != "2345" { + t.Fatalf("model/product = %q / %q", rows[1].Model, rows[1].ProductCode) + } + if rows[1].Instances != "device 2 / system 1" { + t.Fatalf("instances = %q", rows[1].Instances) + } + if rows[1].Software != "4.8.1 · Mk II" || rows[1].Serial != "RAY-42" { + t.Fatalf("software/serial = %q / %q", rows[1].Software, rows[1].Serial) + } + if rows[0].TrafficShareText != "25.0%" || rows[1].TrafficShareText != "75.0%" { + t.Fatalf("traffic shares = %q / %q", rows[0].TrafficShareText, rows[1].TrafficShareText) + } + if rows[0].MessagesPerSecText != "0.50" || rows[1].MessagesPerSecText != "2.00" { + t.Fatalf("message rates = %q / %q", rows[0].MessagesPerSecText, rows[1].MessagesPerSecText) + } + if rows[0].BytesPerSecText != "2 B/s" || rows[1].BytesPerSecText != "8 B/s" { + t.Fatalf("byte rates = %q / %q", rows[0].BytesPerSecText, rows[1].BytesPerSecText) + } +} + +func TestSourceDeviceRowsSortsByEverySupportedColumnAndDirection(t *testing.T) { + now := time.Now().UTC() + metrics := []stats.SourcePGNMetric{ + { + Observed: true, SourceAddress: 3, PGN: 127250, + Messages: 10, PayloadBytesMean: 10, + RecentMessagesPerSec: 2, RecentBytesPerSec: 30, LastSeen: now.Add(-2 * time.Second), + }, + { + Observed: true, SourceAddress: 7, PGN: 127251, + Messages: 30, PayloadBytesMean: 10, + RecentMessagesPerSec: 1, RecentBytesPerSec: 40, LastSeen: now, + }, + { + Observed: true, SourceAddress: 12, PGN: 127252, + Messages: 20, PayloadBytesMean: 10, + RecentMessagesPerSec: 3, RecentBytesPerSec: 20, LastSeen: now.Add(-time.Second), + }, + } + + for _, tc := range []struct { + key string + direction string + want []uint8 + }{ + {sourceDeviceSortAddress, sourceDeviceSortAsc, []uint8{3, 7, 12}}, + {sourceDeviceSortAddress, sourceDeviceSortDesc, []uint8{12, 7, 3}}, + {sourceDeviceSortMessages, sourceDeviceSortAsc, []uint8{7, 3, 12}}, + {sourceDeviceSortMessages, sourceDeviceSortDesc, []uint8{12, 3, 7}}, + {sourceDeviceSortBytes, sourceDeviceSortAsc, []uint8{12, 3, 7}}, + {sourceDeviceSortBytes, sourceDeviceSortDesc, []uint8{7, 3, 12}}, + {sourceDeviceSortShare, sourceDeviceSortAsc, []uint8{3, 12, 7}}, + {sourceDeviceSortShare, sourceDeviceSortDesc, []uint8{7, 12, 3}}, + {sourceDeviceSortLastSeen, sourceDeviceSortAsc, []uint8{3, 12, 7}}, + {sourceDeviceSortLastSeen, sourceDeviceSortDesc, []uint8{7, 12, 3}}, + } { + t.Run(tc.key+"_"+tc.direction, func(t *testing.T) { + rows := sourceDeviceRows(metrics, sourceDeviceSortFromQuery(tc.key, tc.direction)) + for i, want := range tc.want { + if rows[i].Address != want { + t.Fatalf("row %d address = %d, want %d; rows = %+v", i, rows[i].Address, want, rows) + } + } + }) + } +} + +func TestSourceDeviceSortControlsToggleAndDefault(t *testing.T) { + const base = "/frag/sources/src1/overview" + + defaultSort := sourceDeviceSortFromQuery("", "") + defaultPGNSort := sourceDevicePGNSortFromQuery("", "") + if defaultSort.key != sourceDeviceSortAddress || defaultSort.direction != sourceDeviceSortAsc { + t.Fatalf("default sort = %+v, want address ascending", defaultSort) + } + controls := sourceDeviceSortControls(base, defaultSort, nil, defaultPGNSort) + if controls.Address.AriaSort != "ascending" || controls.Address.Indicator != "↑" { + t.Fatalf("default address control = %+v", controls.Address) + } + if controls.Address.Href != base+"?dir=desc&sort=address" { + t.Fatalf("address toggle href = %q", controls.Address.Href) + } + if controls.Messages.Href != base+"?dir=desc&sort=messages" { + t.Fatalf("messages initial href = %q", controls.Messages.Href) + } + if controls.LastSeen.Href != base+"?dir=desc&sort=last_seen" { + t.Fatalf("last seen initial href = %q", controls.LastSeen.Href) + } + + messageSort := sourceDeviceSortFromQuery(sourceDeviceSortMessages, sourceDeviceSortDesc) + controls = sourceDeviceSortControls(base, messageSort, nil, defaultPGNSort) + if controls.Messages.AriaSort != "descending" || controls.Messages.Indicator != "↓" { + t.Fatalf("message control = %+v", controls.Messages) + } + if controls.Messages.Href != base+"?dir=asc&sort=messages" { + t.Fatalf("message toggle href = %q", controls.Messages.Href) + } + + lastSeenSort := sourceDeviceSortFromQuery(sourceDeviceSortLastSeen, sourceDeviceSortDesc) + controls = sourceDeviceSortControls(base, lastSeenSort, nil, defaultPGNSort) + if controls.LastSeen.AriaSort != "descending" || controls.LastSeen.Indicator != "↓" { + t.Fatalf("last seen control = %+v", controls.LastSeen) + } + if controls.LastSeen.Href != base+"?dir=asc&sort=last_seen" { + t.Fatalf("last seen toggle href = %q", controls.LastSeen.Href) + } +} + +func TestSourceDeviceDetailBuildsMetadataAndPGNStatistics(t *testing.T) { + now := time.Now().UTC() + address := uint8(12) + sorting := sourceDeviceSortFromQuery(sourceDeviceSortBytes, sourceDeviceSortDesc) + devices := []sourceDeviceRow{{ + Address: 12, DeviceName: "0x0102030405060708", IdentityNumber: "301", + Manufacturer: "Raymarine", ManufacturerCode: "1851", + Model: "Axiom Pro", ProductCode: "2345", Instances: "device 2 / system 1", + Software: "4.8.1", Serial: "RAY-42", + }} + metrics := []stats.SourcePGNMetric{ + { + Observed: true, SourceAddress: 12, PGN: 129025, PGNName: "Position, Rapid Update", + Status: "active", DecodeStatus: "decoded", Messages: 20, PayloadBytesMean: 8, + PayloadBytesLast: 8, RecentMessagesPerSec: 3.5, RecentBytesPerSec: 28, + EstimatedBusLoadPercent: 0.12, ExpectedPeriodSeconds: 0.1, + ShortestPeriodSeconds: 0.09, LongestPeriodSeconds: 0.14, + PeriodP90Seconds: 0.11, PeriodP99Seconds: 0.13, + JitterMADSeconds: 0.005, JitterPercent: 5, LastSeen: now, AgeSeconds: 0.2, + }, + { + Observed: true, SourceAddress: 12, PGN: 127250, PGNName: "Vessel Heading", + Status: "active", DecodeStatus: "decoded", Messages: 10, PayloadBytesMean: 8, + PayloadBytesLast: 8, RecentMessagesPerSec: 1, RecentBytesPerSec: 8, + ExpectedPeriodSeconds: 1, PeriodP90Seconds: 1.1, PeriodP99Seconds: 1.2, + LastSeen: now.Add(-time.Second), AgeSeconds: 1, + }, + } + + detail := sourceDeviceDetail( + metrics, + devices, + &address, + "/frag/sources/src1/overview", + sorting, + sourceDevicePGNSortFromQuery("", ""), + ) + if detail == nil { + t.Fatal("device detail is nil") + } + if detail.Device.Model != "Axiom Pro" || detail.Device.Serial != "RAY-42" { + t.Fatalf("detail metadata = %+v", detail.Device) + } + if len(detail.PGNs) != 2 || detail.PGNs[0].PGN != 127250 || detail.PGNs[1].PGN != 129025 { + t.Fatalf("PGN ordering = %+v", detail.PGNs) + } + if detail.PGNs[1].MessagesPerSec != "3.50" || detail.PGNs[1].BytesPerSec != "28 B/s" { + t.Fatalf("PGN rates = %+v", detail.PGNs[1]) + } + if detail.PGNs[1].PeriodP90 != "110ms" || detail.PGNs[1].PeriodP99 != "130ms" { + t.Fatalf("PGN percentiles = p90 %q / p99 %q", detail.PGNs[1].PeriodP90, detail.PGNs[1].PeriodP99) + } + if detail.RefreshHref != "/frag/sources/src1/overview?device=12&dir=desc&pgn_dir=asc&pgn_sort=pgn&sort=bytes" { + t.Fatalf("detail refresh href = %q", detail.RefreshHref) + } + if detail.CloseHref != "/frag/sources/src1/overview?dir=desc&sort=bytes" { + t.Fatalf("detail close href = %q", detail.CloseHref) + } +} + +func TestPGNActivityGrade(t *testing.T) { + cases := []struct { + name string + age float64 + periodP90 float64 + want string + }{ + {"unlearned cadence", 5, 0, "warming"}, + {"inside p90", 0.9, 1, "fresh"}, + {"sub-second slack absorbs refresh jitter", 0.55, 0.1, "fresh"}, + {"overdue past p90", 2, 1, "late"}, + {"missed several periods", 10, 1, "stale"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + level, label := pgnActivityGrade(tc.age, tc.periodP90) + if level != tc.want { + t.Fatalf("level = %q, want %q", level, tc.want) + } + if label == "" { + t.Fatal("label is empty; the dot needs a non-color signal") + } + }) + } +} + +func TestSourceDevicePGNSortingAndControls(t *testing.T) { + now := time.Now().UTC() + rows := []sourceDevicePGNRow{ + {PGN: 100, messagesPerSec: 1, trafficShare: 50, payloadBytesMean: 8, lastSeen: now.Add(-time.Second)}, + {PGN: 200, messagesPerSec: 10, trafficShare: 30, payloadBytesMean: 16, lastSeen: now.Add(-2 * time.Second)}, + {PGN: 300, messagesPerSec: 5, trafficShare: 20, payloadBytesMean: 4, lastSeen: now}, + } + cases := []struct { + key string + direction string + want []uint32 + }{ + {sourceDevicePGNSortPGN, sourceDeviceSortAsc, []uint32{100, 200, 300}}, + {sourceDevicePGNSortPGN, sourceDeviceSortDesc, []uint32{300, 200, 100}}, + {sourceDevicePGNSortRates, sourceDeviceSortAsc, []uint32{100, 300, 200}}, + {sourceDevicePGNSortRates, sourceDeviceSortDesc, []uint32{200, 300, 100}}, + {sourceDevicePGNSortTraffic, sourceDeviceSortAsc, []uint32{300, 200, 100}}, + {sourceDevicePGNSortTraffic, sourceDeviceSortDesc, []uint32{100, 200, 300}}, + {sourceDevicePGNSortPayload, sourceDeviceSortAsc, []uint32{300, 100, 200}}, + {sourceDevicePGNSortPayload, sourceDeviceSortDesc, []uint32{200, 100, 300}}, + {sourceDevicePGNSortActivity, sourceDeviceSortAsc, []uint32{200, 100, 300}}, + {sourceDevicePGNSortActivity, sourceDeviceSortDesc, []uint32{300, 100, 200}}, + } + for _, tc := range cases { + t.Run(tc.key+"_"+tc.direction, func(t *testing.T) { + got := append([]sourceDevicePGNRow(nil), rows...) + sortSourceDevicePGNRows(got, sourceDevicePGNSortFromQuery(tc.key, tc.direction)) + for i, want := range tc.want { + if got[i].PGN != want { + t.Fatalf("row %d PGN = %d, want %d; rows = %+v", i, got[i].PGN, want, got) + } + } + }) + } + + const base = "/frag/sources/src1/overview" + deviceSort := sourceDeviceSortFromQuery(sourceDeviceSortBytes, sourceDeviceSortDesc) + defaultSort := sourceDevicePGNSortFromQuery("", "") + if defaultSort.key != sourceDevicePGNSortPGN || defaultSort.direction != sourceDeviceSortAsc { + t.Fatalf("default PGN sort = %+v, want PGN ascending", defaultSort) + } + controls := sourceDevicePGNSortControls(base, deviceSort, 12, defaultSort) + if controls.PGN.AriaSort != "ascending" || controls.PGN.Indicator != "↑" { + t.Fatalf("default PGN control = %+v", controls.PGN) + } + if controls.PGN.Href != base+"?device=12&dir=desc&pgn_dir=desc&pgn_sort=pgn&sort=bytes" { + t.Fatalf("PGN toggle href = %q", controls.PGN.Href) + } + if controls.Rates.Href != base+"?device=12&dir=desc&pgn_dir=desc&pgn_sort=rates&sort=bytes" { + t.Fatalf("rates initial href = %q", controls.Rates.Href) + } +} diff --git a/internal/ui/stream.go b/internal/ui/stream.go new file mode 100644 index 0000000..7bf8134 --- /dev/null +++ b/internal/ui/stream.go @@ -0,0 +1,113 @@ +package ui + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/open-ships/beacon/internal/config" + "github.com/open-ships/beacon/internal/filter" + "github.com/open-ships/beacon/internal/msg" + "github.com/open-ships/beacon/internal/stats" +) + +const streamPreviewBuffer = 256 + +// handleComponentStream exposes a future-only, best-effort SSE tap used by +// source and sink overview pages. The tap is fed by stats.Registry at the +// source-received and sink-sent boundaries, so watching the UI neither +// consumes durable queue entries nor applies backpressure to routing. +func handleComponentStream(svc *config.Service, reg *stats.Registry, kind string, log *slog.Logger) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + var err error + switch kind { + case "source": + _, err = svc.GetSource(r.Context(), id) + case "sink": + _, err = svc.GetSink(r.Context(), id) + default: + http.Error(w, "unsupported stream kind", http.StatusInternalServerError) + return + } + if err != nil { + if errors.Is(err, config.ErrNotFound) { + http.NotFound(w, r) + return + } + log.Error("ui: stream entity lookup failed", "kind", kind, "id", id, "err", err) + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + + filterExpression := strings.TrimSpace(r.URL.Query().Get("filter")) + var streamFilter *filter.Chain + if filterExpression != "" { + streamFilter, err = filter.Compile([]string{filterExpression}) + if err != nil { + http.Error(w, "invalid CEL filter: "+err.Error(), http.StatusBadRequest) + return + } + } + + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", http.StatusInternalServerError) + return + } + + stream, unsubscribe := reg.SubscribeStream(kind, id, streamPreviewBuffer) + defer unsubscribe() + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Accel-Buffering", "no") + w.WriteHeader(http.StatusOK) + if _, err := io.WriteString(w, ": connected\n\n"); err != nil { + return + } + flusher.Flush() + + heartbeat := time.NewTicker(15 * time.Second) + defer heartbeat.Stop() + for { + select { + case <-r.Context().Done(): + return + case document, open := <-stream: + if !open { + return + } + if streamFilter != nil { + var envelope msg.Envelope + if err := json.Unmarshal(document, &envelope); err != nil { + log.Debug("ui: skip malformed stream envelope", "kind", kind, "id", id, "err", err) + continue + } + matches, err := streamFilter.Match(&envelope) + if err != nil { + log.Debug("ui: skip stream envelope after CEL evaluation error", "kind", kind, "id", id, "err", err) + continue + } + if !matches { + continue + } + } + if _, err := fmt.Fprintf(w, "data: %s\n\n", document); err != nil { + return + } + flusher.Flush() + case <-heartbeat.C: + if _, err := io.WriteString(w, ": keepalive\n\n"); err != nil { + return + } + flusher.Flush() + } + } + } +} diff --git a/internal/ui/templates/config.html b/internal/ui/templates/config.html index 5db5508..54f9461 100644 --- a/internal/ui/templates/config.html +++ b/internal/ui/templates/config.html @@ -17,7 +17,7 @@

Configuration JSON

{{end}} {{end}} -
+

Load JSON

diff --git a/internal/ui/templates/connectors.html b/internal/ui/templates/connectors.html index 40ea104..c2b2857 100644 --- a/internal/ui/templates/connectors.html +++ b/internal/ui/templates/connectors.html @@ -2,7 +2,7 @@

Connectors

- Add connector + Add connector
{{if .ConnectorForm}}{{template "connector-form" .ConnectorForm}}{{end}}
{{template "connector-panel" .}} diff --git a/internal/ui/templates/dashboard.html b/internal/ui/templates/dashboard.html index 1c481df..73af11f 100644 --- a/internal/ui/templates/dashboard.html +++ b/internal/ui/templates/dashboard.html @@ -1,5 +1,5 @@ {{/* Data: internal/ui/pages.go's pageData (this page carries nothing - beyond it — see ui.go's "GET /ui/dashboard" handler). The dashboard's + beyond it — see ui.go's "GET /dashboard" handler). The dashboard's actual content ships empty and fetches "dashboard-content" (frag_dashboard.html, part of fragTemplates) client-side immediately after load, then every 2s — exactly templates/connector_detail.html's @@ -9,6 +9,6 @@ renders one round-trip later). */}} {{define "content"}}
-
+
{{end}} diff --git a/internal/ui/templates/docs.html b/internal/ui/templates/docs.html index 77cbd40..e7c27b2 100644 --- a/internal/ui/templates/docs.html +++ b/internal/ui/templates/docs.html @@ -1,5 +1,5 @@ {{/* Data: internal/ui/docspages.go's docsPageData. Served at GET - /ui/docs/{slug} (see handleDocPage; GET /ui/docs itself 302s to the + /docs/{slug} (see handleDocPage; GET /docs itself 302s to the first page rather than rendering here). .Body is goldmark-rendered HTML (raw <...> in the source markdown renders escaped, not passed through — see docsMarkdown's doc comment), safe to emit @@ -8,7 +8,7 @@
diff --git a/internal/ui/templates/frag_connector_form.html b/internal/ui/templates/frag_connector_form.html index 4c8682f..461ba48 100644 --- a/internal/ui/templates/frag_connector_form.html +++ b/internal/ui/templates/frag_connector_form.html @@ -1,6 +1,6 @@ {{/* Data: internal/ui/forms.go's connectorFormViewData. Rendered by canonical connector create/edit pages and re-rendered by POST - /ui/connectors[/{id}] on a validation failure (see + /connectors[/{id}] on a validation failure (see writeConnector). Mirrors frag_source_form.html's id-immutable-on-edit and Enabled-checkbox shape; see its comments for rationale not repeated here. Unlike sources/sinks there is no type switch — source_id @@ -8,7 +8,7 @@ type-driven fieldset. */}} {{define "connector-form"}}

{{if .IsEdit}}Edit connector {{.ID}}{{else}}Add connector{{end}}

@@ -101,7 +101,7 @@

{{if .IsEdit}}Edit connector {{.ID}}{{else}}Add connector
- Cancel + Cancel

diff --git a/internal/ui/templates/frag_connector_stats.html b/internal/ui/templates/frag_connector_stats.html index 97f8100..c9b4ba7 100644 --- a/internal/ui/templates/frag_connector_stats.html +++ b/internal/ui/templates/frag_connector_stats.html @@ -1,5 +1,5 @@ {{/* Data: internal/ui/forms.go's connectorStatsData. Rendered at GET - /ui/frag/connectors/{id}/stats, hx-swap="outerHTML"'d in place of + /frag/connectors/{id}/stats, hx-swap="outerHTML"'d in place of connector_detail.html's #connector-stats container (or its own previous response — see below) every 2s. @@ -17,7 +17,7 @@ comment for the other half of this. */}} {{define "connector-stats"}}
+ hx-get="/frag/connectors/{{.ConnectorID}}/stats" hx-trigger="load, every 2s" hx-swap="outerHTML">
Total messages
{{.Snapshot.TotalMessages}}
diff --git a/internal/ui/templates/frag_connector_table.html b/internal/ui/templates/frag_connector_table.html index 4f50a8b..76e372c 100644 --- a/internal/ui/templates/frag_connector_table.html +++ b/internal/ui/templates/frag_connector_table.html @@ -35,8 +35,8 @@ {{range .Connectors}} - - {{.Name}} + + {{.Name}} {{.SourceName}} → {{.SinkName}} {{.EffectiveMode}} {{len .Filters}} @@ -53,9 +53,9 @@ {{.Snapshot.QueueDepth}} {{printf "%.2f" .Snapshot.MsgPerSec}} - Edit + Edit diff --git a/internal/ui/templates/frag_dashboard.html b/internal/ui/templates/frag_dashboard.html index f4b5d28..71d6205 100644 --- a/internal/ui/templates/frag_dashboard.html +++ b/internal/ui/templates/frag_dashboard.html @@ -1,5 +1,5 @@ {{/* Data: internal/ui/dashboard.go's dashboardData. Rendered at GET - /ui/frag/dashboard, hx-swap="innerHTML"'d into dashboard.html's + /frag/dashboard, hx-swap="innerHTML"'d into dashboard.html's #dashboard-panel container on load and every 2s (hx-trigger="load, every 2s") — see dashboard.html and pages.go's "dashboard" pages map entry doc comment for why the initial page ships @@ -17,9 +17,9 @@

Data flow

@@ -30,7 +30,7 @@

Data flow

Sinks
{{range .Flows}}
- +
source {{if eq .Source.State "up"}}up @@ -44,7 +44,7 @@

Data flow

-
+
connector {{if .Enabled}}enabled{{else}}disabled{{end}} @@ -66,7 +66,7 @@

Data flow

-
+
sink {{if eq .Sink.State "up"}}up @@ -104,13 +104,12 @@

Sources

Detail State Connectors - {{range .Sources}} - -
{{.Name}} + + {{.Name}} {{.ID}} {{.Type}} {{if .Detail}}{{.Detail}}{{else}}-{{end}} @@ -126,10 +125,9 @@

Sources

{{.ConnectorCount}} connector{{if ne .ConnectorCount 1}}s{{end}} - Edit {{else}} - No sources configured. + No sources configured. {{end}} @@ -150,13 +148,12 @@

Sinks

Detail State Connectors - {{range .Sinks}} - - {{.Name}} + + {{.Name}} {{.ID}} {{.Type}} {{if .Detail}}{{.Detail}}{{else}}-{{end}} @@ -172,10 +169,9 @@

Sinks

{{.ConnectorCount}} connector{{if ne .ConnectorCount 1}}s{{end}} - Edit {{else}} - No sinks configured. + No sinks configured. {{end}} @@ -198,18 +194,17 @@

Connectors

Filters Msg/s Queue - {{range .Flows}} - - {{.Name}} + + {{.Name}} {{.ID}} - {{.Source.Name}} + {{.Source.Name}} → - {{.Sink.Name}} + {{.Sink.Name}} {{if .Enabled}}enabled @@ -224,10 +219,9 @@

Connectors

{{len .Filters}} {{printf "%.2f" .Snapshot.MsgPerSec}} {{.Snapshot.QueueDepth}} - Edit {{else}} - No connectors configured. + No connectors configured. {{end}} @@ -262,7 +256,7 @@

SocketCAN diagnostics

Bus devices

Download report - {{if .CanCommitBaseline}}{{end}} + {{if .CanCommitBaseline}}{{end}}
@@ -290,7 +284,7 @@

Bus devices

{{.Endpoint}} {{if eq .Status "online"}}online{{else if eq .Status "new"}}new{{else if eq .Status "changed"}}changed{{else if eq .Status "missing"}}missing{{else}}{{if .Status}}{{.Status}}{{else}}live{{end}}{{end}} - +
{{.Address}} {{.Name}} {{if .Manufacturer}}{{.Manufacturer}}{{else}}-{{end}} diff --git a/internal/ui/templates/frag_filter_validate.html b/internal/ui/templates/frag_filter_validate.html index ec23de3..ef6d652 100644 --- a/internal/ui/templates/frag_filter_validate.html +++ b/internal/ui/templates/frag_filter_validate.html @@ -1,5 +1,5 @@ {{/* Data: internal/ui/forms.go's filterValidateData. Rendered for non-JSON - POST /ui/frag/validate-filters callers. The enhanced connector editor + POST /frag/validate-filters callers. The enhanced connector editor requests structured JSON from the same route; this fragment preserves the endpoint's HTML contract for simple clients and tests. Advisory only: svc.PutConnector re-validates authoritatively on Save. */}} diff --git a/internal/ui/templates/frag_overview_live.html b/internal/ui/templates/frag_overview_live.html index da133fe..7078d38 100644 --- a/internal/ui/templates/frag_overview_live.html +++ b/internal/ui/templates/frag_overview_live.html @@ -1,17 +1,18 @@ {{define "overview-live"}} -
-
-
+
+
+

Status

-
-
- {{if eq .State "up"}}up - {{else if eq .State "degraded"}}degraded - {{else if eq .State "error"}}error - {{else}}{{.State}}{{end}} - {{.ID}} +
+ {{if eq .State "up"}}up + {{else if eq .State "degraded"}}degraded + {{else if eq .State "error"}}error + {{else}}{{.State}}{{end}} + {{.ID}} +
{{range .Logs}} @@ -23,22 +24,19 @@

Status

No recent runtime errors.

{{end}}
-
- -
-
-

Statistics

-
+ {{/* Queue depth and retention are connector concepts: stats.SetQueueStats + is only ever called with a connector ID, so these tiles would read a + permanent zero on source and sink pages. */}}
-
Total messages{{.Snapshot.TotalMessages}}
Msg/s{{printf "%.2f" .Snapshot.MsgPerSec}}
-
Total bytes{{.TotalBytesText}}
Bytes/s{{.BytesPerSecText}}
{{if .Snapshot.Drops}}
Intake drops{{.Snapshot.Drops}}
{{end}} + {{if eq .Kind "connector"}}
Pending delivery{{.Snapshot.QueueDepth}}
Pending bytes{{.QueueBytesText}}
Retained history{{.Snapshot.RetainedDepth}}
Retained bytes{{.RetainedBytesText}}
+ {{end}} {{if .Snapshot.DeliveryClass}}
Delivery class{{.Snapshot.DeliveryClass}}
{{end}} {{if .Snapshot.QueueTail}}
Checkpoint / tail{{.Snapshot.QueueCursor}} / {{.Snapshot.QueueTail}}
{{end}}
@@ -46,168 +44,245 @@

Statistics

{{if eq .Kind "source"}} -
+
-

PGN traffic

-

One row per sender and PGN, including unknown wire payloads. Timing and value distributions use bounded recent windows.

-
-
- {{len .SourcePGNs}} streams - {{if .BaselineCount}}{{.BaselineCount}} expected{{end}} - - {{if .BaselineCount}} - - {{end}} +

Source devices

+

One row per observed source address. Device NAME is the stable identity; traffic share is based on bytes observed in this process.

-
- +
+
+
+ + + + + + + + + + - - - - - - - - + + + + + + + + - - {{range .SourcePGNs}} - - - - - - - - - - + + {{range .SourceDevices}} + {{template "source-device-row" .}} {{else}} - + {{template "source-device-empty-row"}} {{end}}
HealthPGN / decodeDevice / senderDestination / priorityRate / jitterLast seen / gapsTraffic / payloadValues / raw wire + + Device NAME / identityManufacturerClass / function + + + + + + + +
- {{if eq .Status "gap"}}gap - {{else if eq .Status "missing"}}missing - {{else if eq .Status "anomaly"}}anomaly - {{else if eq .Status "changed"}}changed - {{else if eq .Status "awaiting"}}awaiting - {{else if eq .Status "warming"}}learning - {{else}}active{{end}} - {{if eq .BaselineStatus "matching"}}baseline matched - {{else if eq .BaselineStatus "not_baselined"}}not baselined{{end}} - {{range .BaselineIssues}}{{.}}{{end}} - {{if .AnomalyText}}{{.AnomalyText}}{{end}} - {{.PGN}}{{if .PGNName}}
{{.PGNName}}{{end}}
- {{if eq .DecodeStatus "unknown"}}unknown - {{else if eq .DecodeStatus "decoded"}}decoded - {{else}}{{.DecodeStatus}}{{end}} - {{if .DecodeMetadata}}
{{.DecodeMetadata}}{{end}} -
{{.DecodeDetail}}
{{.SourceAddress}}{{if .DeviceNameHex}}
{{.DeviceNameHex}}{{end}}
dest {{.Destinations}}
priority {{.Priorities}}
{{.FrequencyText}}
{{.PeriodText}}
{{.RateDetail}}

- {{.GapText}}
{{.TrafficText}}
{{.TrafficDetail}}
- {{.PayloadText}}
{{.PayloadDetail}}
-
- {{len .Fields}} decoded fields{{if .Raw}} · {{.Raw.DistinctPayloads}} raw payloads{{end}} -
- {{if .Fields}} -

Decoded value distributions

- {{range .Fields}} -
-
{{.Name}}{{if .Anomalous}} changed{{end}}
- {{.Summary}} - {{.Samples}} samples · {{.AvailabilityText}}{{if .Anomalies}} · {{.Anomalies}} anomalies{{end}} - {{.QualityText}} -
- {{end}} - {{else}}

No decoded fields are available for this PGN.

{{end}} - - {{if .Raw}} -

Raw wire payload

-
-
Last payload
{{.Raw.LastHex}}
-
Fingerprint
{{.Raw.LastFingerprint}}
-
Lengths
{{range $length, $count := .Raw.LengthCounts}}{{$length}} B × {{$count}}  {{end}}
-
Unchanged
{{printf "%.1f" .Raw.UnchangedSeconds}} s
-
Hamming distance
mean {{printf "%.1f" .Raw.HammingDistanceMean}} bits · p95 {{printf "%.1f" .Raw.HammingDistanceP95}} bits
-
Last changed bytes
{{if .Raw.LastChangedBytes}}{{.Raw.LastChangedBytes}}{{else}}none{{end}}
-
- {{if .RawFingerprints}} -

Top payload fingerprints

-
{{range .RawFingerprints}}{{.Fingerprint}} {{.Count}}× ({{.Share}}) · {{.Length}} B
{{end}}
- {{end}} - {{if .RawBytes}} -

Byte-position analysis

-
- {{range .RawBytes}}{{end}} -
ByteRangeModeEntropyChangedBit mask
{{.Offset}}{{.Range}}{{.Mode}}{{.Entropy}}{{.Changed}}{{.BitMask}}
- {{end}} - {{if .RawSamples}} -

Recent changed payloads

-
{{range .RawSamples}}
{{.Hex}} {{.Length}} B · {{.Fingerprint}}
{{end}}
- {{end}} - {{end}} -
-
-
No PGNs have been observed on this source in this process.
+
-
-
-

Traffic changes

-
- - {{range .SourceEvents}} - {{else}}{{end}} -
TimeSeverityChangePGNSender
{{.Time}}{{.Severity}}{{.Kind}}
{{.Summary}}
{{if .PGN}}{{.PGN}}{{else}}-{{end}}{{.SourceAddress}}
No source traffic changes have been recorded.
-
-
- {{end}} + {{if .SourceDeviceDetail}} + +
+
+
+

Device {{.SourceDeviceDetail.Device.Address}}

+

+ {{if .SourceDeviceDetail.Device.Model}}{{.SourceDeviceDetail.Device.Model}} + {{else if .SourceDeviceDetail.Device.Manufacturer}}{{.SourceDeviceDetail.Device.Manufacturer}} device + {{else}}Device address {{.SourceDeviceDetail.Device.Address}}{{end}} +

+ {{if .SourceDeviceDetail.Device.Role}}

{{.SourceDeviceDetail.Device.Role}}

{{end}} +
+ + +
-
-
-

Message stream

-
-
- - - - - - - - - - - - - - - {{range .Events}} - - - - - - - - - - - {{else}} - - {{end}} - -
TimeStagePGNSourceDestPriorityConnectorPayload
{{.TimeText}}{{.Stage}}{{.PGN}}{{if .PGNName}}
{{.PGNName}}{{end}}
{{.Source}}{{.Dest}}{{.Priority}}{{if .ConnectorID}}{{.ConnectorID}}{{else}}-{{end}}{{if .Payload}}{{.Payload}}{{else}}empty{{end}}
{{.EmptyStream}}
+
+
+

PGN statistics

+

Live rates use the recent traffic window; timing percentiles use the bounded interval sample.

+
+ {{len .SourceDeviceDetail.PGNs}} PGNs +
+
+ + + + + + + + + + + + + {{range .SourceDeviceDetail.PGNs}} + + + + + + + + + {{else}} + + {{end}} + +
+ + + + + + Timing + + + +
+ {{.PGN}}{{if .PGNName}}
{{.PGNName}}{{end}} +
{{.MessagesPerSec}} msg/s
{{.BytesPerSec}}
{{.DeviceTraffic}} device
{{.EstimatedBusLoad}} bus load
+ Median{{.MedianPeriod}} + P90{{.PeriodP90}} + P99{{.PeriodP99}} +
{{.Payload}} + + +
No PGNs have been observed for this device.
+
+
+
+ {{else}} + + {{end}}
+ {{end}}
{{end}} + +{{define "source-device-row"}} + + {{template "source-device-row-cells" .}} + +{{end}} + +{{define "source-device-row-cells"}} +{{.Address}} +{{if .DeviceName}}{{.DeviceName}}
identity {{.IdentityNumber}}{{else}}not claimed{{end}} +{{if .Manufacturer}}{{.Manufacturer}}
code {{.ManufacturerCode}}{{if .ManufacturerInformation}} · {{.ManufacturerInformation}}{{end}}{{else}}-{{end}} +{{if .Role}}{{.Role}}{{else}}-{{end}} +{{.MessagesPerSecText}} +{{.BytesPerSecText}} +{{.TrafficShareText}}
~{{.BusLoadText}} bus load + +{{end}} + +{{define "source-device-empty-row"}} +No devices have been observed on this source in this process. +{{end}} + +{{define "source-device-row-snapshot"}} + + {{range .SourceDevices}} + {{template "source-device-row" .}} + {{else}} + {{template "source-device-empty-row"}} + {{end}} +
+{{end}} diff --git a/internal/ui/templates/frag_sink_form.html b/internal/ui/templates/frag_sink_form.html index 2441451..b8c1dd4 100644 --- a/internal/ui/templates/frag_sink_form.html +++ b/internal/ui/templates/frag_sink_form.html @@ -2,7 +2,7 @@ frag_source_form.html; see its comments for rationale. */}} {{define "sink-form"}}

{{if .IsEdit}}Edit sink {{.ID}}{{else}}Add sink{{end}}

@@ -28,7 +28,7 @@

{{if .IsEdit}}Edit sink {{.ID}}{{else}}Add sink{{end}} Type @@ -51,7 +51,7 @@

{{if .IsEdit}}Edit source {{.ID}}{{else}}Add source{{end} {{template "source-type-fields" .TypeFields}}

- Cancel + Cancel
diff --git a/internal/ui/templates/frag_source_table.html b/internal/ui/templates/frag_source_table.html index 38b60c8..e487e03 100644 --- a/internal/ui/templates/frag_source_table.html +++ b/internal/ui/templates/frag_source_table.html @@ -36,8 +36,8 @@ {{range .Sources}} - - {{.Name}} + + {{.Name}} {{.ID}} {{.Type}} {{if .Detail}}{{.Detail}}{{else}}-{{end}} @@ -52,9 +52,9 @@ {{else}}{{.State}}{{end}} - Edit + Edit diff --git a/internal/ui/templates/frag_source_type_fields.html b/internal/ui/templates/frag_source_type_fields.html index 2628f78..53588e9 100644 --- a/internal/ui/templates/frag_source_type_fields.html +++ b/internal/ui/templates/frag_source_type_fields.html @@ -1,5 +1,5 @@ {{/* Data: internal/ui/forms.go's sourceTypeFieldsData. Rendered both - standalone (GET /ui/frag/source-type-fields, the type select's hx-get + standalone (GET /frag/source-type-fields, the type select's hx-get target) and inline inside frag_source_form.html's "source-form". Only the selected type's fields render as inputs, so values typed into another type's fields are discarded when the type select changes @@ -65,7 +65,7 @@ Capture file path -

Replays a candump/canboat/YD/Actisense log at its recorded timing.

+

Replays a candump/canboat/YD/Actisense log at its recorded timing. Gzip is detected automatically; if this path is missing, its .gz counterpart is tried.

{{end}} {{end}} diff --git a/internal/ui/templates/layout.html b/internal/ui/templates/layout.html index 10c9526..561199a 100644 --- a/internal/ui/templates/layout.html +++ b/internal/ui/templates/layout.html @@ -4,21 +4,21 @@ {{.Title}} - beacon - - - - + + + +
diff --git a/internal/ui/templates/mcp.html b/internal/ui/templates/mcp.html index 6a2904c..4f02a48 100644 --- a/internal/ui/templates/mcp.html +++ b/internal/ui/templates/mcp.html @@ -167,9 +167,9 @@

Read health

{"name":"get_health","arguments":{}}
-

Read one route's delivery statistics

+

Read one route's delivery metrics

{
-  "name": "get_delivery_statistics",
+  "name": "get_delivery_metrics",
   "arguments": {"connector_id": "navigation"}
 }
@@ -178,13 +178,6 @@

Inspect one source's PGN traffic

{
   "name": "get_source_metrics",
   "arguments": {"source_id": "can0"}
-}
- -
-

Set expected traffic baseline

-
{
-  "name": "commit_source_traffic_baseline",
-  "arguments": {"source_id": "can0"}
 }
@@ -195,7 +188,7 @@

Set expected traffic baseline

Writes
Validated, persisted to SQLite, and reconciled against the running graph in the same call.
Tool errors
Invalid ids, types, references, CEL, durations, and in-use deletes return correctable MCP tool errors.
-
Statistics
Reports pending delivery separately from retained history, plus per-source/sender PGN rates, payload sizes, learned gaps, anomalies, and decoded value distributions.
+
Metrics
Reports pending delivery separately from retained history, plus per-source/sender PGN rates, payload sizes, learned gaps, and decoded value distributions.
Network
No cloud service, CDN, remote schema, telemetry endpoint, or internet connection is required.
diff --git a/internal/ui/templates/overview.html b/internal/ui/templates/overview.html index 43f312c..e03743c 100644 --- a/internal/ui/templates/overview.html +++ b/internal/ui/templates/overview.html @@ -21,7 +21,74 @@

Configuration

+ {{if .StreamHref}} +
+
+
+

Stream contents

+

Future messages only. This best-effort preview never blocks routing and keeps the latest 200 messages in this browser.

+
+
+ + + + +
+
+ +
+
+ + +
+
+ + + + +
+ Stopped · 0 captured +
+ + + +
+ Streaming is stopped. + Choose Start to capture messages arriving at this {{.Kind}}. +
+ +
+ {{end}} +
+ hx-get="{{.LiveHref}}" hx-trigger="load, every 500ms" hx-swap="outerHTML"> {{end}} diff --git a/internal/ui/templates/sinks.html b/internal/ui/templates/sinks.html index fd5a75b..f558a4c 100644 --- a/internal/ui/templates/sinks.html +++ b/internal/ui/templates/sinks.html @@ -3,7 +3,7 @@

Sinks

{{if not .SinkForm}} - Add sink + Add sink {{end}}
{{if .SinkForm}}{{template "sink-form" .SinkForm}}{{end}}
diff --git a/internal/ui/templates/sources.html b/internal/ui/templates/sources.html index 16c3fe4..7243865 100644 --- a/internal/ui/templates/sources.html +++ b/internal/ui/templates/sources.html @@ -3,7 +3,7 @@

Sources

{{if not .SourceForm}} - Add source + Add source {{end}}
{{if .SourceForm}}{{template "source-form" .SourceForm}}{{end}}
diff --git a/internal/ui/ui.go b/internal/ui/ui.go index 1ca63b8..ee9a4e6 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -1,4 +1,4 @@ -// Package ui is beacon's offline, server-rendered web UI mounted at /ui/. +// Package ui is beacon's offline, server-rendered web UI mounted at /. // Every asset it serves (htmx, the CEL autocomplete enhancement, and the // lightweight Open Ships stylesheet) is embedded in the binary via go:embed — // beacon is an offline gateway appliance, so the UI must render with no @@ -32,7 +32,7 @@ type RuntimeInfo struct { } // assetsFS embeds every vendored/compiled static file this package serves -// under /ui/assets/ — see the package doc comment and assets/README.md. +// under /assets/ — see the package doc comment and assets/README.md. // //go:embed assets/* var assetsFS embed.FS @@ -55,23 +55,16 @@ var assetsFS embed.FS // re-vendors an asset would keep serving browsers their stale cached copy. // Development builds usually pass "dev", so assetCacheVersion replaces that // non-unique value with a hash of the embedded UI assets; otherwise `go run` -// iterations would keep reusing /ui/assets/app.css?v=dev while the browser +// iterations would keep reusing /assets/app.css?v=dev while the browser // quite correctly holds onto its immutable cached copy. // // A nil log defaults to slog.Default(), the same convention as api.New and // config.NewService; render failures are logged through it (see render.go). // -// The returned handler is an *http.ServeMux serving "GET /ui/" routes -// plus "GET /ui/assets/", wrapped in sameOriginGuard (see its doc comment). -// It is mounted at internal/app/app.go as mux.Handle("/ui/", handler) AND -// mux.Handle("/ui", handler) — the second, exact-path mount is required -// too: without it, a bare "GET /ui" request never reaches this handler's -// own "GET /ui" redirect route below, because net/http.ServeMux intercepts -// it first with its own 301-to-"/ui/" redirect (a subtree pattern's -// implicit behavior for the path with the trailing slash removed) unless an -// exact-path registration for "/ui" already exists at that outer mux. -// Routes below are registered with the full "/ui/..." path since the mux -// they're added to isn't stripped. +// The returned handler is an *http.ServeMux serving root-level UI routes +// such as GET /dashboard and GET /sources, plus GET /assets/. internal/app +// mounts it as the "/" fallback after registering the API, MCP, health, and +// metrics endpoints. sameOriginGuard protects its state-changing routes. func Handler(svc *config.Service, reg *stats.Registry, statuses func() []supervisor.Status, devices func() []bus.DeviceInfo, version string, log *slog.Logger, runtimeInfo ...RuntimeInfo) http.Handler { if log == nil { log = slog.Default() @@ -83,29 +76,21 @@ func Handler(svc *config.Service, reg *stats.Registry, statuses func() []supervi } mux := http.NewServeMux() - // GET /ui and GET /ui/ both land the operator on the dashboard. Both - // patterns are registered here (rather than relying on http.ServeMux's - // built-in "add the trailing slash" redirect for the bare "/ui") so - // each is a single-hop 302 straight to /ui/dashboard — see the - // package-level mounting note above: the parent mux this Handler is - // mounted under must forward exact "/ui" requests here unmodified (in - // addition to its "/ui/" subtree mount) for the "GET /ui" pattern below - // to ever be reached. + // The bare admin root lands the operator on the dashboard. redirectToDashboard := func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, "/ui/dashboard", http.StatusFound) + http.Redirect(w, r, "/dashboard", http.StatusFound) } - mux.HandleFunc("GET /ui", redirectToDashboard) - mux.HandleFunc("GET /ui/{$}", redirectToDashboard) + mux.HandleFunc("GET /{$}", redirectToDashboard) - mux.HandleFunc("GET /ui/dashboard", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("GET /dashboard", func(w http.ResponseWriter, r *http.Request) { // The dashboard is where create handlers land the operator via // HX-Redirect, so it consumes the one-shot flash (see flash.go). data := newPageData("Home", assetVersion, "dashboard") data.Flash = takeFlash(w, r) renderPage(w, log, "dashboard", data) }) - mux.HandleFunc("GET /ui/frag/dashboard", handleDashboardFrag(svc, reg, statuses, devices, runtime, log)) - mux.HandleFunc("POST /ui/n2k/inventory/baseline", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("GET /frag/dashboard", handleDashboardFrag(svc, reg, statuses, devices, runtime, log)) + mux.HandleFunc("POST /n2k/inventory/baseline", func(w http.ResponseWriter, r *http.Request) { if runtime.Inventory == nil { http.Error(w, "inventory unavailable", http.StatusServiceUnavailable) return @@ -115,10 +100,10 @@ func Handler(svc *config.Service, reg *stats.Registry, statuses func() []supervi http.Error(w, "baseline failed", http.StatusInternalServerError) return } - w.Header().Set("HX-Redirect", "/ui/dashboard") + w.Header().Set("HX-Redirect", "/dashboard") w.WriteHeader(http.StatusNoContent) }) - mux.HandleFunc("POST /ui/n2k/inventory/{name}/label", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("POST /n2k/inventory/{name}/label", func(w http.ResponseWriter, r *http.Request) { if runtime.Inventory == nil { http.Error(w, "inventory unavailable", http.StatusServiceUnavailable) return @@ -137,7 +122,7 @@ func Handler(svc *config.Service, reg *stats.Registry, statuses func() []supervi http.Error(w, "label failed", http.StatusInternalServerError) return } - w.Header().Set("HX-Redirect", "/ui/dashboard") + w.Header().Set("HX-Redirect", "/dashboard") w.WriteHeader(http.StatusNoContent) }) @@ -145,51 +130,53 @@ func Handler(svc *config.Service, reg *stats.Registry, statuses func() []supervi // its create/update/delete write endpoints. See forms.go for every // handler constructor below and the behavior contract in its package // doc comment. - mux.HandleFunc("GET /ui/sources", handleSourcesPage(svc, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/sources/new", handleSourceNewPage(svc, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/sources/{id}/edit", handleSourceEditPage(svc, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/sources/{id}", redirectToTrailingSlash) - mux.HandleFunc("GET /ui/sources/{id}/{$}", handleSourceOverviewPage(svc, reg, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/frag/sources/{id}/overview", handleSourceOverviewFrag(svc, reg, statuses, log)) - mux.HandleFunc("POST /ui/sources/{id}/traffic-baseline", handleSourceTrafficBaselineCommit(svc, reg, statuses, log)) - mux.HandleFunc("POST /ui/sources/{id}/traffic-baseline/clear", handleSourceTrafficBaselineClear(svc, reg, statuses, log)) - mux.HandleFunc("GET /ui/frag/source-type-fields", handleSourceTypeFieldsFrag(log)) - mux.HandleFunc("POST /ui/sources", handleSourceCreate(svc, log)) - mux.HandleFunc("POST /ui/sources/{id}", handleSourceUpdate(svc, log)) - mux.HandleFunc("POST /ui/sources/{id}/delete", handleSourceDelete(svc, log)) + mux.HandleFunc("GET /sources", handleSourcesPage(svc, statuses, assetVersion, log)) + mux.HandleFunc("GET /sources/new", handleSourceNewPage(svc, statuses, assetVersion, log)) + mux.HandleFunc("GET /sources/{id}/edit", handleSourceEditPage(svc, statuses, assetVersion, log)) + mux.HandleFunc("GET /sources/{id}", redirectToTrailingSlash) + mux.HandleFunc("GET /sources/{id}/{$}", handleSourceOverviewPage(svc, reg, statuses, assetVersion, log)) + mux.HandleFunc("GET /frag/sources/{id}/overview", handleSourceOverviewFrag(svc, reg, statuses, log)) + mux.HandleFunc("GET /frag/sources/{id}/device-rows", handleSourceDeviceRowsFrag(svc, reg, statuses, log)) + mux.HandleFunc("GET /ui/streams/sources/{id}", handleComponentStream(svc, reg, "source", log)) + mux.HandleFunc("GET /frag/source-type-fields", handleSourceTypeFieldsFrag(log)) + mux.HandleFunc("POST /sources", handleSourceCreate(svc, log)) + mux.HandleFunc("POST /sources/{id}", handleSourceUpdate(svc, log)) + mux.HandleFunc("POST /sources/{id}/delete", handleSourceDelete(svc, log)) // Sinks: exactly parallel to sources above. - mux.HandleFunc("GET /ui/sinks", handleSinksPage(svc, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/sinks/new", handleSinkNewPage(svc, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/sinks/{id}/edit", handleSinkEditPage(svc, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/sinks/{id}", redirectToTrailingSlash) - mux.HandleFunc("GET /ui/sinks/{id}/{$}", handleSinkOverviewPage(svc, reg, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/frag/sinks/{id}/overview", handleSinkOverviewFrag(svc, reg, statuses, log)) - mux.HandleFunc("GET /ui/frag/sink-type-fields", handleSinkTypeFieldsFrag(log)) - mux.HandleFunc("POST /ui/sinks", handleSinkCreate(svc, log)) - mux.HandleFunc("POST /ui/sinks/{id}", handleSinkUpdate(svc, log)) - mux.HandleFunc("POST /ui/sinks/{id}/delete", handleSinkDelete(svc, log)) + mux.HandleFunc("GET /sinks", handleSinksPage(svc, statuses, assetVersion, log)) + mux.HandleFunc("GET /sinks/new", handleSinkNewPage(svc, statuses, assetVersion, log)) + mux.HandleFunc("GET /sinks/{id}/edit", handleSinkEditPage(svc, statuses, assetVersion, log)) + mux.HandleFunc("GET /sinks/{id}", redirectToTrailingSlash) + mux.HandleFunc("GET /sinks/{id}/{$}", handleSinkOverviewPage(svc, reg, statuses, assetVersion, log)) + mux.HandleFunc("GET /frag/sinks/{id}/overview", handleSinkOverviewFrag(svc, reg, statuses, log)) + mux.HandleFunc("GET /ui/streams/sinks/{id}", handleComponentStream(svc, reg, "sink", log)) + mux.HandleFunc("GET /frag/sink-type-fields", handleSinkTypeFieldsFrag(log)) + mux.HandleFunc("POST /sinks", handleSinkCreate(svc, log)) + mux.HandleFunc("POST /sinks/{id}", handleSinkUpdate(svc, log)) + mux.HandleFunc("POST /sinks/{id}/delete", handleSinkDelete(svc, log)) // Connectors: the list/add/edit/delete pages parallel sources/sinks // above, plus a per-connector overview page with live stats/streaming // data and live CEL validation diagnostics. See forms.go's // "--- Connectors ---" section for the behavior contract. - mux.HandleFunc("GET /ui/connectors", handleConnectorsPage(svc, reg, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/connectors/new", handleConnectorNewPage(svc, reg, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/connectors/{id}/edit", handleConnectorEditPage(svc, reg, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/connectors/{id}", redirectToTrailingSlash) - mux.HandleFunc("GET /ui/connectors/{id}/{$}", handleConnectorOverviewPage(svc, reg, statuses, assetVersion, log)) - mux.HandleFunc("GET /ui/cel-completions", handleCELCompletions) - mux.HandleFunc("POST /ui/frag/validate-filters", handleValidateFiltersFrag(svc, log)) - mux.HandleFunc("GET /ui/frag/connectors/{id}/stats", handleConnectorStatsFrag(svc, reg, log)) - mux.HandleFunc("GET /ui/frag/connectors/{id}/overview", handleConnectorOverviewFrag(svc, reg, statuses, log)) - mux.HandleFunc("POST /ui/connectors", handleConnectorCreate(svc, reg, statuses, log)) - mux.HandleFunc("POST /ui/connectors/{id}", handleConnectorUpdate(svc, reg, statuses, log)) - mux.HandleFunc("POST /ui/connectors/{id}/delete", handleConnectorDelete(svc, reg, statuses, log)) + mux.HandleFunc("GET /connectors", handleConnectorsPage(svc, reg, statuses, assetVersion, log)) + mux.HandleFunc("GET /connectors/new", handleConnectorNewPage(svc, reg, statuses, assetVersion, log)) + mux.HandleFunc("GET /connectors/{id}/edit", handleConnectorEditPage(svc, reg, statuses, assetVersion, log)) + mux.HandleFunc("GET /connectors/{id}", redirectToTrailingSlash) + mux.HandleFunc("GET /connectors/{id}/{$}", handleConnectorOverviewPage(svc, reg, statuses, assetVersion, log)) + mux.HandleFunc("GET /cel-completions", handleCELCompletions) + mux.HandleFunc("POST /frag/validate-filters", handleValidateFiltersFrag(svc, log)) + mux.HandleFunc("GET /frag/connectors/{id}/stats", handleConnectorStatsFrag(svc, reg, log)) + mux.HandleFunc("GET /frag/connectors/{id}/overview", handleConnectorOverviewFrag(svc, reg, statuses, log)) + mux.HandleFunc("POST /connectors", handleConnectorCreate(svc, reg, statuses, log)) + mux.HandleFunc("POST /connectors/{id}", handleConnectorUpdate(svc, reg, statuses, log)) + mux.HandleFunc("POST /connectors/{id}/delete", handleConnectorDelete(svc, reg, statuses, log)) - mux.HandleFunc("GET /ui/config", handleConfigPage(svc, assetVersion, log)) - mux.HandleFunc("POST /ui/config/import", handleConfigImport(svc, assetVersion, log)) - mux.HandleFunc("GET /ui/mcp", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("GET /config", handleConfigPage(svc, assetVersion, log)) + mux.HandleFunc("POST /config/import", handleConfigImport(svc, assetVersion, log)) + // /mcp itself is the protocol endpoint registered by internal/app. + mux.HandleFunc("GET /mcp/info", func(w http.ResponseWriter, r *http.Request) { data := mcpPageData{ pageData: newPageData("MCP", assetVersion, "mcp"), Tools: mcpserver.Catalog(), @@ -197,13 +184,10 @@ func Handler(svc *config.Service, reg *stats.Registry, statuses func() []supervi renderPage(w, log, "mcp", data) }) - // Docs: the onboard operator manual — see docspages.go. /ui/docs 302s to - // the first page; /ui/docs/{slug} serves one page or 404s for an unknown - // slug. internal/app additionally 301-redirects the bare "/docs" and - // "/docs/{slug}" paths (mounted on the admin mux, not this one) to these - // two routes. - mux.HandleFunc("GET /ui/docs", handleDocsIndex()) - mux.HandleFunc("GET /ui/docs/{slug}", handleDocPage(assetVersion, log)) + // Docs: /docs redirects to the first onboard manual page; /docs/{slug} + // serves one page or returns 404 for an unknown slug. + mux.HandleFunc("GET /docs", handleDocsIndex()) + mux.HandleFunc("GET /docs/{slug}", handleDocPage(assetVersion, log)) assets, err := fs.Sub(assetsFS, "assets") if err != nil { @@ -211,8 +195,8 @@ func Handler(svc *config.Service, reg *stats.Registry, statuses func() []supervi // directive below, so it always exists in assetsFS. panic(err) } - fileServer := http.StripPrefix("/ui/assets/", http.FileServer(http.FS(assets))) - mux.Handle("GET /ui/assets/", withImmutableCache(fileServer)) + fileServer := http.StripPrefix("/assets/", http.FileServer(http.FS(assets))) + mux.Handle("GET /assets/", withImmutableCache(fileServer)) return sameOriginGuard(mux) } @@ -245,8 +229,8 @@ func assetCacheVersion(version string) string { // sameOriginGuard wraps next so every POST request is checked against a // same-origin policy before reaching next — beacon's only defense against // cross-site form/fetch submissions forging a write through the UI's -// state-changing endpoints (POST /ui/sources, /ui/sinks, /ui/connectors, -// and their .../delete and /ui/frag/validate-filters counterparts). GET +// state-changing endpoints (POST /sources, /sinks, /connectors, +// and their .../delete and /frag/validate-filters counterparts). GET // requests (and every other method) pass through untouched: they're not // state-changing, so there's nothing here for a forged cross-site request // to gain by making one. @@ -266,7 +250,7 @@ func assetCacheVersion(version string) string { // // A request carrying NEITHER header — curl, most non-browser HTTP clients, // and any htmx request a stripping proxy scrubbed both headers from — is -// allowed. beacon has no cookie or bearer-token auth on /ui/* for such a +// allowed. beacon has no cookie or bearer-token auth on /* for such a // client to have forged in the first place, so a headerless request is // exactly as trusted as one that proves same-origin. func sameOriginGuard(next http.Handler) http.Handler { @@ -299,7 +283,7 @@ func sameOrigin(r *http.Request) bool { // withImmutableCache marks every response next serves as safe for clients // and intermediaries to cache for a year without revalidating. Safe because -// content under /ui/assets/ is embedded in the binary and versioned by the +// content under /assets/ is embedded in the binary and versioned by the // binary itself (see Handler's "?v=" doc comment above) — the same pattern // internal/api/docsui.go uses for /api/assets/. func withImmutableCache(next http.Handler) http.Handler { diff --git a/internal/ui/ui_test.go b/internal/ui/ui_test.go index 1b3e023..cd84c9b 100644 --- a/internal/ui/ui_test.go +++ b/internal/ui/ui_test.go @@ -30,14 +30,9 @@ type fakeReconciler struct{} func (fakeReconciler) Reconcile(ctx context.Context) error { return nil } func (fakeReconciler) Statuses() []supervisor.Status { return nil } -// newAppMountedServer builds ui.Handler and serves it exactly as -// internal/app mounts it in the real deployment: mux.Handle("/ui/", -// handler), mux.Handle("/ui", handler) (see ui.Handler's doc comment for -// why both are needed), plus the "GET /{$}" -> /ui/dashboard redirect (see -// app.go's Run). ui.go's routes are registered with full "/ui/..." paths on -// the assumption they're reached this way, so tests exercise that exact -// shape rather than serving the handler bare at "/" (same reasoning as -// internal/api/docsui_test.go's newAppMountedServer). +// newAppMountedServer serves ui.Handler as internal/app's root fallback. +// The handler itself owns the bare-root redirect and every root-level UI +// route; app registers the more-specific API, MCP, health, and metrics paths. func newAppMountedServer(t *testing.T) *httptest.Server { t.Helper() st, err := store.Open(filepath.Join(t.TempDir(), "test.db")) @@ -48,13 +43,7 @@ func newAppMountedServer(t *testing.T) *httptest.Server { svc := config.NewService(st, fakeReconciler{}, nil) handler := ui.Handler(svc, stats.NewRegistry(), fakeReconciler{}.Statuses, nil, "test", nil) - mux := http.NewServeMux() - mux.Handle("/ui/", handler) - mux.Handle("/ui", handler) - mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, "/ui/dashboard", http.StatusFound) - }) - srv := httptest.NewServer(mux) + srv := httptest.NewServer(handler) t.Cleanup(srv.Close) return srv } @@ -105,36 +94,26 @@ func TestRootRedirectsToDashboard(t *testing.T) { mustStatus(t, resp, http.StatusFound) loc := resp.Header.Get("Location") - if loc != "/ui/dashboard" { - t.Fatalf("Location = %q, want /ui/dashboard", loc) + if loc != "/dashboard" { + t.Fatalf("Location = %q, want /dashboard", loc) } } -// --- GET /ui and GET /ui/ -> 302 /ui/dashboard --- - -func TestUIRootPathsRedirectToDashboard(t *testing.T) { +func TestLegacyUIPathsAreRemoved(t *testing.T) { srv := newAppMountedServer(t) - client := &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - } - for _, path := range []string{"/ui", "/ui/"} { + for _, path := range []string{"/ui", "/ui/dashboard", "/ui/sources"} { t.Run(path, func(t *testing.T) { - resp, err := client.Get(srv.URL + path) + resp, err := http.Get(srv.URL + path) if err != nil { t.Fatal(err) } defer func() { _ = resp.Body.Close() }() - mustStatus(t, resp, http.StatusFound) - if loc := resp.Header.Get("Location"); loc != "/ui/dashboard" { - t.Fatalf("Location = %q, want /ui/dashboard", loc) - } + mustStatus(t, resp, http.StatusNotFound) }) } } -// --- Same-origin guard on POST /ui/* --- +// --- Same-origin guard on POST /* --- // samePOSTOriginTarget is a POST endpoint that never needs any prior setup // and, when the request reaches the handler (i.e. the guard let it @@ -142,7 +121,7 @@ func TestUIRootPathsRedirectToDashboard(t *testing.T) { // "not found" alert rather than erroring. That makes it a good same-origin // guard test target: any status other than 200 or 403 would mean the test // itself is broken, not the guard. -const samePOSTOriginTarget = "/ui/connectors/nope/delete" +const samePOSTOriginTarget = "/connectors/nope/delete" func TestSameOriginGuardBlocksCrossOriginPOST(t *testing.T) { srv := newAppMountedServer(t) @@ -219,7 +198,7 @@ func TestSameOriginGuardAllowsHeaderlessPOST(t *testing.T) { func TestSameOriginGuardDoesNotApplyToGET(t *testing.T) { srv := newAppMountedServer(t) - req, err := http.NewRequest(http.MethodGet, srv.URL+"/ui/dashboard", nil) + req, err := http.NewRequest(http.MethodGet, srv.URL+"/dashboard", nil) if err != nil { t.Fatal(err) } @@ -234,7 +213,7 @@ func TestSameOriginGuardDoesNotApplyToGET(t *testing.T) { func TestDashboardPageIsSelfContained(t *testing.T) { srv := newAppMountedServer(t) - resp, err := http.Get(srv.URL + "/ui/dashboard") + resp, err := http.Get(srv.URL + "/dashboard") if err != nil { t.Fatal(err) } @@ -265,10 +244,10 @@ func TestDashboardPageIsSelfContained(t *testing.T) { if strings.Contains(html, "app-sidebar") { t.Fatalf("dashboard page should not render a sidebar:\n%s", html) } - if !strings.Contains(html, `href="/ui/assets/favicon.svg`) { + if !strings.Contains(html, `href="/assets/favicon.svg`) { t.Fatalf("dashboard page does not reference the site favicon:\n%s", html) } - for _, nav := range []string{`href="https://openships.ai"`, `href="/ui/dashboard"`, `href="/ui/docs"`, `href="/ui/mcp"`} { + for _, nav := range []string{`href="https://openships.ai"`, `href="/dashboard"`, `href="/docs"`, `href="/mcp/info"`} { if !strings.Contains(html, nav) { t.Fatalf("dashboard page header does not link to %q:\n%s", nav, html) } @@ -284,7 +263,7 @@ func TestDashboardPageIsSelfContained(t *testing.T) { if strings.Contains(html, `class="nav-link brand-site" href="https://openships.ai"`) { t.Fatalf("openships.ai should use the same nav-link text styling as docs:\n%s", html) } - if product := strings.Index(html, `href="/ui/dashboard"`); product < brandStart || product > navStart { + if product := strings.Index(html, `href="/dashboard"`); product < brandStart || product > navStart { t.Fatalf("beacon should be the only left-side brand link:\n%s", html) } if ext := externalURLs(html); len(ext) != 1 || ext[0] != "https://openships.ai" { @@ -294,7 +273,7 @@ func TestDashboardPageIsSelfContained(t *testing.T) { func TestMCPReferencePageIsCompleteAndSelfContained(t *testing.T) { srv := newAppMountedServer(t) - resp, err := http.Get(srv.URL + "/ui/mcp") + resp, err := http.Get(srv.URL + "/mcp/info") if err != nil { t.Fatal(err) } @@ -312,7 +291,7 @@ func TestMCPReferencePageIsCompleteAndSelfContained(t *testing.T) { "claude mcp add --transport http beacon http://127.0.0.1:2112/mcp", "gemini mcp add beacon http://127.0.0.1:2112/mcp --transport http", `.vscode/mcp.json`, "get_health", - `"url": "http://127.0.0.1:2112/mcp"`, `href="/ui/mcp" class="nav-link menu-active"`, + `"url": "http://127.0.0.1:2112/mcp"`, `href="/mcp/info" class="nav-link menu-active"`, } { if !strings.Contains(html, want) { t.Fatalf("MCP page does not contain %q:\n%s", want, html) @@ -343,7 +322,7 @@ func TestAssetsServed(t *testing.T) { } for _, tc := range cases { t.Run(tc.path, func(t *testing.T) { - resp, err := http.Get(srv.URL + "/ui/assets/" + tc.path) + resp, err := http.Get(srv.URL + "/assets/" + tc.path) if err != nil { t.Fatal(err) } @@ -379,7 +358,7 @@ func TestOpenBridgeAssetsAreNotServed(t *testing.T) { for _, path := range []string{"openbridge.bundle.js", "palettes.css", "NotoSans.ttf"} { t.Run(path, func(t *testing.T) { - resp, err := http.Get(srv.URL + "/ui/assets/" + path) + resp, err := http.Get(srv.URL + "/assets/" + path) if err != nil { t.Fatal(err) } @@ -446,13 +425,13 @@ func TestDocsIndexRedirectsToFirstPage(t *testing.T) { return http.ErrUseLastResponse }, } - resp, err := client.Get(srv.URL + "/ui/docs") + resp, err := client.Get(srv.URL + "/docs") if err != nil { t.Fatal(err) } defer func() { _ = resp.Body.Close() }() mustStatus(t, resp, http.StatusFound) - want := "/ui/docs/" + wantDocPages[0].slug + want := "/docs/" + wantDocPages[0].slug if loc := resp.Header.Get("Location"); loc != want { t.Fatalf("Location = %q, want %q", loc, want) } @@ -460,7 +439,7 @@ func TestDocsIndexRedirectsToFirstPage(t *testing.T) { func TestDocPageServesKnownSlug(t *testing.T) { srv := newAppMountedServer(t) - resp, err := http.Get(srv.URL + "/ui/docs/getting-started") + resp, err := http.Get(srv.URL + "/docs/getting-started") if err != nil { t.Fatal(err) } @@ -487,7 +466,7 @@ func TestDocPageServesKnownSlug(t *testing.T) { func TestDocPageUnknownSlugIs404(t *testing.T) { srv := newAppMountedServer(t) - resp, err := http.Get(srv.URL + "/ui/docs/does-not-exist") + resp, err := http.Get(srv.URL + "/docs/does-not-exist") if err != nil { t.Fatal(err) } @@ -496,7 +475,7 @@ func TestDocPageUnknownSlugIs404(t *testing.T) { } // TestDocsSidebarListsAllPages checks every shipped page's sidebar entry — -// href AND title, as one `{title}` anchor, +// href AND title, as one `{title}` anchor, // so a page whose title went blank or got swapped fails here even though // its slug link would still be present — appears on a single rendered docs // page (the sidebar is identical across every page; see docs.html), and @@ -504,7 +483,7 @@ func TestDocPageUnknownSlugIs404(t *testing.T) { // (neither list a stray extra nor missing one). func TestDocsSidebarListsAllPages(t *testing.T) { srv := newAppMountedServer(t) - resp, err := http.Get(srv.URL + "/ui/docs/getting-started") + resp, err := http.Get(srv.URL + "/docs/getting-started") if err != nil { t.Fatal(err) } @@ -516,14 +495,14 @@ func TestDocsSidebarListsAllPages(t *testing.T) { } html := string(body) - if got := strings.Count(html, `href="/ui/docs/`); got != len(wantDocPages) { - t.Fatalf("sidebar has %d /ui/docs/ links, want %d (%v)", got, len(wantDocPages), wantDocPages) + if got := strings.Count(html, `href="/docs/`); got != len(wantDocPages) { + t.Fatalf("sidebar has %d /docs/ links, want %d (%v)", got, len(wantDocPages), wantDocPages) } // One regexp per page: its anchor must carry the right href AND close // with the right link text (docs.html renders the active page's anchor // with an extra class attribute between href and >, hence the [^>]*). for _, p := range wantDocPages { - anchor := regexp.MustCompile(`]*>` + regexp.QuoteMeta(p.title) + ``) + anchor := regexp.MustCompile(`]*>` + regexp.QuoteMeta(p.title) + ``) if !anchor.MatchString(html) { t.Fatalf("sidebar missing anchor for slug %q with title %q:\n%s", p.slug, p.title, html) } @@ -539,7 +518,7 @@ func TestDocsPagesAreSelfContained(t *testing.T) { srv := newAppMountedServer(t) for _, p := range wantDocPages { t.Run(p.slug, func(t *testing.T) { - resp, err := http.Get(srv.URL + "/ui/docs/" + p.slug) + resp, err := http.Get(srv.URL + "/docs/" + p.slug) if err != nil { t.Fatal(err) } @@ -571,7 +550,7 @@ func TestDocsPagesContainNoInlineStyling(t *testing.T) { styleTag := regexp.MustCompile(`(?i)]`) for _, p := range wantDocPages { t.Run(p.slug, func(t *testing.T) { - resp, err := http.Get(srv.URL + "/ui/docs/" + p.slug) + resp, err := http.Get(srv.URL + "/docs/" + p.slug) if err != nil { t.Fatal(err) } @@ -592,10 +571,10 @@ func TestDocsPagesContainNoInlineStyling(t *testing.T) { } // TestDocsNavItemPresent checks layout.html's main nav (shared by every -// full page, not just docs) has the docs entry pointing at /ui/docs. +// full page, not just docs) has the docs entry pointing at /docs. func TestDocsNavItemPresent(t *testing.T) { srv := newAppMountedServer(t) - resp, err := http.Get(srv.URL + "/ui/dashboard") + resp, err := http.Get(srv.URL + "/dashboard") if err != nil { t.Fatal(err) } @@ -605,8 +584,8 @@ func TestDocsNavItemPresent(t *testing.T) { t.Fatal(err) } html := string(body) - if !strings.Contains(html, `href="/ui/docs"`) { - t.Fatalf("main nav does not link to /ui/docs:\n%s", html) + if !strings.Contains(html, `href="/docs"`) { + t.Fatalf("main nav does not link to /docs:\n%s", html) } if !strings.Contains(html, ">docs<") { t.Fatalf("main nav does not label the docs link \"docs\":\n%s", html) diff --git a/justfile b/justfile index 6a40276..bd08d17 100644 --- a/justfile +++ b/justfile @@ -6,6 +6,7 @@ golangci_lint_version := "v2.12.0" secure_go_toolchain := "go1.25.12" govulncheck_version := "v1.5.0" gosec_version := "v2.27.1" +air_version := "v1.64.5" # list available recipes default: @@ -13,6 +14,12 @@ default: # ensure dev tools are installed at the right versions setup: + @if command -v air >/dev/null && go version -m "$(command -v air)" 2>&1 | grep -q "github.com/air-verse/air[[:space:]]*{{air_version}}"; then \ + echo "air {{air_version}} already installed"; \ + else \ + echo "Installing air {{air_version}}..."; \ + go install github.com/air-verse/air@{{air_version}}; \ + fi @if command -v golangci-lint >/dev/null && golangci-lint --version 2>&1 | grep -q "{{trim_start_match(golangci_lint_version, "v")}}"; then \ echo "golangci-lint {{golangci_lint_version}} already installed"; \ else \ @@ -60,6 +67,11 @@ test-race: run *args: go run {{cmd}} {{args}} +# run with automatic reloads when Go or embedded UI files change +dev *args: + @if ! command -v air >/dev/null; then echo "air is required; run 'just setup' first"; exit 1; fi + air -- {{args}} + # format all Go source files fmt: gofmt -w . diff --git a/tests/browser/ui.spec.ts b/tests/browser/ui.spec.ts index b0d1cd8..9319f0a 100644 --- a/tests/browser/ui.spec.ts +++ b/tests/browser/ui.spec.ts @@ -1,4 +1,5 @@ import { expect, test } from '@playwright/test'; +import { readFile } from 'node:fs/promises'; test('operator can create and visually trace a status-colored pipeline', async ({ page }) => { const sourceID = `browser-test-${Date.now()}`; @@ -10,13 +11,13 @@ test('operator can create and visually trace a status-colored pipeline', async ( await page.goto('/'); - await expect(page).toHaveURL(/\/ui\/dashboard$/); + await expect(page).toHaveURL(/\/dashboard$/); await expect(page).toHaveTitle('Home - beacon'); await expect(page.getByRole('heading', { name: 'Data flow' })).toBeVisible(); await expect(page.getByText('Add your first source')).toBeVisible(); await page.getByRole('link', { name: 'Add a source', exact: true }).click(); - await expect(page).toHaveURL(/\/ui\/sources\/new$/); + await expect(page).toHaveURL(/\/sources\/new$/); await expect(page).toHaveTitle('Sources - beacon'); await expect(page.getByRole('heading', { name: 'Add source' })).toBeVisible(); @@ -25,7 +26,7 @@ test('operator can create and visually trace a status-colored pipeline', async ( await page.locator('#src-interface').fill('can0'); await page.getByRole('button', { name: 'Save' }).click(); - await expect(page).toHaveURL(/\/ui\/dashboard$/); + await expect(page).toHaveURL(/\/dashboard$/); await expect(page.getByRole('status')).toContainText(`Source "${sourceID}" created`); const sourceLink = page.getByRole('link', { name: sourceName }); await expect(sourceLink).toBeVisible(); @@ -40,7 +41,7 @@ test('operator can create and visually trace a status-colored pipeline', async ( await expect(page.locator('#sink-address')).toBeVisible(); await page.locator('#sink-address').fill('127.0.0.1:0'); await page.getByRole('button', { name: 'Save' }).click(); - await expect(page).toHaveURL(/\/ui\/dashboard$/); + await expect(page).toHaveURL(/\/dashboard$/); await page.getByRole('link', { name: 'New connector', exact: true }).click(); await page.locator('#conn-id').fill(connectorID); @@ -48,7 +49,7 @@ test('operator can create and visually trace a status-colored pipeline', async ( await page.locator('#conn-source').selectOption(sourceID); await page.locator('#conn-sink').selectOption(sinkID); await page.getByRole('button', { name: 'Save' }).click(); - await expect(page).toHaveURL(/\/ui\/dashboard$/); + await expect(page).toHaveURL(/\/dashboard$/); const connectorNode = page.locator(`[data-connector-id="${connectorID}"]`); await expect(connectorNode).toHaveClass(/component-status-surface state-(up|degraded|error|restarting|disabled)/); @@ -63,14 +64,14 @@ test('operator can create and visually trace a status-colored pipeline', async ( expect(await routeLink.evaluate((link) => getComputedStyle(link, '::before').backgroundColor)).not.toBe('rgba(0, 0, 0, 0)'); } - await page.goto('/ui/connectors'); + await page.goto('/connectors'); const connectorRow = page.getByRole('link', { name: connectorName }).locator('xpath=ancestor::tr'); await expect(connectorRow).toHaveClass(/component-status-row state-(up|degraded|error|restarting|disabled)/); expect(await connectorRow.evaluate((row) => getComputedStyle(row).backgroundColor)).not.toBe('rgba(0, 0, 0, 0)'); }); test('connector CEL editor provides autocomplete and live diagnostics', async ({ page }) => { - await page.goto('/ui/connectors/new'); + await page.goto('/connectors/new'); const filters = page.locator('#conn-filters'); const completions = page.getByRole('listbox', { name: 'CEL completions' }); @@ -112,16 +113,236 @@ test('connector CEL editor provides autocomplete and live diagnostics', async ({ await expect(page.locator('#filter-feedback')).toContainText('filters OK'); }); +test('source and sink overviews provide controllable JSON and CAN stream inspectors', async ({ page }) => { + await page.context().grantPermissions(['clipboard-read', 'clipboard-write'], { + origin: 'http://127.0.0.1:32112', + }); + const suffix = Date.now(); + const sourceID = `stream-source-${suffix}`; + const sinkID = `stream-sink-${suffix}`; + const envelope = { + payload: { + info: { + timestamp: '2026-06-29T10:10:17.530566931-04:00', + receivedAt: '2026-07-25T08:41:35.729702-04:00', + adapterId: 'can0', + networkId: 'can0', + direction: 'received', + priority: 5, + pgn: 130314, + sourceId: 6, + targetId: null, + }, + instance: 0, + source: 0, + pressure: 1020690, + }, + metadata: { + observed_at: '2026-07-25T12:41:35.729721Z', + pgn_name: 'Actual Pressure', + decode: { status: 'decoded', complete: true }, + }, + raw: '/wAAEg==', + }; + const envelopeDocument = JSON.stringify(envelope).replace( + '"pressure":1020690', + '"pressure":1020690,"serialNumber":18446744073709551615', + ); + let streamBody = `data: ${envelopeDocument}\n\n`; + let streamRequests = 0; + let requestedFilter: string | null = null; + const requestedFilters: Array = []; + + await page.goto('/sources/new'); + await page.locator('#src-id').fill(sourceID); + await page.locator('#src-name').fill('Stream source'); + await page.locator('#src-interface').fill('can0'); + await page.getByRole('button', { name: 'Save' }).click(); + await expect(page).toHaveURL(/\/dashboard$/); + + await page.route('**/ui/streams/sources/**', async (route) => { + streamRequests += 1; + requestedFilter = new URL(route.request().url()).searchParams.get('filter'); + requestedFilters.push(requestedFilter); + await route.fulfill({ + status: 200, + contentType: 'text/event-stream', + headers: { 'Cache-Control': 'no-store' }, + body: streamBody, + }); + }); + await page.goto(`/sources/${sourceID}/`); + + const sourcePanel = page.locator('[data-stream-panel]'); + const streamFilter = sourcePanel.getByRole('textbox', { name: 'CEL stream filter' }); + const startButton = sourcePanel.getByRole('button', { name: 'Start', exact: true }); + const stopButton = sourcePanel.getByRole('button', { name: 'Stop', exact: true }); + const jsonViewButton = sourcePanel.getByRole('button', { name: 'JSONL', exact: true }); + const canViewButton = sourcePanel.getByRole('button', { name: 'CAN bytes', exact: true }); + await expect(sourcePanel.getByRole('heading', { name: 'Stream contents' })).toBeVisible(); + await expect(sourcePanel.getByText('Stopped · 0 captured')).toBeVisible(); + await expect(jsonViewButton).toHaveAttribute('aria-pressed', 'true'); + await expect(jsonViewButton).toHaveClass(/btn-primary/); + await expect(canViewButton).toHaveAttribute('aria-pressed', 'false'); + await expect(canViewButton).toHaveClass(/btn-ghost/); + await expect(startButton).toBeVisible(); + await expect(stopButton).toBeHidden(); + expect(await streamFilter.evaluate((input) => ( + input.closest('.stream-filter-control')?.nextElementSibling?.hasAttribute('data-stream-start') + ))).toBe(true); + const filterBounds = await streamFilter.boundingBox(); + const startBounds = await startButton.boundingBox(); + expect(filterBounds).not.toBeNull(); + expect(startBounds).not.toBeNull(); + expect(filterBounds!.x + filterBounds!.width).toBeLessThanOrEqual(startBounds!.x); + await streamFilter.fill('msg.pgn == 130314'); + await startButton.click(); + await expect.poll(() => requestedFilter).toBe('msg.pgn == 130314'); + await expect(startButton).toBeHidden(); + await expect(stopButton).toBeVisible(); + await expect(streamFilter).toBeEnabled(); + await expect(sourcePanel.locator('.stream-message-content')).toContainText('"pressure":1020690'); + await expect(sourcePanel.locator('.stream-message-content')).toContainText( + '"serialNumber":18446744073709551615', + ); + await streamFilter.fill('msg.source == 6'); + await expect.poll(() => requestedFilters.at(-1)).toBe('msg.source == 6'); + await expect(sourcePanel.locator('[data-stream-status]')).toContainText('2 captured'); + const requestsBeforeInvalidLiveFilter = streamRequests; + await streamFilter.fill('msg.pgn == @'); + await expect(sourcePanel.locator('[data-stream-filter-feedback]')).toContainText( + 'current stream filter is unchanged', + ); + expect(streamRequests).toBe(requestsBeforeInvalidLiveFilter); + await expect(stopButton).toBeVisible(); + await streamFilter.fill('msg.source == 6'); + await expect(streamFilter).toHaveAttribute('aria-invalid', 'false'); + await stopButton.click(); + await expect(sourcePanel.locator('[data-stream-status]')).toContainText('Stopped'); + await expect(startButton).toBeVisible(); + await expect(stopButton).toBeHidden(); + const requestsBeforeInvalidFilter = streamRequests; + await streamFilter.fill('msg.pgn == @'); + await startButton.click(); + await expect(streamFilter).toHaveAttribute('aria-invalid', 'true'); + await expect(sourcePanel.locator('[data-stream-filter-feedback]')).toContainText('Invalid CEL'); + expect(streamRequests).toBe(requestsBeforeInvalidFilter); + await streamFilter.fill('msg.pgn == 130314'); + + const celTarget = sourcePanel.locator( + '[data-cel-path="msg.payload.pressure"][data-cel-expression="msg.payload.pressure == 1020690"]', + ).first(); + await expect(celTarget).toBeVisible(); + await sourcePanel.locator('.stream-message-content').first().evaluate((content) => content.click()); + await expect(sourcePanel.locator('[data-stream-cel-expression]')).toHaveText('has(msg.payload)'); + await celTarget.click(); + const celInspector = sourcePanel.locator('[data-stream-cel-inspector]'); + await expect(celInspector).toBeVisible(); + await expect(celInspector.locator('[data-stream-cel-path]')).toHaveText('msg.payload.pressure'); + await expect(celInspector.locator('[data-stream-cel-expression]')).toHaveText( + 'msg.payload.pressure == 1020690', + ); + await celInspector.getByRole('button', { name: 'Use filter', exact: true }).click(); + await expect(streamFilter).toHaveValue('msg.payload.pressure == 1020690'); + await streamFilter.fill('msg.pgn == 130314'); + + const visibleJSONLines = await sourcePanel.locator('.stream-message-content').allTextContents(); + await sourcePanel.getByRole('button', { name: 'Copy stream', exact: true }).click(); + const copiedJSONL = await page.evaluate(() => navigator.clipboard.readText()); + expect(copiedJSONL.trimEnd().split('\n')).toEqual(visibleJSONLines.slice().reverse()); + expect(copiedJSONL).toContain('"serialNumber":18446744073709551615'); + + await canViewButton.click(); + await expect(canViewButton).toHaveAttribute('aria-pressed', 'true'); + await expect(canViewButton).toHaveClass(/btn-primary/); + await expect(jsonViewButton).toHaveAttribute('aria-pressed', 'false'); + await expect(jsonViewButton).toHaveClass(/btn-ghost/); + await expect(sourcePanel.locator('.stream-message-content').first()).toContainText('FF 00 00 12'); + expect(await sourcePanel.locator('.stream-message-content').first().textContent()).not.toContain('\n'); + await sourcePanel.getByRole('button', { name: 'Copy stream', exact: true }).click(); + const copiedCAN = await page.evaluate(() => navigator.clipboard.readText()); + expect(copiedCAN.trimEnd().split('\n')).toEqual( + Array.from({ length: visibleJSONLines.length }, () => 'FF 00 00 12'), + ); + + const jsonDownloadPromise = page.waitForEvent('download'); + await sourcePanel.getByRole('button', { name: 'Export JSONL', exact: true }).click(); + const jsonDownload = await jsonDownloadPromise; + expect(jsonDownload.suggestedFilename()).toContain(`${sourceID}-`); + expect(jsonDownload.suggestedFilename()).toMatch(/\.n2k\.jsonl$/); + const jsonPath = await jsonDownload.path(); + expect(jsonPath).not.toBeNull(); + expect(await readFile(jsonPath!, 'utf8')).toContain('"pressure":1020690'); + expect(await readFile(jsonPath!, 'utf8')).toContain('"serialNumber":18446744073709551615'); + + const canDownloadPromise = page.waitForEvent('download'); + await sourcePanel.getByRole('button', { name: 'Export CAN', exact: true }).click(); + const canDownload = await canDownloadPromise; + expect(canDownload.suggestedFilename()).toMatch(/\.can\.hex$/); + const canPath = await canDownload.path(); + expect(canPath).not.toBeNull(); + expect(await readFile(canPath!, 'utf8')).toBe( + Array.from({ length: visibleJSONLines.length }, () => 'FF000012').join('\n') + '\n', + ); + + await sourcePanel.getByRole('button', { name: 'Clear', exact: true }).click(); + await expect(sourcePanel.locator('[data-stream-status]')).toContainText('0 captured'); + streamBody = Array.from({ length: 250 }, (_, index) => ( + `data: ${envelopeDocument.replace('"pressure":1020690', `"pressure":${1020690 + index}`)}\n\n` + )).join(''); + await jsonViewButton.click(); + await expect(jsonViewButton).toHaveAttribute('aria-pressed', 'true'); + await expect(jsonViewButton).toHaveClass(/btn-primary/); + await startButton.click(); + await expect(sourcePanel.locator('[data-stream-status]')).toContainText('250 captured'); + await expect(sourcePanel.locator('[data-stream-status]')).toContainText('latest 200 shown'); + await stopButton.click(); + + const capturedMessages = sourcePanel.locator('.stream-message'); + await expect(capturedMessages).toHaveCount(200); + await expect(capturedMessages.first().locator('.stream-message-content')).toBeVisible(); + await expect(sourcePanel.locator('[data-stream-list]')).toHaveCSS('background-color', 'rgb(255, 255, 255)'); + await expect(capturedMessages.first().locator('.stream-message-content')).toHaveCSS('color', 'rgb(17, 24, 39)'); + await expect(capturedMessages.nth(1)).toHaveCSS('border-top-style', 'none'); + const displayedLines = await capturedMessages.locator('.stream-message-content').allTextContents(); + expect(displayedLines).toHaveLength(200); + expect(displayedLines.every((line) => !line.includes('\n') && line.startsWith('{') && line.endsWith('}'))).toBe(true); + await sourcePanel.getByRole('button', { name: 'Copy stream', exact: true }).click(); + const copiedRetainedJSONL = await page.evaluate(() => navigator.clipboard.readText()); + expect(copiedRetainedJSONL.trimEnd().split('\n')).toHaveLength(200); + expect(await capturedMessages.first().evaluate((message) => message.getBoundingClientRect().height)).toBeLessThan(34); + expect(await sourcePanel.locator('[data-stream-list]').evaluate((list) => list.scrollHeight)).toBeGreaterThan( + await sourcePanel.locator('[data-stream-list]').evaluate((list) => list.clientHeight), + ); + + await page.goto('/sinks/new'); + await page.locator('#sink-id').fill(sinkID); + await page.locator('#sink-name').fill('Stream sink'); + await page.locator('#sink-type').selectOption('tcp'); + await page.locator('#sink-address').fill('127.0.0.1:0'); + await page.getByRole('button', { name: 'Save' }).click(); + await expect(page).toHaveURL(/\/dashboard$/); + await page.goto(`/sinks/${sinkID}/`); + + const sinkPanel = page.locator('[data-stream-panel]'); + await expect(sinkPanel).toHaveAttribute('data-stream-url', `/ui/streams/sinks/${sinkID}`); + await expect(sinkPanel.getByRole('button', { name: 'Start', exact: true })).toBeVisible(); + await expect(sinkPanel.getByRole('button', { name: 'Stop', exact: true })).toBeHidden(); + await expect(sinkPanel.getByRole('button', { name: 'Export JSONL', exact: true })).toBeDisabled(); + await expect(sinkPanel.getByRole('button', { name: 'Export CAN', exact: true })).toBeDisabled(); + await expect(sinkPanel.getByRole('button', { name: 'Copy stream', exact: true })).toBeDisabled(); +}); + test('MCP reference is available from the header and stays local', async ({ page }) => { const requestedOrigins = new Set(); page.on('request', (request) => requestedOrigins.add(new URL(request.url()).origin)); - await page.goto('/ui/dashboard'); + await page.goto('/dashboard'); const mcpLink = page.getByRole('navigation', { name: 'Primary' }).getByRole('link', { name: 'mcp', exact: true }); await expect(mcpLink).toBeVisible(); await mcpLink.click(); - await expect(page).toHaveURL(/\/ui\/mcp$/); + await expect(page).toHaveURL(/\/mcp\/info$/); await expect(page).toHaveTitle('MCP - beacon'); await expect(page.getByRole('heading', { name: 'Model Context Protocol' })).toBeVisible(); await expect(page.getByRole('heading', { name: 'Install Beacon MCP' })).toBeVisible(); @@ -130,9 +351,9 @@ test('MCP reference is available from the header and stays local', async ({ page await expect(page.getByText('gemini mcp add beacon http://127.0.0.1:2112/mcp --transport http', { exact: true })).toBeVisible(); await expect(page.getByText('.vscode/mcp.json', { exact: true })).toBeVisible(); await expect(page.getByText('/mcp', { exact: true }).first()).toBeVisible(); - await expect(page.getByRole('table').getByText('get_delivery_statistics', { exact: true })).toBeVisible(); + await expect(page.getByRole('table').getByText('get_delivery_metrics', { exact: true })).toBeVisible(); await expect(page.getByRole('table').getByText('get_source_metrics', { exact: true })).toBeVisible(); - await expect(page.getByRole('table').getByText('commit_source_traffic_baseline', { exact: true })).toBeVisible(); + await expect(page.getByRole('table').getByText('commit_source_traffic_baseline', { exact: true })).toHaveCount(0); await expect(page.getByText('offline ready', { exact: true })).toBeVisible(); expect([...requestedOrigins]).toEqual(['http://127.0.0.1:32112']); });