From c74f91b38155cc102dea15f3227700f6e284d68b Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 18:56:24 -0400 Subject: [PATCH 01/23] fix(routes): keep current prefixes and pin retained evidence paths --- db/queries/queries.sql | 3 +- db/route_evidence.go | 39 ++++++++++- db/route_evidence_integration_test.go | 68 +++++++++++++++++++- db/sqlc/querier.go | 2 +- db/sqlc/queries.sql.go | 3 +- docs/docs.go | 18 +++++- docs/swagger.json | 18 +++++- docs/swagger.yaml | 27 ++++++-- internal/api/handlers/route_evidence.go | 22 +++++-- internal/api/handlers/route_evidence_test.go | 25 +++++++ internal/api/route_evidence.go | 47 ++++++++++---- internal/api/route_evidence_test.go | 20 ++++++ 12 files changed, 261 insertions(+), 31 deletions(-) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index ca7584c8..fd38c206 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1275,10 +1275,11 @@ ORDER BY t.last_heard_at DESC; -- name: UpsertKnownRoute :exec -- Route identity is path_key, an md5 of node_ids computed by the caller. --- On conflict, observation_count and last_seen are bumped. +-- Keep the latest processed representation without changing the node-chain identity. INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (iata, path_key) DO UPDATE SET + hash_prefix = EXCLUDED.hash_prefix, last_seen = NOW(), observation_count = known_routes.observation_count + 1; diff --git a/db/route_evidence.go b/db/route_evidence.go index e7017ea5..f31f50c0 100644 --- a/db/route_evidence.go +++ b/db/route_evidence.go @@ -39,6 +39,16 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap if c.IATA != iata || c.PathKey != key || !c.Since.Equal(query.Since) || !c.Until.Equal(query.Until) || c.ID <= 0 || c.HeardAt.Before(query.Since) || !c.HeardAt.Before(query.Until) { return nil, api.ErrRouteEvidenceInput } + if c.HashSize != 0 || c.PathBytes != "" { + if (query.HashSize != 0 || query.PathBytes != "") && (query.HashSize != c.HashSize || query.PathBytes != c.PathBytes) { + return nil, api.ErrRouteEvidenceInput + } + query.HashSize, query.PathBytes = c.HashSize, c.PathBytes + } + } + pinned := query.HashSize != 0 || query.PathBytes != "" + if pinned && !api.ValidRouteEvidencePath(query.HashSize, query.PathBytes) { + return nil, api.ErrRouteEvidenceInput } keyBytes, _ := hex.DecodeString(key) row, err := s.q.GetRouteEvidenceRoute(ctx, sqlc.GetRouteEvidenceRouteParams{Iata: iata, PathKey: keyBytes}) @@ -49,10 +59,35 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap if err != nil { return nil, err } + width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) + if pinned { + selected, _ := hex.DecodeString(query.PathBytes) + hops := len(selected) / int(query.HashSize) + if hops != len(row.NodeIds) || hops != int(row.HopCount) { + return nil, api.ErrRouteEvidenceInput + } + prefixes := make([][]byte, hops) + for i, id := range row.NodeIds { + prefixes[i] = selected[i*int(query.HashSize) : (i+1)*int(query.HashSize)] + // A changed representation must still belong to this node chain. + // Reuse the already-loaded keys; never widen the evidence query. + if !valid || width != query.HashSize || !bytes.Equal(path, selected) { + node := nodes[id] + if node == nil { + return nil, api.ErrRouteEvidenceInput + } + pubkey, err := hex.DecodeString(node.PublicKey) + if err != nil || !bytes.HasPrefix(pubkey, prefixes[i]) { + return nil, api.ErrRouteEvidenceInput + } + } + } + row.HashPrefix = prefixes + width, path, valid = query.HashSize, selected, true + } route := toKnownRoutes([]knownRouteRow{{ID: row.ID, NodeIds: row.NodeIds, HashPrefix: row.HashPrefix, Iata: row.Iata, HopCount: row.HopCount, FirstSeen: row.FirstSeen, LastSeen: row.LastSeen, ObservationCount: row.ObservationCount}}, nodes)[0] route.PathKey = hex.EncodeToString(row.PathKey) out := &api.RouteEvidence{Items: []api.RouteObservation{}, Route: route, WindowStart: query.Since.UnixMilli(), WindowEnd: query.Until.UnixMilli(), GeneratedAt: time.Now().UnixMilli(), MatchType: "saved_path_prefixes"} - width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) if !valid { return out, nil } @@ -86,7 +121,7 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap } if out.HasMore { last := rows[len(rows)-1] - cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID}).String() + cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID, HashSize: width, PathBytes: out.PathBytes}).String() out.NextPageCursor = &cursor } return out, nil diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index f4a3e177..bef6989d 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -47,7 +47,7 @@ func TestRouteEvidencePostgres(t *testing.T) { _, err = tx.Exec(ctx, `CREATE TABLE iata_codes (iata char(3) PRIMARY KEY); INSERT INTO iata_codes VALUES ('YOW'),('YVR'); ALTER TABLE nodes ADD COLUMN latitude double precision, ADD COLUMN longitude double precision;`+table+` INSERT INTO observers(id,display_name) VALUES ('00000000-0000-0000-0000-000000000001','One'),('00000000-0000-0000-0000-000000000002','Two'); - INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa','A'),('00000000-0000-0000-0000-000000000002','\xbb','B'); + INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa0102','A'),('00000000-0000-0000-0000-000000000002','\xbb0304','B'); INSERT INTO packets(packet_hash) SELECT int4send(i) FROM generate_series(1,10) i;`) if err != nil { t.Fatal(err) @@ -94,6 +94,34 @@ func TestRouteEvidencePostgres(t *testing.T) { if err != nil { t.Fatal(err) } + if query.Cursor.HashSize != 1 || query.Cursor.PathBytes != "aabb" { + t.Fatalf("cursor did not pin the original bytes: %+v", query.Cursor) + } + // A new representation arrives while page one is open. New reads follow it, + // but pagination and copied links retain the exact original representation. + _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash) VALUES (int4send(12)),(int4send(13)),(int4send(14)); + INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) + SELECT i,int4send(i),'00000000-0000-0000-0000-000000000001','YOW',$1,2,2,'\xaa01bb03',4 FROM generate_series(12,14) i`, at) + if err != nil { + t.Fatal(err) + } + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa, 1}, {0xbb, 3}}, "YOW", 2); err != nil { + t.Fatal(err) + } + freshQuery := api.RouteEvidenceQuery{Since: since, Until: until, Limit: 2} + fresh, err := store.GetRouteEvidence(ctx, "YOW", key, freshQuery) + if err != nil || fresh.HashSize != 2 || fresh.PathBytes != "aa01bb03" || len(fresh.Items) != 2 || fresh.Items[0].ID != 14 || fresh.Route.Hops[0].HashBytes != "aa01" { + t.Fatalf("new representation lost: %+v %v", fresh, err) + } + if fresh.Route.ID != first.Route.ID || fresh.Route.PathKey != first.Route.PathKey || fresh.Route.FirstSeen != first.Route.FirstSeen || fresh.Route.ObservationCount != first.Route.ObservationCount+1 { + t.Fatalf("route identity/counters changed: %+v", fresh.Route) + } + shared := freshQuery + shared.HashSize, shared.PathBytes = first.HashSize, first.PathBytes + copyPage, err := store.GetRouteEvidence(ctx, "YOW", key, shared) + if err != nil || len(copyPage.Items) != 2 || copyPage.Items[0].ID != 3 || copyPage.Route.Hops[0].HashBytes != "aa" { + t.Fatalf("shared representation changed: %+v %v", copyPage, err) + } second, err := store.GetRouteEvidence(ctx, "YOW", key, query) if err != nil { t.Fatal(err) @@ -104,6 +132,44 @@ func TestRouteEvidencePostgres(t *testing.T) { if _, err = json.Marshal(second); err != nil { t.Fatalf("non-finite JSON: %v", err) } + for _, prefixes := range [][][]byte{{{0xaa, 1, 2}, {0xbb, 3, 4}}, {{0xaa, 1, 2}, {0xbb, 3, 4}}} { + if err := store.UpsertKnownRoute(ctx, ids, prefixes, "YOW", 2); err != nil { + t.Fatal(err) + } + } + thirdWidth, err := store.GetRouteEvidence(ctx, "YOW", key, freshQuery) + if err != nil || thirdWidth.HashSize != 3 || thirdWidth.PathBytes != "aa0102bb0304" || len(thirdWidth.Items) != 0 || thirdWidth.Route.ID != first.Route.ID || thirdWidth.Route.ObservationCount != first.Route.ObservationCount+3 { + t.Fatalf("third width or repeated upsert: %+v %v", thirdWidth, err) + } + // Missing identities or mismatched bytes cannot make a pinned query search + // a different chain. Prefix collisions likewise never widen the SQL match. + wrong := shared + wrong.PathBytes = "aacc" + if _, err := store.GetRouteEvidence(ctx, "YOW", key, wrong); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatalf("unrelated path accepted: %v", err) + } + wrong.PathBytes = "aabbcc" + if _, err := store.GetRouteEvidence(ctx, "YOW", key, wrong); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatalf("wrong hop count accepted: %v", err) + } + _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000003','\xaa9988','Collision'); + DELETE FROM nodes WHERE id='00000000-0000-0000-0000-000000000001'`) + if err != nil { + t.Fatal(err) + } + if _, err := store.GetRouteEvidence(ctx, "YOW", key, shared); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatalf("missing node substituted by colliding node: %v", err) + } + if _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa0102','A')`); err != nil { + t.Fatal(err) + } + copyPage, err = store.GetRouteEvidence(ctx, "YOW", key, shared) + if err != nil || len(copyPage.Items) != 2 || copyPage.Items[0].ID != 3 { + t.Fatalf("collision broadened evidence: %+v %v", copyPage, err) + } + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } query.Cursor.IATA = "YVR" if _, err = store.GetRouteEvidence(ctx, "YOW", key, query); !errors.Is(err, api.ErrRouteEvidenceInput) { t.Fatal("cross-route cursor accepted") diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index ef53cb36..ae84cc0a 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -306,7 +306,7 @@ type Querier interface { // ROUTES // ============================================================ // Route identity is path_key, an md5 of node_ids computed by the caller. - // On conflict, observation_count and last_seen are bumped. + // Keep the latest processed representation without changing the node-chain identity. UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error // ============================================================ // NODES diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 9ea06a5b..be72f7a7 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -4961,6 +4961,7 @@ const upsertKnownRoute = `-- name: UpsertKnownRoute :exec INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (iata, path_key) DO UPDATE SET + hash_prefix = EXCLUDED.hash_prefix, last_seen = NOW(), observation_count = known_routes.observation_count + 1 ` @@ -4977,7 +4978,7 @@ type UpsertKnownRouteParams struct { // ROUTES // ============================================================ // Route identity is path_key, an md5 of node_ids computed by the caller. -// On conflict, observation_count and last_seen are bumped. +// Keep the latest processed representation without changing the node-chain identity. func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error { _, err := q.db.Exec(ctx, upsertKnownRoute, arg.PathKey, diff --git a/docs/docs.go b/docs/docs.go index df1ccc9e..2c79a821 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2133,7 +2133,7 @@ const docTemplate = `{ }, "/routes/{iata}/{pathKey}/observations": { "get": { - "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.", "produces": [ "application/json" ], @@ -2174,9 +2174,23 @@ const docTemplate = `{ "name": "until", "in": "query" }, + { + "maximum": 3, + "minimum": 1, + "type": "integer", + "description": "Exact hash width (1–3); provide with pathBytes to pin a response's representation", + "name": "hashSize", + "in": "query" + }, + { + "type": "string", + "description": "Complete lower-case hex path from the response; provide with hashSize", + "name": "pathBytes", + "in": "query" + }, { "type": "string", - "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "description": "Opaque precise cursor from nextPageCursor; route, window and path are pinned", "name": "pageCursor", "in": "query" }, diff --git a/docs/swagger.json b/docs/swagger.json index 183611af..f16f2166 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2131,7 +2131,7 @@ }, "/routes/{iata}/{pathKey}/observations": { "get": { - "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.", "produces": [ "application/json" ], @@ -2172,9 +2172,23 @@ "name": "until", "in": "query" }, + { + "maximum": 3, + "minimum": 1, + "type": "integer", + "description": "Exact hash width (1–3); provide with pathBytes to pin a response's representation", + "name": "hashSize", + "in": "query" + }, + { + "type": "string", + "description": "Complete lower-case hex path from the response; provide with hashSize", + "name": "pathBytes", + "in": "query" + }, { "type": "string", - "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "description": "Opaque precise cursor from nextPageCursor; route, window and path are pinned", "name": "pageCursor", "in": "query" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index ca79d350..9b029830 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2868,11 +2868,13 @@ paths: - Routes /routes/{iata}/{pathKey}/observations: get: - description: Matches the saved IATA, complete path bytes and hash width. Excludes - TRACE and unclassified reports; other widths and search-result subsegments - are not included. Byte matches do not prove hop identities or delivery. The - route and its historical counter can outlive raw evidence. Cursors retain - full timestamp precision and pin route/window scope. + description: Matches one complete path representation in the saved IATA. Defaults + to the latest processed prefixes; hashSize/pathBytes pin a prior response. + Excludes TRACE and unclassified reports; widths are never combined. Pinned + prefixes must match the saved node chain or its current metadata. Byte matches + do not prove hop identities or delivery. The route counter can outlive raw + evidence. New cursors pin path, route, window and full timestamp precision; + legacy unpinned cursors remain supported. parameters: - description: Three-character IATA code in: path @@ -2898,7 +2900,20 @@ paths: in: query name: until type: integer - - description: Opaque precise cursor from nextPageCursor; window is pinned + - description: Exact hash width (1–3); provide with pathBytes to pin a response's + representation + in: query + maximum: 3 + minimum: 1 + name: hashSize + type: integer + - description: Complete lower-case hex path from the response; provide with + hashSize + in: query + name: pathBytes + type: string + - description: Opaque precise cursor from nextPageCursor; route, window and + path are pinned in: query name: pageCursor type: string diff --git a/internal/api/handlers/route_evidence.go b/internal/api/handlers/route_evidence.go index a194306c..3819cbec 100644 --- a/internal/api/handlers/route_evidence.go +++ b/internal/api/handlers/route_evidence.go @@ -17,7 +17,7 @@ import ( func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.RouteEvidenceQuery, error) { q := api.RouteEvidenceQuery{} values := r.URL.Query() - for _, name := range []string{"pageCursor", "since", "until", "range", "limit"} { + for _, name := range []string{"pageCursor", "since", "until", "range", "limit", "hashSize", "pathBytes"} { if len(values[name]) > 1 { return q, api.ErrRouteEvidenceInput } @@ -38,6 +38,7 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R q.Cursor = c q.Since = c.Since q.Until = c.Until + q.HashSize, q.PathBytes = c.HashSize, c.PathBytes } else { q.Until = now.UTC().Truncate(time.Millisecond) q.Since = q.Until.Add(-24 * time.Hour) @@ -67,6 +68,17 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R return q, api.ErrRouteEvidenceInput } } + if values.Has("hashSize") || values.Has("pathBytes") { + width, err := strconv.ParseInt(values.Get("hashSize"), 10, 16) + path := values.Get("pathBytes") + if err != nil || !api.ValidRouteEvidencePath(int16(width), path) || strconv.FormatInt(width, 10) != values.Get("hashSize") { + return q, api.ErrRouteEvidenceInput + } + if q.Cursor != nil && q.Cursor.HashSize != 0 && (q.Cursor.HashSize != int16(width) || q.Cursor.PathBytes != path) { + return q, api.ErrRouteEvidenceInput + } + q.HashSize, q.PathBytes = int16(width), path + } if !api.ValidRouteEvidenceWindow(q.Since, q.Until) || q.Until.After(now.Add(clockSkewTolerance)) { return q, api.ErrRouteEvidenceInput } @@ -76,7 +88,7 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R // getRouteEvidence godoc // // @Summary Get retained reports matching a full saved route prefix sequence -// @Description Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope. +// @Description Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported. // @Tags Routes // @Produce json // @Param iata path string true "Three-character IATA code" @@ -84,7 +96,9 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R // @Param range query string false "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor" // @Param since query int false "Window start epoch ms; provide with until (default last 24h)" // @Param until query int false "Exclusive window end epoch ms; maximum span 30d, end may be up to 5 minutes ahead of server time (clock skew tolerance)" -// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; window is pinned" +// @Param hashSize query int false "Exact hash width (1–3); provide with pathBytes to pin a response's representation" minimum(1) maximum(3) +// @Param pathBytes query string false "Complete lower-case hex path from the response; provide with hashSize" +// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; route, window and path are pinned" // @Param limit query int false "Default 50; positive, capped at 200" minimum(1) maximum(200) // @Success 200 {object} api.RouteEvidence // @Failure 400 {object} handlers.APIError @@ -100,7 +114,7 @@ func getRouteEvidence(reader api.Reader) http.HandlerFunc { } query, err := routeEvidenceQuery(r, iata, key, time.Now()) if err != nil { - respondError(w, 400, "invalid route evidence window, limit or page cursor") + respondError(w, 400, "invalid route evidence window, path, limit or page cursor") return } result, err := reader.GetRouteEvidence(r.Context(), iata, key, query) diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go index f2268d86..88f057df 100644 --- a/internal/api/handlers/route_evidence_test.go +++ b/internal/api/handlers/route_evidence_test.go @@ -85,3 +85,28 @@ func TestRouteEvidenceHandler(t *testing.T) { } } } + +func TestRouteEvidencePinnedQuery(t *testing.T) { + now := time.Now().UTC().Truncate(time.Millisecond) + q, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?hashSize=2&pathBytes=aa01bb02", nil), "YOW", evidenceKey, now) + if err != nil || q.HashSize != 2 || q.PathBytes != "aa01bb02" { + t.Fatalf("selected path: %+v %v", q, err) + } + c := api.RouteEvidenceCursor{IATA: "YOW", PathKey: evidenceKey, Since: q.Since, Until: q.Until, HeardAt: q.Since.Add(time.Hour), ID: 5, HashSize: q.HashSize, PathBytes: q.PathBytes} + encoded := url.QueryEscape(c.String()) + next, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+encoded, nil), "YOW", evidenceKey, now) + if err != nil || next.HashSize != 2 || next.PathBytes != q.PathBytes { + t.Fatalf("cursor path lost: %+v %v", next, err) + } + for _, query := range []string{ + "hashSize=1", "pathBytes=aabb", "hashSize=0&pathBytes=aabb", "hashSize=4&pathBytes=aa01bb02", + "hashSize=01&pathBytes=aabb", "hashSize=1&hashSize=2&pathBytes=aabb", + "hashSize=1&pathBytes=aabb&pathBytes=aabb", "hashSize=1&pathBytes=aa", "hashSize=1&pathBytes=AABB", + "pageCursor=" + encoded + "&hashSize=1&pathBytes=aabb", + "pageCursor=" + encoded + "&hashSize=2&pathBytes=aa02bb01", + } { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } +} diff --git a/internal/api/route_evidence.go b/internal/api/route_evidence.go index d320ade5..490c91b2 100644 --- a/internal/api/route_evidence.go +++ b/internal/api/route_evidence.go @@ -50,16 +50,22 @@ type RouteEvidenceQuery struct { Until time.Time Cursor *RouteEvidenceCursor Limit int32 + // Optional exact representation copied from a prior response or cursor. + HashSize int16 + PathBytes string } -// RouteEvidenceCursor binds the precise (heard_at DESC,id DESC) boundary to its route/window. +// RouteEvidenceCursor pins the route, window and exact representation as well as +// the precise (heard_at DESC,id DESC) boundary. Legacy v1 cursors have no path pin. type RouteEvidenceCursor struct { - IATA string - PathKey string - Since time.Time - Until time.Time - HeardAt time.Time - ID int64 + IATA string + PathKey string + Since time.Time + Until time.Time + HeardAt time.Time + ID int64 + HashSize int16 + PathBytes string } func ValidRouteEvidenceKey(iata, key string) bool { @@ -79,20 +85,32 @@ func ValidRouteEvidenceWindow(since, until time.Time) bool { return !since.Before(time.Unix(0, 0)) && until.After(since) && until.Sub(since) <= MaxRouteEvidenceWindow } +func ValidRouteEvidencePath(width int16, path string) bool { + if width < 1 || width > 3 || len(path) < 4*int(width) || len(path) > 126*int(width) || len(path)%(2*int(width)) != 0 { + return false + } + _, err := hex.DecodeString(path) + return err == nil && path == strings.ToLower(path) +} + func (c RouteEvidenceCursor) String() string { - return fmt.Sprintf("v1:%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) + value := fmt.Sprintf("%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) + if c.HashSize != 0 || c.PathBytes != "" { + return fmt.Sprintf("v2:%s:%d:%s", value, c.HashSize, c.PathBytes) + } + return "v1:" + value } func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { - if len(raw) > 200 { + if len(raw) > 512 { return nil, ErrRouteEvidenceInput } p := strings.Split(raw, ":") - if len(p) != 7 || p[0] != "v1" || !ValidRouteEvidenceKey(p[1], p[2]) { + if !((len(p) == 7 && p[0] == "v1") || (len(p) == 9 && p[0] == "v2")) || !ValidRouteEvidenceKey(p[1], p[2]) { return nil, ErrRouteEvidenceInput } values := [4]int64{} - for i, s := range p[3:] { + for i, s := range p[3:7] { n, err := strconv.ParseInt(s, 10, 64) if err != nil || n < 0 || strconv.FormatInt(n, 10) != s { return nil, ErrRouteEvidenceInput @@ -104,6 +122,13 @@ func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { return nil, ErrRouteEvidenceInput } c := &RouteEvidenceCursor{IATA: p[1], PathKey: p[2], Since: time.UnixMilli(values[0]).UTC(), Until: time.UnixMilli(values[1]).UTC(), HeardAt: time.UnixMicro(values[2]).UTC(), ID: values[3]} + if p[0] == "v2" { + width, err := strconv.ParseInt(p[7], 10, 16) + if err != nil || strconv.FormatInt(width, 10) != p[7] || !ValidRouteEvidencePath(int16(width), p[8]) { + return nil, ErrRouteEvidenceInput + } + c.HashSize, c.PathBytes = int16(width), p[8] + } if !ValidRouteEvidenceWindow(c.Since, c.Until) || c.HeardAt.Before(c.Since) || !c.HeardAt.Before(c.Until) { return nil, ErrRouteEvidenceInput } diff --git a/internal/api/route_evidence_test.go b/internal/api/route_evidence_test.go index 42c9a73a..a025a440 100644 --- a/internal/api/route_evidence_test.go +++ b/internal/api/route_evidence_test.go @@ -23,3 +23,23 @@ func TestRouteEvidenceCursor(t *testing.T) { } } } + +func TestRouteEvidencePinnedCursor(t *testing.T) { + c := RouteEvidenceCursor{IATA: "YOW", PathKey: strings.Repeat("a", 32), Since: time.UnixMilli(1000).UTC(), Until: time.UnixMilli(3000).UTC(), HeardAt: time.UnixMicro(2000123).UTC(), ID: 9, HashSize: 3, PathBytes: strings.Repeat("aabbcc", 63)} + got, err := ParseRouteEvidenceCursor(c.String()) + if err != nil || *got != c || !strings.HasPrefix(c.String(), "v2:") { + t.Fatalf("pinned cursor round-trip: %+v %v", got, err) + } + for _, bad := range []string{ + strings.Replace(c.String(), ":3:", ":4:", 1), + strings.Replace(c.String(), ":3:", ":03:", 1), + c.String() + "00", + strings.TrimSuffix(c.String(), c.PathBytes) + "aabbcc", + strings.ToUpper(c.String()), + strings.TrimSuffix(c.String(), c.PathBytes) + "zzzzzzzzzzzz", + } { + if _, err := ParseRouteEvidenceCursor(bad); err == nil { + t.Errorf("accepted invalid pinned cursor %q", bad) + } + } +} From af2604a610ee9758d448ffb1caa44b082182248f Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 19:01:54 -0400 Subject: [PATCH 02/23] test(routes): separate parameterized PostgreSQL fixture setup --- db/route_evidence_integration_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index bef6989d..b716fbdb 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -99,8 +99,10 @@ func TestRouteEvidencePostgres(t *testing.T) { } // A new representation arrives while page one is open. New reads follow it, // but pagination and copied links retain the exact original representation. - _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash) VALUES (int4send(12)),(int4send(13)),(int4send(14)); - INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) + if _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash) VALUES (int4send(12)),(int4send(13)),(int4send(14))`); err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) SELECT i,int4send(i),'00000000-0000-0000-0000-000000000001','YOW',$1,2,2,'\xaa01bb03',4 FROM generate_series(12,14) i`, at) if err != nil { t.Fatal(err) From a5c8d2839b984f43afd5506a1fbcf64a02b7d1ac Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 21:07:42 -0400 Subject: [PATCH 03/23] feat(scopes): expose cached regional catalogue context --- cmd/beacon/main.go | 1 + docs/docs.go | 78 +++++++++++++++++++ docs/swagger.json | 78 +++++++++++++++++++ docs/swagger.yaml | 55 ++++++++++++++ internal/api/handlers/scope_catalogues.go | 33 ++++++++ internal/api/router/options.go | 3 + internal/api/router/router.go | 1 + internal/api/router/scope_catalogues_test.go | 28 +++++++ internal/meshmapper/catalogues.go | 75 ++++++++++++++++++ internal/meshmapper/catalogues_test.go | 80 ++++++++++++++++++++ internal/meshmapper/scopes.go | 4 + 11 files changed, 436 insertions(+) create mode 100644 internal/api/handlers/scope_catalogues.go create mode 100644 internal/api/router/scope_catalogues_test.go create mode 100644 internal/meshmapper/catalogues.go create mode 100644 internal/meshmapper/catalogues_test.go diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 5017dffb..daafc06f 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -367,6 +367,7 @@ func main() { // CachedReader. Only response projections receive the geographic annotation. reader = api.WithLocalBorders(reader, localBorders) r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, router.Options{ + ScopeCatalogues: scopeImporter, MaxConnsPerIP: resolved.MaxConnsPerIP, MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, WSAllowedOrigins: cfg.WebSocket.AllowedOrigins, diff --git a/docs/docs.go b/docs/docs.go index 2c79a821..81b8afca 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2231,6 +2231,29 @@ const docTemplate = `{ } } }, + "/scope-catalogues": { + "get": { + "description": "Configured sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", + "produces": [ + "application/json" + ], + "tags": [ + "Scopes" + ], + "summary": "Cached MeshMapper regional scope catalogues", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue" + } + } + } + } + } + }, "/scopes": { "get": { "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", @@ -5442,6 +5465,61 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue": { + "type": "object", + "properties": { + "checkedAt": { + "type": "integer" + }, + "freshUntil": { + "type": "integer" + }, + "generatedAt": { + "type": "integer" + }, + "iata": { + "type": "string" + }, + "lastError": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "scoped": { + "type": "integer" + }, + "scopes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope" + } + }, + "url": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { + "type": "object", + "properties": { + "default": { + "type": "integer" + }, + "monitored": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "wardriving": { + "type": "boolean" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index f16f2166..ca0e504a 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2229,6 +2229,29 @@ } } }, + "/scope-catalogues": { + "get": { + "description": "Configured sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", + "produces": [ + "application/json" + ], + "tags": [ + "Scopes" + ], + "summary": "Cached MeshMapper regional scope catalogues", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue" + } + } + } + } + } + }, "/scopes": { "get": { "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", @@ -5440,6 +5463,61 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue": { + "type": "object", + "properties": { + "checkedAt": { + "type": "integer" + }, + "freshUntil": { + "type": "integer" + }, + "generatedAt": { + "type": "integer" + }, + "iata": { + "type": "string" + }, + "lastError": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "scoped": { + "type": "integer" + }, + "scopes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope" + } + }, + "url": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { + "type": "object", + "properties": { + "default": { + "type": "integer" + }, + "monitored": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "wardriving": { + "type": "boolean" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9b029830..796cc770 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1492,6 +1492,42 @@ definitions: requires_restart: type: boolean type: object + github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue: + properties: + checkedAt: + type: integer + freshUntil: + type: integer + generatedAt: + type: integer + iata: + type: string + lastError: + type: string + repeaters: + type: integer + scoped: + type: integer + scopes: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope' + type: array + url: + type: string + type: object + github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope: + properties: + default: + type: integer + monitored: + type: boolean + name: + type: string + repeaters: + type: integer + wardriving: + type: boolean + type: object internal_api_handlers.APIError: properties: code: @@ -3026,6 +3062,25 @@ paths: summary: Search known routes by source and destination hash tags: - Routes + /scope-catalogues: + get: + description: Configured sources only. Counts describe the source region, not + individual nodes or links. No upstream request is made. An empty list means + no configured importer; checkedAt=0 means no successful check. lastError or + a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence + are separate. + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.Catalogue' + type: array + summary: Cached MeshMapper regional scope catalogues + tags: + - Scopes /scopes: get: description: The unfiltered list includes stored imported names after an importer diff --git a/internal/api/handlers/scope_catalogues.go b/internal/api/handlers/scope_catalogues.go new file mode 100644 index 00000000..86ec772e --- /dev/null +++ b/internal/api/handlers/scope_catalogues.go @@ -0,0 +1,33 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "net/http" + + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" +) + +// ScopeCatalogueSource supplies immutable, already-cached public metadata. +type ScopeCatalogueSource interface { + Catalogues() []meshmapper.Catalogue +} + +// ScopeCatalogues godoc +// @Summary Cached MeshMapper regional scope catalogues +// @Description Configured sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate. +// @Tags Scopes +// @Produce json +// @Success 200 {array} meshmapper.Catalogue +// @Router /scope-catalogues [get] +func ScopeCatalogues(source ScopeCatalogueSource) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + items := []meshmapper.Catalogue{} + if source != nil { + items = source.Catalogues() + } + w.Header().Set("Cache-Control", "public, max-age=60") + respond(w, http.StatusOK, items) + } +} diff --git a/internal/api/router/options.go b/internal/api/router/options.go index 757a6593..f033bd92 100644 --- a/internal/api/router/options.go +++ b/internal/api/router/options.go @@ -6,12 +6,15 @@ package router import ( "net/http" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/config" ) // Options names the startup settings used by the router. Adding an optional // route does not change New's signature or require unrelated callers to change. type Options struct { + ScopeCatalogues handlers.ScopeCatalogueSource MaxConnsPerIP int MaxConnectsPerMinute int WSAllowedOrigins []string diff --git a/internal/api/router/router.go b/internal/api/router/router.go index 58bbfac5..95ba3cc3 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -108,6 +108,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) r.Mount("/regions", handlers.RegionsRouter(reader)) r.Mount("/routes", handlers.RoutesRouter(reader)) r.Mount("/scopes", handlers.ScopesRouter(reader)) + r.Get("/scope-catalogues", handlers.ScopeCatalogues(opts.ScopeCatalogues)) r.Mount("/stats", handlers.StatsRouter(reader)) r.Mount("/traces", handlers.TracesRouter(reader)) }) diff --git a/internal/api/router/scope_catalogues_test.go b/internal/api/router/scope_catalogues_test.go new file mode 100644 index 00000000..5092e4c0 --- /dev/null +++ b/internal/api/router/scope_catalogues_test.go @@ -0,0 +1,28 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" +) + +func TestScopeCataloguesDisabledAndReadOnly(t *testing.T) { + var importer *meshmapper.Importer + r := New(nil, nil, nil, Options{ScopeCatalogues: importer}) + for _, method := range []string{http.MethodGet, http.MethodPost} { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(method, "/api/v1/scope-catalogues", nil)) + if method == http.MethodGet && (w.Code != 200 || strings.TrimSpace(w.Body.String()) != "[]" || w.Header().Get("Cache-Control") != "public, max-age=60") { + t.Fatal(w.Code, w.Body.String()) + } + if method == http.MethodPost && w.Code != 405 { + t.Fatal(w.Code) + } + } +} diff --git a/internal/meshmapper/catalogues.go b/internal/meshmapper/catalogues.go new file mode 100644 index 00000000..530aafc2 --- /dev/null +++ b/internal/meshmapper/catalogues.go @@ -0,0 +1,75 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "encoding/json" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +// CatalogueScope is regional source metadata, not per-node forwarding evidence. +type CatalogueScope struct { + Name string `json:"name"` + Repeaters int `json:"repeaters"` + Default int `json:"default"` + Monitored bool `json:"monitored"` + Wardriving bool `json:"wardriving"` +} + +type Catalogue struct { + IATA string `json:"iata"` + URL string `json:"url"` + GeneratedAt int64 `json:"generatedAt"` + CheckedAt int64 `json:"checkedAt"` + FreshUntil int64 `json:"freshUntil"` + LastError string `json:"lastError,omitempty"` + Repeaters int `json:"repeaters"` + Scoped int `json:"scoped"` + Scopes []CatalogueScope `json:"scopes"` +} + +// Catalogues returns an immutable snapshot of configured sources. No request +// triggers an upstream fetch or reads ingestion tables. Callers must not mutate it. +func (i *Importer) Catalogues() []Catalogue { + if i != nil { + if snapshot := i.catalogues.Load(); snapshot != nil { + return snapshot.([]Catalogue) + } + } + return []Catalogue{} +} + +func (i *Importer) publishCatalogues() { + result := make([]Catalogue, 0, len(i.sources)) + for _, s := range i.sources { + c := Catalogue{IATA: s.iata, URL: s.url, LastError: s.cache.LastError, Scopes: []CatalogueScope{}} + if !s.cache.CheckedAt.IsZero() { + c.CheckedAt = s.cache.CheckedAt.UnixMilli() + c.FreshUntil = s.cache.CheckedAt.Add(i.interval + 5*time.Minute).UnixMilli() + } + if len(s.cache.Payload) > 0 { + // Reuse the importer validation before exposing saved metadata. Invalid + // saved catalogues are omitted just as they are from the scope matcher. + if _, generated, err := decode(s.cache.Payload, s.iata); err == nil { + var payload struct { + Repeaters int + Scoped int + Scopes []CatalogueScope + } + if json.Unmarshal(s.cache.Payload, &payload) == nil { + c.GeneratedAt = generated.UnixMilli() + c.Repeaters, c.Scoped = payload.Repeaters, payload.Scoped + for _, scope := range payload.Scopes { + scope.Name = scopestore.FromName(scope.Name).Name + c.Scopes = append(c.Scopes, scope) + } + } + } + } + result = append(result, c) + } + i.catalogues.Store(result) +} diff --git a/internal/meshmapper/catalogues_test.go b/internal/meshmapper/catalogues_test.go new file mode 100644 index 00000000..01f6cb88 --- /dev/null +++ b/internal/meshmapper/catalogues_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { + status := http.StatusOK + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(status) + if status == http.StatusOK { + _, _ = w.Write(catalogue("yow", "YOW", "monitored-zero")) + } + })) + defer server.Close() + cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} + store := &memoryStore{rows: map[string]Cache{}} + imp, err := New(context.Background(), cfg, store, scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + if got := imp.Catalogues(); len(got) != 1 || got[0].CheckedAt != 0 || len(got[0].Scopes) != 0 { + t.Fatal(got) + } + now := time.Now() + if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil { + t.Fatal(err) + } + old := imp.Catalogues() + if old[0].Scopes[0].Name != "#yow" || old[0].Scopes[1].Name != "#YOW" || len(old[0].Scopes) != 3 || old[0].Repeaters != 5 || old[0].Scoped != 3 || old[0].FreshUntil <= old[0].CheckedAt { + t.Fatal(old) + } + status = http.StatusServiceUnavailable + if err := imp.refresh(context.Background(), &imp.sources[0], now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + current := imp.Catalogues()[0] + if current.LastError != "HTTP 503" || current.CheckedAt != old[0].CheckedAt || current.GeneratedAt != old[0].GeneratedAt || len(current.Scopes) != 3 || old[0].LastError != "" { + t.Fatal(current, old) + } + // Readers only access published immutable data; the race suite exercises concurrent refreshes. + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for range 100 { + _, _ = json.Marshal(imp.Catalogues()) + } + }() + for range 100 { + imp.publishCatalogues() + } + wg.Wait() + if calls != 2 { + t.Fatal("reading public metadata performed upstream requests", calls) + } + disabled, err := New(context.Background(), config.MeshMapperScopesConfig{}, store, scopestore.New(), nil) + if err != nil || len(disabled.Catalogues()) != 0 { + t.Fatal("disabled importer exposed old membership") + } + store.rows["YOW"+server.URL] = Cache{Payload: []byte(`{"scopes":"invalid"}`)} + restored, err := New(context.Background(), cfg, store, scopestore.New(), nil) + if err != nil || len(restored.Catalogues()[0].Scopes) != 0 || restored.Catalogues()[0].LastError == "" { + t.Fatal("invalid saved metadata leaked") + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index ac389a14..3f210bed 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -14,6 +14,7 @@ import ( "slices" "strconv" "strings" + "sync/atomic" "time" "unicode" "unicode/utf8" @@ -52,6 +53,7 @@ type source struct { // Importer is owned by one background task; ScopeStore synchronizes its consumers. type Importer struct { + catalogues atomic.Value // immutable []Catalogue, refreshed off the ingest path store Store scopes *scopestore.ScopeStore manual []scopestore.Entry @@ -103,6 +105,7 @@ func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, sc i.log(s, "restored") } i.publish() + i.publishCatalogues() return i, nil } @@ -224,6 +227,7 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error i.onChange(ctx) } } + i.publishCatalogues() i.log(*s, "checked") return nil } From bcbc8b73c1fbfed89054f48e7ff384f0ba796064 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 21:55:16 -0400 Subject: [PATCH 04/23] Adapt pinned-route fixtures to required baseline columns --- db/route_evidence_integration_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index d41ee166..da0e4dde 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -35,7 +35,6 @@ func TestRouteEvidencePostgres(t *testing.T) { ctx, tx := retentionTx(t) applyBaseline(t, ctx, tx) _, err := tx.Exec(ctx, ` - INSERT INTO iata_codes(iata) VALUES ('YOW'),('YVR') ON CONFLICT DO NOTHING; INSERT INTO observers(id,public_key,display_name) VALUES ('00000000-0000-0000-0000-000000000001','\x01','One'),('00000000-0000-0000-0000-000000000002','\x02','Two'); INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa0102',2,'A'),('00000000-0000-0000-0000-000000000002','\xbb0304',2,'B'); INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) @@ -87,11 +86,11 @@ func TestRouteEvidencePostgres(t *testing.T) { } // A new representation arrives while page one is open. New reads follow it, // but pagination and copied links retain the exact original representation. - if _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash) VALUES (int4send(12)),(int4send(13)),(int4send(14))`); err != nil { + if _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(12,14) i`); err != nil { t.Fatal(err) } - _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) - SELECT i,int4send(i),'00000000-0000-0000-0000-000000000001','YOW',$1,2,2,'\xaa01bb03',4 FROM generate_series(12,14) i`, at) + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,path_length_byte) + SELECT i,int4send(i),'00000000-0000-0000-0000-000000000001','YOW',$1,2,2,'\xaa01bb03',4,66 FROM generate_series(12,14) i`, at) if err != nil { t.Fatal(err) } From eda1f3c25e44b1c4031a211bd5a88b8dd872c845 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 21:56:24 -0400 Subject: [PATCH 05/23] Keep node types explicit in baseline evidence fixtures --- db/route_evidence_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index da0e4dde..53e70c0c 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -141,7 +141,7 @@ func TestRouteEvidencePostgres(t *testing.T) { if _, err := store.GetRouteEvidence(ctx, "YOW", key, wrong); !errors.Is(err, api.ErrRouteEvidenceInput) { t.Fatalf("wrong hop count accepted: %v", err) } - _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000003','\xaa9988','Collision'); + _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000003','\xaa9988',2,'Collision'); DELETE FROM nodes WHERE id='00000000-0000-0000-0000-000000000001'`) if err != nil { t.Fatal(err) @@ -149,7 +149,7 @@ func TestRouteEvidencePostgres(t *testing.T) { if _, err := store.GetRouteEvidence(ctx, "YOW", key, shared); !errors.Is(err, api.ErrRouteEvidenceInput) { t.Fatalf("missing node substituted by colliding node: %v", err) } - if _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa0102','A')`); err != nil { + if _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa0102',2,'A')`); err != nil { t.Fatal(err) } copyPage, err = store.GetRouteEvidence(ctx, "YOW", key, shared) From 644960e4a6d6a1c02e10ace182f6d335358d3f44 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 22:00:12 -0400 Subject: [PATCH 06/23] fix(nodes): include known default scope in list responses --- db/nodes.go | 1 + db/nodes_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/db/nodes.go b/db/nodes.go index a602cb3d..cff12107 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -127,6 +127,7 @@ func (s *Store) ListNodes(ctx context.Context, nodeType int16, iatas []string, s Longitude: v.Longitude, IsObserver: v.IsObserver, ObserverID: nullableUUID(v.ObserverID), + DefaultScope: v.DefaultScopeName, KnownNeighborCount: v.KnownNeighborCount, NeighborIDs: v.NeighborIds, Stale: v.LastSeen.Valid && v.LastSeen.Time.Before(time.Now().Add(-s.staleThreshold)), diff --git a/db/nodes_test.go b/db/nodes_test.go index db9a0dd4..590c08cd 100644 --- a/db/nodes_test.go +++ b/db/nodes_test.go @@ -706,3 +706,15 @@ func TestDeleteOldNodes(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestListNodes_DefaultScopePreservesCaseAndMissingValue(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + scope := "#YOW" + mock.EXPECT().ListNodes(gomock.Any(), gomock.Any()).Return([]sqlc.ListNodesRow{{DefaultScopeName: &scope}, {}}, nil) + store := &Store{q: mock} + page, err := store.ListNodes(context.Background(), 0, nil, nil, nil, nil, "", "", "", 0, 10, false) + if err != nil || len(page.Items) != 2 || page.Items[0].DefaultScope == nil || *page.Items[0].DefaultScope != scope || page.Items[1].DefaultScope != nil { + t.Fatalf("scope metadata lost or invented: %+v %v", page, err) + } +} From 835d1cdd4887576a6ca0cb687ef8077b73640f33 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 22:00:12 -0400 Subject: [PATCH 07/23] fix(nodes): include known default scope in list responses --- db/nodes.go | 1 + db/nodes_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/db/nodes.go b/db/nodes.go index a602cb3d..cff12107 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -127,6 +127,7 @@ func (s *Store) ListNodes(ctx context.Context, nodeType int16, iatas []string, s Longitude: v.Longitude, IsObserver: v.IsObserver, ObserverID: nullableUUID(v.ObserverID), + DefaultScope: v.DefaultScopeName, KnownNeighborCount: v.KnownNeighborCount, NeighborIDs: v.NeighborIds, Stale: v.LastSeen.Valid && v.LastSeen.Time.Before(time.Now().Add(-s.staleThreshold)), diff --git a/db/nodes_test.go b/db/nodes_test.go index db9a0dd4..590c08cd 100644 --- a/db/nodes_test.go +++ b/db/nodes_test.go @@ -706,3 +706,15 @@ func TestDeleteOldNodes(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestListNodes_DefaultScopePreservesCaseAndMissingValue(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + scope := "#YOW" + mock.EXPECT().ListNodes(gomock.Any(), gomock.Any()).Return([]sqlc.ListNodesRow{{DefaultScopeName: &scope}, {}}, nil) + store := &Store{q: mock} + page, err := store.ListNodes(context.Background(), 0, nil, nil, nil, nil, "", "", "", 0, 10, false) + if err != nil || len(page.Items) != 2 || page.Items[0].DefaultScope == nil || *page.Items[0].DefaultScope != scope || page.Items[1].DefaultScope != nil { + t.Fatalf("scope metadata lost or invented: %+v %v", page, err) + } +} From 6ad59d31291b8abe6a7d2bd7a055de05dbeb319d Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 2 Oct 2026 08:56:35 -0400 Subject: [PATCH 08/23] feat(topology): expose cached regional scope context for 2.0 --- cmd/beacon/main.go | 1 + db/nodes.go | 1 + db/nodes_test.go | 12 +++ docs/docs.go | 78 ++++++++++++++ docs/swagger.json | 78 ++++++++++++++ docs/swagger.yaml | 55 ++++++++++ internal/api/handlers/scope_catalogues.go | 33 ++++++ internal/api/router/options.go | 1 + internal/api/router/router.go | 1 + internal/api/router/scope_catalogues_test.go | 28 +++++ internal/meshmapper/catalogues.go | 75 ++++++++++++++ internal/meshmapper/catalogues_test.go | 103 +++++++++++++++++++ internal/meshmapper/scopes.go | 24 +++-- 13 files changed, 480 insertions(+), 10 deletions(-) create mode 100644 internal/api/handlers/scope_catalogues.go create mode 100644 internal/api/router/scope_catalogues_test.go create mode 100644 internal/meshmapper/catalogues.go create mode 100644 internal/meshmapper/catalogues_test.go diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 800c88aa..fac3720e 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -425,6 +425,7 @@ func main() { // CachedReader. Only response projections receive the geographic annotation. reader = api.WithLocalBorders(reader, localBorders) r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, router.Options{ + ScopeCatalogues: scopeImporter, MaxConnsPerIP: resolved.MaxConnsPerIP, MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, WSAllowedOrigins: cfg.WebSocket.AllowedOrigins, diff --git a/db/nodes.go b/db/nodes.go index a602cb3d..cff12107 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -127,6 +127,7 @@ func (s *Store) ListNodes(ctx context.Context, nodeType int16, iatas []string, s Longitude: v.Longitude, IsObserver: v.IsObserver, ObserverID: nullableUUID(v.ObserverID), + DefaultScope: v.DefaultScopeName, KnownNeighborCount: v.KnownNeighborCount, NeighborIDs: v.NeighborIds, Stale: v.LastSeen.Valid && v.LastSeen.Time.Before(time.Now().Add(-s.staleThreshold)), diff --git a/db/nodes_test.go b/db/nodes_test.go index db9a0dd4..590c08cd 100644 --- a/db/nodes_test.go +++ b/db/nodes_test.go @@ -706,3 +706,15 @@ func TestDeleteOldNodes(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestListNodes_DefaultScopePreservesCaseAndMissingValue(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + scope := "#YOW" + mock.EXPECT().ListNodes(gomock.Any(), gomock.Any()).Return([]sqlc.ListNodesRow{{DefaultScopeName: &scope}, {}}, nil) + store := &Store{q: mock} + page, err := store.ListNodes(context.Background(), 0, nil, nil, nil, nil, "", "", "", 0, 10, false) + if err != nil || len(page.Items) != 2 || page.Items[0].DefaultScope == nil || *page.Items[0].DefaultScope != scope || page.Items[1].DefaultScope != nil { + t.Fatalf("scope metadata lost or invented: %+v %v", page, err) + } +} diff --git a/docs/docs.go b/docs/docs.go index 425a460f..e75b2879 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2223,6 +2223,29 @@ const docTemplate = `{ } } }, + "/scope-catalogues": { + "get": { + "description": "Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", + "produces": [ + "application/json" + ], + "tags": [ + "Scopes" + ], + "summary": "Cached MeshMapper regional scope catalogues", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue" + } + } + } + } + } + }, "/scopes": { "get": { "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", @@ -5649,6 +5672,61 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { + "type": "object", + "properties": { + "default": { + "type": "integer" + }, + "monitored": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "wardriving": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue": { + "type": "object", + "properties": { + "checkedAt": { + "type": "integer" + }, + "freshUntil": { + "type": "integer" + }, + "generatedAt": { + "type": "integer" + }, + "iata": { + "type": "string" + }, + "lastError": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "scoped": { + "type": "integer" + }, + "scopes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope" + } + }, + "url": { + "type": "string" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 757caae0..b2ed0348 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2221,6 +2221,29 @@ } } }, + "/scope-catalogues": { + "get": { + "description": "Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", + "produces": [ + "application/json" + ], + "tags": [ + "Scopes" + ], + "summary": "Cached MeshMapper regional scope catalogues", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue" + } + } + } + } + } + }, "/scopes": { "get": { "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", @@ -5647,6 +5670,61 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { + "type": "object", + "properties": { + "default": { + "type": "integer" + }, + "monitored": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "wardriving": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue": { + "type": "object", + "properties": { + "checkedAt": { + "type": "integer" + }, + "freshUntil": { + "type": "integer" + }, + "generatedAt": { + "type": "integer" + }, + "iata": { + "type": "string" + }, + "lastError": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "scoped": { + "type": "integer" + }, + "scopes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope" + } + }, + "url": { + "type": "string" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index b1431feb..9fb6cd9c 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1575,6 +1575,42 @@ definitions: requires_restart: type: boolean type: object + github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope: + properties: + default: + type: integer + monitored: + type: boolean + name: + type: string + repeaters: + type: integer + wardriving: + type: boolean + type: object + github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue: + properties: + checkedAt: + type: integer + freshUntil: + type: integer + generatedAt: + type: integer + iata: + type: string + lastError: + type: string + repeaters: + type: integer + scoped: + type: integer + scopes: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope' + type: array + url: + type: string + type: object internal_api_handlers.APIError: properties: code: @@ -3101,6 +3137,25 @@ paths: summary: Search known routes by source and destination hash tags: - Routes + /scope-catalogues: + get: + description: Discovered sources only. Counts describe the source region, not + individual nodes or links. No upstream request is made. An empty list means + no configured importer; checkedAt=0 means no successful check. lastError or + a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence + are separate. + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue' + type: array + summary: Cached MeshMapper regional scope catalogues + tags: + - Scopes /scopes: get: description: Without filters, lists every stored scope name, including imported diff --git a/internal/api/handlers/scope_catalogues.go b/internal/api/handlers/scope_catalogues.go new file mode 100644 index 00000000..886a874a --- /dev/null +++ b/internal/api/handlers/scope_catalogues.go @@ -0,0 +1,33 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "net/http" + + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" +) + +// ScopeCatalogueSource supplies immutable, already-cached public metadata. +type ScopeCatalogueSource interface { + Catalogues() []meshmapper.PublicScopeCatalogue +} + +// ScopeCatalogues godoc +// @Summary Cached MeshMapper regional scope catalogues +// @Description Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate. +// @Tags Scopes +// @Produce json +// @Success 200 {array} meshmapper.PublicScopeCatalogue +// @Router /scope-catalogues [get] +func ScopeCatalogues(source ScopeCatalogueSource) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + items := []meshmapper.PublicScopeCatalogue{} + if source != nil { + items = source.Catalogues() + } + w.Header().Set("Cache-Control", "public, max-age=60") + respond(w, http.StatusOK, items) + } +} diff --git a/internal/api/router/options.go b/internal/api/router/options.go index e838834e..a62fff0d 100644 --- a/internal/api/router/options.go +++ b/internal/api/router/options.go @@ -14,6 +14,7 @@ import ( // Options names the startup settings used by the router. Adding an optional // route does not change New's signature or require unrelated callers to change. type Options struct { + ScopeCatalogues handlers.ScopeCatalogueSource MaxConnsPerIP int MaxConnectsPerMinute int WSAllowedOrigins []string diff --git a/internal/api/router/router.go b/internal/api/router/router.go index d6bbbc97..7ce218b3 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -109,6 +109,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) r.Mount("/routes", handlers.RoutesRouter(reader)) r.Mount("/scopes", handlers.ScopesRouter(reader, opts.Scopes)) r.Mount("/stats", handlers.StatsRouter(reader, handlers.StatsOptions{Scopes: opts.Scopes, SeriesWindow: opts.RollupRetention})) + r.Get("/scope-catalogues", handlers.ScopeCatalogues(opts.ScopeCatalogues)) r.Mount("/traces", handlers.TracesRouter(reader)) }) diff --git a/internal/api/router/scope_catalogues_test.go b/internal/api/router/scope_catalogues_test.go new file mode 100644 index 00000000..5092e4c0 --- /dev/null +++ b/internal/api/router/scope_catalogues_test.go @@ -0,0 +1,28 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" +) + +func TestScopeCataloguesDisabledAndReadOnly(t *testing.T) { + var importer *meshmapper.Importer + r := New(nil, nil, nil, Options{ScopeCatalogues: importer}) + for _, method := range []string{http.MethodGet, http.MethodPost} { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(method, "/api/v1/scope-catalogues", nil)) + if method == http.MethodGet && (w.Code != 200 || strings.TrimSpace(w.Body.String()) != "[]" || w.Header().Get("Cache-Control") != "public, max-age=60") { + t.Fatal(w.Code, w.Body.String()) + } + if method == http.MethodPost && w.Code != 405 { + t.Fatal(w.Code) + } + } +} diff --git a/internal/meshmapper/catalogues.go b/internal/meshmapper/catalogues.go new file mode 100644 index 00000000..76552291 --- /dev/null +++ b/internal/meshmapper/catalogues.go @@ -0,0 +1,75 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "encoding/json" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +// CatalogueScope is regional source metadata, not per-node forwarding evidence. +type CatalogueScope struct { + Name string `json:"name"` + Repeaters int `json:"repeaters"` + Default int `json:"default"` + Monitored bool `json:"monitored"` + Wardriving bool `json:"wardriving"` +} + +type PublicScopeCatalogue struct { + IATA string `json:"iata"` + URL string `json:"url"` + GeneratedAt int64 `json:"generatedAt"` + CheckedAt int64 `json:"checkedAt"` + FreshUntil int64 `json:"freshUntil"` + LastError string `json:"lastError,omitempty"` + Repeaters int `json:"repeaters"` + Scoped int `json:"scoped"` + Scopes []CatalogueScope `json:"scopes"` +} + +// Catalogues returns an immutable snapshot of discovered sources. No request +// triggers an upstream fetch or reads ingestion tables. Callers must not mutate it. +func (i *Importer) Catalogues() []PublicScopeCatalogue { + if i != nil { + if snapshot := i.catalogues.Load(); snapshot != nil { + return snapshot.([]PublicScopeCatalogue) + } + } + return []PublicScopeCatalogue{} +} + +func (i *Importer) publishCatalogues() { + result := make([]PublicScopeCatalogue, 0, len(i.sources)) + for _, s := range i.sources { + c := PublicScopeCatalogue{IATA: s.iata, URL: s.url, LastError: s.cache.LastError, Scopes: []CatalogueScope{}} + if !s.cache.CheckedAt.IsZero() { + c.CheckedAt = s.cache.CheckedAt.UnixMilli() + c.FreshUntil = s.cache.CheckedAt.Add(i.interval + 5*time.Minute).UnixMilli() + } + if len(s.cache.Payload) > 0 { + // Reuse the importer validation before exposing saved metadata. Invalid + // saved catalogues are omitted just as they are from the scope matcher. + if _, generated, err := decode(s.cache.Payload, s.iata); err == nil { + var payload struct { + Repeaters int + Scoped int + Scopes []CatalogueScope + } + if json.Unmarshal(s.cache.Payload, &payload) == nil { + c.GeneratedAt = generated.UnixMilli() + c.Repeaters, c.Scoped = payload.Repeaters, payload.Scoped + for _, scope := range payload.Scopes { + scope.Name = scopestore.FromName(scope.Name).Name + c.Scopes = append(c.Scopes, scope) + } + } + } + } + result = append(result, c) + } + i.catalogues.Store(result) +} diff --git a/internal/meshmapper/catalogues_test.go b/internal/meshmapper/catalogues_test.go new file mode 100644 index 00000000..baefa72d --- /dev/null +++ b/internal/meshmapper/catalogues_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { + status := http.StatusOK + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(status) + if status == http.StatusOK { + _, _ = w.Write(catalogue("yow", "YOW", "monitored-zero")) + } + })) + defer server.Close() + cfg := config.MeshMapperScopesConfig{Enabled: true} + store := newMemoryStore("YOW") + imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + if got := imp.Catalogues(); len(got) != 1 || got[0].CheckedAt != 0 || len(got[0].Scopes) != 0 { + t.Fatal(got) + } + imp.sources[0].url = server.URL + now := time.Now() + if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil { + t.Fatal(err) + } + imp.publishCatalogues() + old := imp.Catalogues() + if old[0].Scopes[0].Name != "#yow" || old[0].Scopes[1].Name != "#YOW" || len(old[0].Scopes) != 3 || old[0].Repeaters != 5 || old[0].Scoped != 3 || old[0].FreshUntil <= old[0].CheckedAt { + t.Fatal(old) + } + status = http.StatusServiceUnavailable + if err := imp.refresh(context.Background(), &imp.sources[0], now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + imp.publishCatalogues() + current := imp.Catalogues()[0] + if current.LastError != "HTTP 503" || current.CheckedAt != old[0].CheckedAt || current.GeneratedAt != old[0].GeneratedAt || len(current.Scopes) != 3 || old[0].LastError != "" { + t.Fatal(current, old) + } + // Readers only access published immutable data; the race suite exercises concurrent refreshes. + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for range 100 { + _, _ = json.Marshal(imp.Catalogues()) + } + }() + for range 100 { + imp.publishCatalogues() + } + wg.Wait() + if calls != 2 { + t.Fatal("reading public metadata performed upstream requests", calls) + } + disabled, err := New(context.Background(), config.MeshMapperScopesConfig{}, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) + if err != nil || len(disabled.Catalogues()) != 0 { + t.Fatal("disabled importer exposed old membership") + } + store.rows[sourceKey{"YOW", server.URL}] = Cache{Payload: []byte(`{"scopes":"invalid"}`)} + restored, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) + if err != nil || len(restored.Catalogues()[0].Scopes) != 0 || restored.Catalogues()[0].LastError == "" { + t.Fatal("invalid saved metadata leaked") + } +} + +func TestPublicCataloguesIncludeDiscoveredAndUnlistedRegions(t *testing.T) { + ctx := context.Background() + store := newMemoryStore("YOW") + dir := NewDirectory(newZoneListMemory()) + dir.lists["CA"] = &zoneList{zones: map[string]zoneEntry{}, fetchedAt: time.Now()} + imp, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + before := imp.Catalogues() + store.iatas = append(store.iatas, "YKF") + if err := imp.Refresh(ctx); err != nil { + t.Fatal(err) + } + after := imp.Catalogues() + if len(before) != 1 || before[0].LastError != "" || len(after) != 2 || after[1].IATA != "YKF" || after[1].LastError != "not listed" { + t.Fatal(before, after) + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index 3ecdcbe1..330686b4 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -14,6 +14,7 @@ import ( "slices" "strconv" "strings" + "sync/atomic" "time" "unicode" "unicode/utf8" @@ -60,16 +61,17 @@ type source struct { // Importer is owned by one background task; ScopeStore synchronizes its consumers. type Importer struct { - store Store - scopes *scopestore.ScopeStore - manual []scopestore.Entry - dir *Directory - scopesURL func(site string) (string, bool) - seen map[string]bool - sources []source - interval time.Duration - client *http.Client - onChange func(context.Context) + catalogues atomic.Value // immutable []PublicScopeCatalogue, off the ingest path + store Store + scopes *scopestore.ScopeStore + manual []scopestore.Entry + dir *Directory + scopesURL func(site string) (string, bool) + seen map[string]bool + sources []source + interval time.Duration + client *http.Client + onChange func(context.Context) } // SetCacheInvalidator is wired once at startup, before the background task starts. @@ -102,6 +104,7 @@ func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, di } i.track(iatas, latest) i.publish() + i.publishCatalogues() return i, nil } @@ -136,6 +139,7 @@ func (i *Importer) track(iatas []string, saved map[string]Catalogue) { // Refresh checks only one due source. The scheduler serializes calls every 15s. func (i *Importer) Refresh(ctx context.Context) (err error) { + defer i.publishCatalogues() parent := ctx defer func() { if parent.Err() != nil { From 4fda8ed993c7fb130075c7e82fcc20c02e1fc321 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 2 Oct 2026 08:56:40 -0400 Subject: [PATCH 09/23] fix(routes): pin observation evidence to its path representation --- db/route_evidence.go | 39 ++++++++++- db/route_evidence_integration_test.go | 71 +++++++++++++++++++- docs/docs.go | 18 ++++- docs/swagger.json | 18 ++++- docs/swagger.yaml | 27 ++++++-- internal/api/handlers/route_evidence.go | 22 ++++-- internal/api/handlers/route_evidence_test.go | 25 +++++++ internal/api/route_evidence.go | 47 ++++++++++--- internal/api/route_evidence_test.go | 20 ++++++ 9 files changed, 259 insertions(+), 28 deletions(-) diff --git a/db/route_evidence.go b/db/route_evidence.go index e7017ea5..f31f50c0 100644 --- a/db/route_evidence.go +++ b/db/route_evidence.go @@ -39,6 +39,16 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap if c.IATA != iata || c.PathKey != key || !c.Since.Equal(query.Since) || !c.Until.Equal(query.Until) || c.ID <= 0 || c.HeardAt.Before(query.Since) || !c.HeardAt.Before(query.Until) { return nil, api.ErrRouteEvidenceInput } + if c.HashSize != 0 || c.PathBytes != "" { + if (query.HashSize != 0 || query.PathBytes != "") && (query.HashSize != c.HashSize || query.PathBytes != c.PathBytes) { + return nil, api.ErrRouteEvidenceInput + } + query.HashSize, query.PathBytes = c.HashSize, c.PathBytes + } + } + pinned := query.HashSize != 0 || query.PathBytes != "" + if pinned && !api.ValidRouteEvidencePath(query.HashSize, query.PathBytes) { + return nil, api.ErrRouteEvidenceInput } keyBytes, _ := hex.DecodeString(key) row, err := s.q.GetRouteEvidenceRoute(ctx, sqlc.GetRouteEvidenceRouteParams{Iata: iata, PathKey: keyBytes}) @@ -49,10 +59,35 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap if err != nil { return nil, err } + width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) + if pinned { + selected, _ := hex.DecodeString(query.PathBytes) + hops := len(selected) / int(query.HashSize) + if hops != len(row.NodeIds) || hops != int(row.HopCount) { + return nil, api.ErrRouteEvidenceInput + } + prefixes := make([][]byte, hops) + for i, id := range row.NodeIds { + prefixes[i] = selected[i*int(query.HashSize) : (i+1)*int(query.HashSize)] + // A changed representation must still belong to this node chain. + // Reuse the already-loaded keys; never widen the evidence query. + if !valid || width != query.HashSize || !bytes.Equal(path, selected) { + node := nodes[id] + if node == nil { + return nil, api.ErrRouteEvidenceInput + } + pubkey, err := hex.DecodeString(node.PublicKey) + if err != nil || !bytes.HasPrefix(pubkey, prefixes[i]) { + return nil, api.ErrRouteEvidenceInput + } + } + } + row.HashPrefix = prefixes + width, path, valid = query.HashSize, selected, true + } route := toKnownRoutes([]knownRouteRow{{ID: row.ID, NodeIds: row.NodeIds, HashPrefix: row.HashPrefix, Iata: row.Iata, HopCount: row.HopCount, FirstSeen: row.FirstSeen, LastSeen: row.LastSeen, ObservationCount: row.ObservationCount}}, nodes)[0] route.PathKey = hex.EncodeToString(row.PathKey) out := &api.RouteEvidence{Items: []api.RouteObservation{}, Route: route, WindowStart: query.Since.UnixMilli(), WindowEnd: query.Until.UnixMilli(), GeneratedAt: time.Now().UnixMilli(), MatchType: "saved_path_prefixes"} - width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) if !valid { return out, nil } @@ -86,7 +121,7 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap } if out.HasMore { last := rows[len(rows)-1] - cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID}).String() + cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID, HashSize: width, PathBytes: out.PathBytes}).String() out.NextPageCursor = &cursor } return out, nil diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index a84b534f..23abc34b 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -36,12 +36,13 @@ func TestRouteEvidencePostgres(t *testing.T) { applyBaseline(t, ctx, tx) _, err := tx.Exec(ctx, ` INSERT INTO observers(id,public_key,display_name) VALUES ('00000000-0000-0000-0000-000000000001','\x01','One'),('00000000-0000-0000-0000-000000000002','\x02','Two'); - INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa',2,'A'),('00000000-0000-0000-0000-000000000002','\xbb',2,'B'); + INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa0102',2,'A'),('00000000-0000-0000-0000-000000000002','\xbb0304',2,'B'); INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(1,10) i;`) if err != nil { t.Fatal(err) } + capture := &evidenceCapture{Tx: tx} store := &Store{q: sqlc.New(capture)} ids := []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000001"), uuid.MustParse("00000000-0000-0000-0000-000000000002")} @@ -80,6 +81,36 @@ func TestRouteEvidencePostgres(t *testing.T) { if err != nil { t.Fatal(err) } + if query.Cursor.HashSize != 1 || query.Cursor.PathBytes != "aabb" { + t.Fatalf("cursor did not pin the original bytes: %+v", query.Cursor) + } + // A new representation arrives while page one is open. New reads follow it, + // but pagination and copied links retain the exact original representation. + if _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(12,14) i`); err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,path_length_byte) + SELECT i,int4send(i),'00000000-0000-0000-0000-000000000001','YOW',$1,2,2,'\xaa01bb03',4,66 FROM generate_series(12,14) i`, at) + if err != nil { + t.Fatal(err) + } + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa, 1}, {0xbb, 3}}, "YOW", 2); err != nil { + t.Fatal(err) + } + freshQuery := api.RouteEvidenceQuery{Since: since, Until: until, Limit: 2} + fresh, err := store.GetRouteEvidence(ctx, "YOW", key, freshQuery) + if err != nil || fresh.HashSize != 2 || fresh.PathBytes != "aa01bb03" || len(fresh.Items) != 2 || fresh.Items[0].ID != 14 || fresh.Route.Hops[0].HashBytes != "aa01" { + t.Fatalf("new representation lost: %+v %v", fresh, err) + } + if fresh.Route.ID != first.Route.ID || fresh.Route.PathKey != first.Route.PathKey || fresh.Route.FirstSeen != first.Route.FirstSeen || fresh.Route.ObservationCount != first.Route.ObservationCount+1 { + t.Fatalf("route identity/counters changed: %+v", fresh.Route) + } + shared := freshQuery + shared.HashSize, shared.PathBytes = first.HashSize, first.PathBytes + copyPage, err := store.GetRouteEvidence(ctx, "YOW", key, shared) + if err != nil || len(copyPage.Items) != 2 || copyPage.Items[0].ID != 3 || copyPage.Route.Hops[0].HashBytes != "aa" { + t.Fatalf("shared representation changed: %+v %v", copyPage, err) + } second, err := store.GetRouteEvidence(ctx, "YOW", key, query) if err != nil { t.Fatal(err) @@ -90,6 +121,44 @@ func TestRouteEvidencePostgres(t *testing.T) { if _, err = json.Marshal(second); err != nil { t.Fatalf("non-finite JSON: %v", err) } + for _, prefixes := range [][][]byte{{{0xaa, 1, 2}, {0xbb, 3, 4}}, {{0xaa, 1, 2}, {0xbb, 3, 4}}} { + if err := store.UpsertKnownRoute(ctx, ids, prefixes, "YOW", 2); err != nil { + t.Fatal(err) + } + } + thirdWidth, err := store.GetRouteEvidence(ctx, "YOW", key, freshQuery) + if err != nil || thirdWidth.HashSize != 3 || thirdWidth.PathBytes != "aa0102bb0304" || len(thirdWidth.Items) != 0 || thirdWidth.Route.ID != first.Route.ID || thirdWidth.Route.ObservationCount != first.Route.ObservationCount+3 { + t.Fatalf("third width or repeated upsert: %+v %v", thirdWidth, err) + } + // Missing identities or mismatched bytes cannot make a pinned query search + // a different chain. Prefix collisions likewise never widen the SQL match. + wrong := shared + wrong.PathBytes = "aacc" + if _, err := store.GetRouteEvidence(ctx, "YOW", key, wrong); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatalf("unrelated path accepted: %v", err) + } + wrong.PathBytes = "aabbcc" + if _, err := store.GetRouteEvidence(ctx, "YOW", key, wrong); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatalf("wrong hop count accepted: %v", err) + } + _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000003','\xaa9988',2,'Collision'); + DELETE FROM nodes WHERE id='00000000-0000-0000-0000-000000000001'`) + if err != nil { + t.Fatal(err) + } + if _, err := store.GetRouteEvidence(ctx, "YOW", key, shared); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatalf("missing node substituted by colliding node: %v", err) + } + if _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa0102',2,'A')`); err != nil { + t.Fatal(err) + } + copyPage, err = store.GetRouteEvidence(ctx, "YOW", key, shared) + if err != nil || len(copyPage.Items) != 2 || copyPage.Items[0].ID != 3 { + t.Fatalf("collision broadened evidence: %+v %v", copyPage, err) + } + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } query.Cursor.IATA = "YVR" if _, err = store.GetRouteEvidence(ctx, "YOW", key, query); !errors.Is(err, api.ErrRouteEvidenceInput) { t.Fatal("cross-route cursor accepted") diff --git a/docs/docs.go b/docs/docs.go index e75b2879..22831d40 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2139,7 +2139,7 @@ const docTemplate = `{ }, "/routes/{iata}/{pathKey}/observations": { "get": { - "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.", "produces": [ "application/json" ], @@ -2180,9 +2180,23 @@ const docTemplate = `{ "name": "until", "in": "query" }, + { + "maximum": 3, + "minimum": 1, + "type": "integer", + "description": "Exact hash width (1–3); provide with pathBytes to pin a response's representation", + "name": "hashSize", + "in": "query" + }, + { + "type": "string", + "description": "Complete lower-case hex path from the response; provide with hashSize", + "name": "pathBytes", + "in": "query" + }, { "type": "string", - "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "description": "Opaque precise cursor from nextPageCursor; route, window and path are pinned", "name": "pageCursor", "in": "query" }, diff --git a/docs/swagger.json b/docs/swagger.json index b2ed0348..52c2c9d6 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2137,7 +2137,7 @@ }, "/routes/{iata}/{pathKey}/observations": { "get": { - "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.", "produces": [ "application/json" ], @@ -2178,9 +2178,23 @@ "name": "until", "in": "query" }, + { + "maximum": 3, + "minimum": 1, + "type": "integer", + "description": "Exact hash width (1–3); provide with pathBytes to pin a response's representation", + "name": "hashSize", + "in": "query" + }, + { + "type": "string", + "description": "Complete lower-case hex path from the response; provide with hashSize", + "name": "pathBytes", + "in": "query" + }, { "type": "string", - "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "description": "Opaque precise cursor from nextPageCursor; route, window and path are pinned", "name": "pageCursor", "in": "query" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9fb6cd9c..95e7171c 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2994,11 +2994,13 @@ paths: - Routes /routes/{iata}/{pathKey}/observations: get: - description: Matches the saved IATA, complete path bytes and hash width. Excludes - TRACE and unclassified reports; other widths and search-result subsegments - are not included. Byte matches do not prove hop identities or delivery. The - route and its historical counter can outlive raw evidence. Cursors retain - full timestamp precision and pin route/window scope. + description: Matches one complete path representation in the saved IATA. Defaults + to the latest processed prefixes; hashSize/pathBytes pin a prior response. + Excludes TRACE and unclassified reports; widths are never combined. Pinned + prefixes must match the saved node chain or its current metadata. Byte matches + do not prove hop identities or delivery. The route counter can outlive raw + evidence. New cursors pin path, route, window and full timestamp precision; + legacy unpinned cursors remain supported. parameters: - description: Three-character IATA code in: path @@ -3024,7 +3026,20 @@ paths: in: query name: until type: integer - - description: Opaque precise cursor from nextPageCursor; window is pinned + - description: Exact hash width (1–3); provide with pathBytes to pin a response's + representation + in: query + maximum: 3 + minimum: 1 + name: hashSize + type: integer + - description: Complete lower-case hex path from the response; provide with + hashSize + in: query + name: pathBytes + type: string + - description: Opaque precise cursor from nextPageCursor; route, window and + path are pinned in: query name: pageCursor type: string diff --git a/internal/api/handlers/route_evidence.go b/internal/api/handlers/route_evidence.go index a194306c..3819cbec 100644 --- a/internal/api/handlers/route_evidence.go +++ b/internal/api/handlers/route_evidence.go @@ -17,7 +17,7 @@ import ( func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.RouteEvidenceQuery, error) { q := api.RouteEvidenceQuery{} values := r.URL.Query() - for _, name := range []string{"pageCursor", "since", "until", "range", "limit"} { + for _, name := range []string{"pageCursor", "since", "until", "range", "limit", "hashSize", "pathBytes"} { if len(values[name]) > 1 { return q, api.ErrRouteEvidenceInput } @@ -38,6 +38,7 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R q.Cursor = c q.Since = c.Since q.Until = c.Until + q.HashSize, q.PathBytes = c.HashSize, c.PathBytes } else { q.Until = now.UTC().Truncate(time.Millisecond) q.Since = q.Until.Add(-24 * time.Hour) @@ -67,6 +68,17 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R return q, api.ErrRouteEvidenceInput } } + if values.Has("hashSize") || values.Has("pathBytes") { + width, err := strconv.ParseInt(values.Get("hashSize"), 10, 16) + path := values.Get("pathBytes") + if err != nil || !api.ValidRouteEvidencePath(int16(width), path) || strconv.FormatInt(width, 10) != values.Get("hashSize") { + return q, api.ErrRouteEvidenceInput + } + if q.Cursor != nil && q.Cursor.HashSize != 0 && (q.Cursor.HashSize != int16(width) || q.Cursor.PathBytes != path) { + return q, api.ErrRouteEvidenceInput + } + q.HashSize, q.PathBytes = int16(width), path + } if !api.ValidRouteEvidenceWindow(q.Since, q.Until) || q.Until.After(now.Add(clockSkewTolerance)) { return q, api.ErrRouteEvidenceInput } @@ -76,7 +88,7 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R // getRouteEvidence godoc // // @Summary Get retained reports matching a full saved route prefix sequence -// @Description Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope. +// @Description Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported. // @Tags Routes // @Produce json // @Param iata path string true "Three-character IATA code" @@ -84,7 +96,9 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R // @Param range query string false "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor" // @Param since query int false "Window start epoch ms; provide with until (default last 24h)" // @Param until query int false "Exclusive window end epoch ms; maximum span 30d, end may be up to 5 minutes ahead of server time (clock skew tolerance)" -// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; window is pinned" +// @Param hashSize query int false "Exact hash width (1–3); provide with pathBytes to pin a response's representation" minimum(1) maximum(3) +// @Param pathBytes query string false "Complete lower-case hex path from the response; provide with hashSize" +// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; route, window and path are pinned" // @Param limit query int false "Default 50; positive, capped at 200" minimum(1) maximum(200) // @Success 200 {object} api.RouteEvidence // @Failure 400 {object} handlers.APIError @@ -100,7 +114,7 @@ func getRouteEvidence(reader api.Reader) http.HandlerFunc { } query, err := routeEvidenceQuery(r, iata, key, time.Now()) if err != nil { - respondError(w, 400, "invalid route evidence window, limit or page cursor") + respondError(w, 400, "invalid route evidence window, path, limit or page cursor") return } result, err := reader.GetRouteEvidence(r.Context(), iata, key, query) diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go index f2268d86..88f057df 100644 --- a/internal/api/handlers/route_evidence_test.go +++ b/internal/api/handlers/route_evidence_test.go @@ -85,3 +85,28 @@ func TestRouteEvidenceHandler(t *testing.T) { } } } + +func TestRouteEvidencePinnedQuery(t *testing.T) { + now := time.Now().UTC().Truncate(time.Millisecond) + q, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?hashSize=2&pathBytes=aa01bb02", nil), "YOW", evidenceKey, now) + if err != nil || q.HashSize != 2 || q.PathBytes != "aa01bb02" { + t.Fatalf("selected path: %+v %v", q, err) + } + c := api.RouteEvidenceCursor{IATA: "YOW", PathKey: evidenceKey, Since: q.Since, Until: q.Until, HeardAt: q.Since.Add(time.Hour), ID: 5, HashSize: q.HashSize, PathBytes: q.PathBytes} + encoded := url.QueryEscape(c.String()) + next, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+encoded, nil), "YOW", evidenceKey, now) + if err != nil || next.HashSize != 2 || next.PathBytes != q.PathBytes { + t.Fatalf("cursor path lost: %+v %v", next, err) + } + for _, query := range []string{ + "hashSize=1", "pathBytes=aabb", "hashSize=0&pathBytes=aabb", "hashSize=4&pathBytes=aa01bb02", + "hashSize=01&pathBytes=aabb", "hashSize=1&hashSize=2&pathBytes=aabb", + "hashSize=1&pathBytes=aabb&pathBytes=aabb", "hashSize=1&pathBytes=aa", "hashSize=1&pathBytes=AABB", + "pageCursor=" + encoded + "&hashSize=1&pathBytes=aabb", + "pageCursor=" + encoded + "&hashSize=2&pathBytes=aa02bb01", + } { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } +} diff --git a/internal/api/route_evidence.go b/internal/api/route_evidence.go index d320ade5..490c91b2 100644 --- a/internal/api/route_evidence.go +++ b/internal/api/route_evidence.go @@ -50,16 +50,22 @@ type RouteEvidenceQuery struct { Until time.Time Cursor *RouteEvidenceCursor Limit int32 + // Optional exact representation copied from a prior response or cursor. + HashSize int16 + PathBytes string } -// RouteEvidenceCursor binds the precise (heard_at DESC,id DESC) boundary to its route/window. +// RouteEvidenceCursor pins the route, window and exact representation as well as +// the precise (heard_at DESC,id DESC) boundary. Legacy v1 cursors have no path pin. type RouteEvidenceCursor struct { - IATA string - PathKey string - Since time.Time - Until time.Time - HeardAt time.Time - ID int64 + IATA string + PathKey string + Since time.Time + Until time.Time + HeardAt time.Time + ID int64 + HashSize int16 + PathBytes string } func ValidRouteEvidenceKey(iata, key string) bool { @@ -79,20 +85,32 @@ func ValidRouteEvidenceWindow(since, until time.Time) bool { return !since.Before(time.Unix(0, 0)) && until.After(since) && until.Sub(since) <= MaxRouteEvidenceWindow } +func ValidRouteEvidencePath(width int16, path string) bool { + if width < 1 || width > 3 || len(path) < 4*int(width) || len(path) > 126*int(width) || len(path)%(2*int(width)) != 0 { + return false + } + _, err := hex.DecodeString(path) + return err == nil && path == strings.ToLower(path) +} + func (c RouteEvidenceCursor) String() string { - return fmt.Sprintf("v1:%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) + value := fmt.Sprintf("%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) + if c.HashSize != 0 || c.PathBytes != "" { + return fmt.Sprintf("v2:%s:%d:%s", value, c.HashSize, c.PathBytes) + } + return "v1:" + value } func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { - if len(raw) > 200 { + if len(raw) > 512 { return nil, ErrRouteEvidenceInput } p := strings.Split(raw, ":") - if len(p) != 7 || p[0] != "v1" || !ValidRouteEvidenceKey(p[1], p[2]) { + if !((len(p) == 7 && p[0] == "v1") || (len(p) == 9 && p[0] == "v2")) || !ValidRouteEvidenceKey(p[1], p[2]) { return nil, ErrRouteEvidenceInput } values := [4]int64{} - for i, s := range p[3:] { + for i, s := range p[3:7] { n, err := strconv.ParseInt(s, 10, 64) if err != nil || n < 0 || strconv.FormatInt(n, 10) != s { return nil, ErrRouteEvidenceInput @@ -104,6 +122,13 @@ func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { return nil, ErrRouteEvidenceInput } c := &RouteEvidenceCursor{IATA: p[1], PathKey: p[2], Since: time.UnixMilli(values[0]).UTC(), Until: time.UnixMilli(values[1]).UTC(), HeardAt: time.UnixMicro(values[2]).UTC(), ID: values[3]} + if p[0] == "v2" { + width, err := strconv.ParseInt(p[7], 10, 16) + if err != nil || strconv.FormatInt(width, 10) != p[7] || !ValidRouteEvidencePath(int16(width), p[8]) { + return nil, ErrRouteEvidenceInput + } + c.HashSize, c.PathBytes = int16(width), p[8] + } if !ValidRouteEvidenceWindow(c.Since, c.Until) || c.HeardAt.Before(c.Since) || !c.HeardAt.Before(c.Until) { return nil, ErrRouteEvidenceInput } diff --git a/internal/api/route_evidence_test.go b/internal/api/route_evidence_test.go index 42c9a73a..a025a440 100644 --- a/internal/api/route_evidence_test.go +++ b/internal/api/route_evidence_test.go @@ -23,3 +23,23 @@ func TestRouteEvidenceCursor(t *testing.T) { } } } + +func TestRouteEvidencePinnedCursor(t *testing.T) { + c := RouteEvidenceCursor{IATA: "YOW", PathKey: strings.Repeat("a", 32), Since: time.UnixMilli(1000).UTC(), Until: time.UnixMilli(3000).UTC(), HeardAt: time.UnixMicro(2000123).UTC(), ID: 9, HashSize: 3, PathBytes: strings.Repeat("aabbcc", 63)} + got, err := ParseRouteEvidenceCursor(c.String()) + if err != nil || *got != c || !strings.HasPrefix(c.String(), "v2:") { + t.Fatalf("pinned cursor round-trip: %+v %v", got, err) + } + for _, bad := range []string{ + strings.Replace(c.String(), ":3:", ":4:", 1), + strings.Replace(c.String(), ":3:", ":03:", 1), + c.String() + "00", + strings.TrimSuffix(c.String(), c.PathBytes) + "aabbcc", + strings.ToUpper(c.String()), + strings.TrimSuffix(c.String(), c.PathBytes) + "zzzzzzzzzzzz", + } { + if _, err := ParseRouteEvidenceCursor(bad); err == nil { + t.Errorf("accepted invalid pinned cursor %q", bad) + } + } +} From 52759a2149d54cd0550df2584affe576b72430f4 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:06:00 -0400 Subject: [PATCH 10/23] chore: add .github/CODEOWNERS for main branch policy Signed-off-by: MrAlders0n --- .github/CODEOWNERS | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b8732e9d..d369ebd2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,2 @@ -# Default owners for everything +# Every change to main needs a review from the repo owner * @MrAlders0n - - -# EXAMPLE Specific ownership by directory -#/internal/ingest/ @your-github-username -#/db/ @your-github-username From c7209b70433b8b127a5b1062fdfb17d4a676245c Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:06:01 -0400 Subject: [PATCH 11/23] chore: add .github/workflows/main-source-guard.yml for main branch policy Signed-off-by: MrAlders0n --- .github/workflows/main-source-guard.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/main-source-guard.yml diff --git a/.github/workflows/main-source-guard.yml b/.github/workflows/main-source-guard.yml new file mode 100644 index 00000000..911186f5 --- /dev/null +++ b/.github/workflows/main-source-guard.yml @@ -0,0 +1,24 @@ +name: Main PR source guard + +# Runs from main's copy of this file, so a PR can't edit the check away. +on: + pull_request_target: + branches: [main] + types: [opened, reopened, synchronize, edited] + +permissions: {} + +jobs: + main-source-is-dev: + name: main-source-is-dev + runs-on: ubuntu-latest + steps: + - name: Require PR from this repo's dev branch + env: + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + run: | + if [ "$HEAD_REF" != "dev" ] || [ "$HEAD_REPO" != "$GITHUB_REPOSITORY" ]; then + echo "::error::PRs into main must come from $GITHUB_REPOSITORY:dev (got $HEAD_REPO:$HEAD_REF)" + exit 1 + fi From c554238485e24ce6b2cd3db200dd86c967414088 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 3 Oct 2026 23:58:00 -0400 Subject: [PATCH 12/23] fix(traces): validate trace frames and preserve ambiguous identities --- db/packets.go | 23 +++- db/queries/queries.sql | 21 ++- db/sqlc/mock/querier.go | 17 ++- db/sqlc/querier.go | 3 + db/sqlc/queries.sql.go | 109 +++++++++++++--- db/trace_quality_integration_test.go | 77 +++++++++++ db/traces.go | 179 ++++++++++++++++++-------- db/traces_integration_test.go | 13 +- db/traces_test.go | 13 +- docs/docs.go | 31 +++++ docs/swagger.json | 31 +++++ docs/swagger.yaml | 21 +++ internal/api/packets.go | 31 +++-- internal/api/traces.go | 36 +++--- internal/ingest/ingest.go | 1 + internal/ingest/ingest_test.go | 3 + internal/ingest/packet.go | 58 +++++++-- internal/ingest/trace_quality_test.go | 75 +++++++++++ internal/tracequality/quality.go | 96 ++++++++++++++ internal/tracequality/quality_test.go | 51 ++++++++ 20 files changed, 763 insertions(+), 126 deletions(-) create mode 100644 db/trace_quality_integration_test.go create mode 100644 internal/ingest/trace_quality_test.go create mode 100644 internal/tracequality/quality.go create mode 100644 internal/tracequality/quality_test.go diff --git a/db/packets.go b/db/packets.go index 620acceb..a142ab6b 100644 --- a/db/packets.go +++ b/db/packets.go @@ -15,6 +15,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" "github.com/meshcore-go/meshcore-go" @@ -406,7 +407,15 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, // compute it once rather than per observation. var traceRawHashes [][]byte if row.PayloadType == int16(meshcore.PayloadTypeTrace) { - if trace, err := meshcore.TraceFromBytes(row.RawPayload); err == nil { + quality := tracequality.Assess(row.RawPayload, int(row.PayloadVersion), int(row.RouteType), -1) + for _, obs := range obsRows { + q := tracequality.Assess(row.RawPayload, int(row.PayloadVersion), int(row.RouteType), int(obs.PathLengthByte)) + for _, reason := range q.Reasons { + quality.Add(reason) + } + } + p.TraceQuality = &quality + if trace, err := meshcore.TraceFromBytes(row.RawPayload); err == nil && quality.Status == "supported" { hashSize := int(trace.PathHashSize()) for i := 0; i+hashSize <= len(trace.PathHashes); i += hashSize { traceRawHashes = append(traceRawHashes, trace.PathHashes[i:i+hashSize]) @@ -442,8 +451,11 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, obs.PropagationTimeMs = &prop resolvedPath := []api.ResolvedHop{} if row.PayloadType == int16(meshcore.PayloadTypeTrace) { + rawPath := hex.EncodeToString(v.PathBytes) + obs.WirePathBytes = &rawPath + obs.TraceQuality = p.TraceQuality if len(traceRawHashes) > 0 { - resolved, err := s.ResolvePathHashes(ctx, v.Iata, traceRawHashes) + resolved, err := s.ResolveTracePathHashes(ctx, v.Iata, traceRawHashes) if err != nil { slog.Error(fmt.Sprintf("store: path resolution failed for observation %d", v.ID), "component", "db", "error", err) } else { @@ -491,7 +503,7 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, } p.Observations = append(p.Observations, obs) } - if row.PayloadType == 9 && len(obsRows) > 0 { + if row.PayloadType == 9 && len(obsRows) > 0 && p.TraceQuality.Status == "supported" { iatas := make([]string, 0, len(obsRows)) seen := make(map[string]struct{}) for _, v := range obsRows { @@ -503,6 +515,11 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, var parsed tracePayload if err := json.Unmarshal(row.ParsedPayload, &parsed); err == nil { p.ResolvedRoute = s.resolveTraceRoute(ctx, &parsed, iatas) + for _, hop := range p.ResolvedRoute { + if hop.Confidence == "ambiguous" { + p.TraceQuality.Add("ambiguous_prefix") + } + } } } return p, nil diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 2a2c470b..9f513b13 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -455,6 +455,8 @@ WHERE p.packet_hash = $1; -- Return distinct observation IATAs in first-heard order for path resolution, -- without fetching full observations separately for every trace packet. SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, + p.raw_payload, p.payload_version, + EXISTS(SELECT 1 FROM packet_observations po WHERE po.packet_hash=p.packet_hash AND po.path_length_byte>=64) AS invalid_snr_path, p.route_type, p.first_heard_at, p.last_heard_at, @@ -1422,7 +1424,11 @@ SELECT WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, COALESCE(t.trace_type, '')::text AS trace_type, - t.best_payload + t.best_payload, + ARRAY(SELECT ti.iata FROM trace_iatas ti WHERE ti.trace_tag=t.trace_tag)::bpchar[] AS iatas, + EXISTS(SELECT 1 FROM packets p WHERE p.trace_tag=t.trace_tag AND p.payload_version<>0) AS unsupported_version, + EXISTS(SELECT 1 FROM packets p WHERE p.trace_tag=t.trace_tag AND p.route_type NOT IN (2,3)) AS non_direct, + EXISTS(SELECT 1 FROM packets p JOIN packet_observations po ON po.packet_hash=p.packet_hash WHERE p.trace_tag=t.trace_tag AND po.path_length_byte>=64) AS invalid_snr_path FROM trace_tags t WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) @@ -1834,3 +1840,16 @@ ON CONFLICT (country) DO UPDATE SET -- name: PruneZoneBoundaries :many -- Drops imports for IATAs no longer configured, so their manual border returns. DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY(@keep::text[])) RETURNING iata; + +-- name: ResolveTraceHashes :many +-- TRACE can terminate at companions. Preserve the full prefix (including eight +-- bytes), and return every candidate across heard regions instead of picking one. +SELECT h.hash::bytea AS hash, si.iata, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM unnest(@hashes::bytea[]) AS h(hash) +JOIN node_short_ids si ON si.iata = ANY(@iatas::bpchar[]) AND + ((octet_length(h.hash)=1 AND si.prefix_1=h.hash) OR + (octet_length(h.hash)=2 AND si.prefix_2=h.hash) OR + (octet_length(h.hash) IN (4,8) AND si.prefix_4=substring(h.hash FROM 1 FOR 4))) +JOIN nodes n ON n.id=si.node_id +WHERE substring(n.public_key FROM 1 FOR octet_length(h.hash))=h.hash +ORDER BY si.iata, h.hash, n.id; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index b34664f2..6813cc5e 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -source db/sqlc/querier.go -destination db/sqlc/mock/querier.go -package mockdb +// mockgen -source=db/sqlc/querier.go -destination=db/sqlc/mock/querier.go -package=mockdb // // Package mockdb is a generated GoMock package. @@ -1896,6 +1896,21 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } +// ResolveTraceHashes mocks base method. +func (m *MockQuerier) ResolveTraceHashes(ctx context.Context, arg db.ResolveTraceHashesParams) ([]db.ResolveTraceHashesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveTraceHashes", ctx, arg) + ret0, _ := ret[0].([]db.ResolveTraceHashesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveTraceHashes indicates an expected call of ResolveTraceHashes. +func (mr *MockQuerierMockRecorder) ResolveTraceHashes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveTraceHashes", reflect.TypeOf((*MockQuerier)(nil).ResolveTraceHashes), ctx, arg) +} + // RollAdvertHearings mocks base method. func (m *MockQuerier) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index f3c54d3c..5912fa44 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -314,6 +314,9 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + // TRACE can terminate at companions. Preserve the full prefix (including eight + // bytes), and return every candidate across heard regions instead of picking one. + ResolveTraceHashes(ctx context.Context, arg ResolveTraceHashesParams) ([]ResolveTraceHashesRow, error) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error // Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 46b13b50..92a6a176 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1405,6 +1405,8 @@ func (q *Queries) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPa const getPacketsByTraceTag = `-- name: GetPacketsByTraceTag :many SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, + p.raw_payload, p.payload_version, + EXISTS(SELECT 1 FROM packet_observations po WHERE po.packet_hash=p.packet_hash AND po.path_length_byte>=64) AS invalid_snr_path, p.route_type, p.first_heard_at, p.last_heard_at, @@ -1425,14 +1427,17 @@ ORDER BY p.first_heard_at ASC ` type GetPacketsByTraceTagRow struct { - PacketHashHex string `json:"packet_hash_hex"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ParsedPayload []byte `json:"parsed_payload"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - Iatas []string `json:"iatas"` + PacketHashHex string `json:"packet_hash_hex"` + RawPayload []byte `json:"raw_payload"` + PayloadVersion int16 `json:"payload_version"` + InvalidSnrPath bool `json:"invalid_snr_path"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ParsedPayload []byte `json:"parsed_payload"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + Iatas []string `json:"iatas"` } // Return distinct observation IATAs in first-heard order for path resolution, @@ -1448,6 +1453,9 @@ func (q *Queries) GetPacketsByTraceTag(ctx context.Context, decode string) ([]Ge var i GetPacketsByTraceTagRow if err := rows.Scan( &i.PacketHashHex, + &i.RawPayload, + &i.PayloadVersion, + &i.InvalidSnrPath, &i.RouteType, &i.FirstHeardAt, &i.LastHeardAt, @@ -4137,7 +4145,11 @@ SELECT WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, COALESCE(t.trace_type, '')::text AS trace_type, - t.best_payload + t.best_payload, + ARRAY(SELECT ti.iata FROM trace_iatas ti WHERE ti.trace_tag=t.trace_tag)::bpchar[] AS iatas, + EXISTS(SELECT 1 FROM packets p WHERE p.trace_tag=t.trace_tag AND p.payload_version<>0) AS unsupported_version, + EXISTS(SELECT 1 FROM packets p WHERE p.trace_tag=t.trace_tag AND p.route_type NOT IN (2,3)) AS non_direct, + EXISTS(SELECT 1 FROM packets p JOIN packet_observations po ON po.packet_hash=p.packet_hash WHERE p.trace_tag=t.trace_tag AND po.path_length_byte>=64) AS invalid_snr_path FROM trace_tags t WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) @@ -4165,13 +4177,17 @@ type ListTraceTagsParams struct { } type ListTraceTagsRow struct { - TraceTag string `json:"trace_tag"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - PacketCount int64 `json:"packet_count"` - IataCount int64 `json:"iata_count"` - TraceType string `json:"trace_type"` - BestPayload []byte `json:"best_payload"` + TraceTag string `json:"trace_tag"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + PacketCount int64 `json:"packet_count"` + IataCount int64 `json:"iata_count"` + TraceType string `json:"trace_type"` + BestPayload []byte `json:"best_payload"` + Iatas []string `json:"iatas"` + UnsupportedVersion bool `json:"unsupported_version"` + NonDirect bool `json:"non_direct"` + InvalidSnrPath bool `json:"invalid_snr_path"` } // ============================================================ @@ -4204,6 +4220,10 @@ func (q *Queries) ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([ &i.IataCount, &i.TraceType, &i.BestPayload, + &i.Iatas, + &i.UnsupportedVersion, + &i.NonDirect, + &i.InvalidSnrPath, ); err != nil { return nil, err } @@ -4856,6 +4876,63 @@ func (q *Queries) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashes return items, nil } +const resolveTraceHashes = `-- name: ResolveTraceHashes :many +SELECT h.hash::bytea AS hash, si.iata, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM unnest($1::bytea[]) AS h(hash) +JOIN node_short_ids si ON si.iata = ANY($2::bpchar[]) AND + ((octet_length(h.hash)=1 AND si.prefix_1=h.hash) OR + (octet_length(h.hash)=2 AND si.prefix_2=h.hash) OR + (octet_length(h.hash) IN (4,8) AND si.prefix_4=substring(h.hash FROM 1 FOR 4))) +JOIN nodes n ON n.id=si.node_id +WHERE substring(n.public_key FROM 1 FOR octet_length(h.hash))=h.hash +ORDER BY si.iata, h.hash, n.id +` + +type ResolveTraceHashesParams struct { + Hashes [][]byte `json:"hashes"` + Iatas []string `json:"iatas"` +} + +type ResolveTraceHashesRow struct { + Hash []byte `json:"hash"` + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` + PublicKey []byte `json:"public_key"` +} + +// TRACE can terminate at companions. Preserve the full prefix (including eight +// bytes), and return every candidate across heard regions instead of picking one. +func (q *Queries) ResolveTraceHashes(ctx context.Context, arg ResolveTraceHashesParams) ([]ResolveTraceHashesRow, error) { + rows, err := q.db.Query(ctx, resolveTraceHashes, arg.Hashes, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ResolveTraceHashesRow{} + for rows.Next() { + var i ResolveTraceHashesRow + if err := rows.Scan( + &i.Hash, + &i.Iata, + &i.NodeID, + &i.Name, + &i.Latitude, + &i.Longitude, + &i.PublicKey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const saveChannelCatalogue = `-- name: SaveChannelCatalogue :exec INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) VALUES ($1, $2, $3::jsonb, $4::text, diff --git a/db/trace_quality_integration_test.go b/db/trace_quality_integration_test.go new file mode 100644 index 00000000..d623ab6d --- /dev/null +++ b/db/trace_quality_integration_test.go @@ -0,0 +1,77 @@ +package db + +import ( + "context" + "encoding/hex" + "os" + "testing" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +func TestTraceQualityPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx := context.Background() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(ctx) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + for _, name := range []string{"nodes", "node_short_ids"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+name+" (LIKE public."+name+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` + INSERT INTO nodes (id,public_key,node_type,name) VALUES + ('00000000-0000-0000-0000-000000000001',decode('aabbccdd11223344'||repeat('00',24),'hex'),1,'Companion'), + ('00000000-0000-0000-0000-000000000002',decode('aabbccdd55667788'||repeat('00',24),'hex'),2,'Repeater'); + INSERT INTO node_short_ids (node_id,iata,prefix_4) VALUES + ('00000000-0000-0000-0000-000000000001','YKF','\xaabbccdd'), + ('00000000-0000-0000-0000-000000000002','YKF','\xaabbccdd'), + ('00000000-0000-0000-0000-000000000001','YHM','\xaabbccdd');`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + for _, width := range []int{1, 2, 4, 8} { + full := "aabbccdd11223344" + h := full[:width*2] + b, _ := hex.DecodeString(h) + candidates, err := store.traceCandidates(ctx, []string{"YKF", "YHM"}, [][]byte{b}) + if err != nil { + t.Fatal(err) + } + flags := byte(0) + switch width { + case 2: + flags = 1 + case 4: + flags = 2 + case 8: + flags = 3 + } + route := traceRoute(&tracePayload{Flags: flags, PathHashes: []string{h}}, []string{"YKF", "YHM"}, candidates) + count := 2 + confidence := "ambiguous" + if width == 8 { + count = 1 + confidence = "high" + } + if len(route) != 1 || len(route[0].Nodes) != count || route[0].Confidence != confidence { + t.Fatalf("width %d: %+v", width, route) + } + if route[0].Nodes[0].PublicKey[:16] != "aabbccdd11223344" { + t.Fatal("companion excluded or eight-byte hash truncated") + } + } +} diff --git a/db/traces.go b/db/traces.go index 93ed18cd..cc933b11 100644 --- a/db/traces.go +++ b/db/traces.go @@ -8,6 +8,8 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" + "github.com/google/uuid" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -17,9 +19,11 @@ import ( ) type tracePayload struct { - PathHashes []string `json:"pathHashes"` - Flags byte `json:"flags"` - SNRValues []float32 `json:"snrValues"` + Raw string `json:"raw"` + Quality tracequality.Quality `json:"quality"` + PathHashes []string `json:"pathHashes"` + Flags byte `json:"flags"` + SNRValues []float32 `json:"snrValues"` } func (s *Store) RecordTrace(ctx context.Context, h ingest.TraceHearing) error { @@ -71,12 +75,41 @@ func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceT return nil, err } items := make([]api.TraceTagSummary, 0, len(rows)) + payloads := make([]tracePayload, 0, len(rows)) + allIATAs := map[string]bool{} + allHashes := map[string][]byte{} for _, r := range rows { var best tracePayload if len(r.BestPayload) > 0 { _ = json.Unmarshal(r.BestPayload, &best) } + quality := tracequality.Stored(best.Raw, best.Flags, best.PathHashes, best.SNRValues) + for _, reason := range best.Quality.Reasons { + if reason != "ambiguous_prefix" { + quality.Add(reason) + } + } + if r.UnsupportedVersion { + quality.Add("unsupported_version") + } + if r.InvalidSnrPath { + quality.Add("invalid_snr_path") + } + if r.NonDirect { + quality.Add("non_direct_trace") + } + payloads = append(payloads, best) + if quality.Status == "supported" { + for _, iata := range r.Iatas { + allIATAs[iata] = true + } + for _, h := range best.PathHashes { + b, _ := hex.DecodeString(h) + allHashes[h] = b + } + } items = append(items, api.TraceTagSummary{ + Quality: quality, TraceTag: r.TraceTag, FirstHeardAt: r.FirstHeardAt.Time.UnixMilli(), LastHeardAt: r.LastHeardAt.Time.UnixMilli(), @@ -87,6 +120,30 @@ func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceT SNRValues: best.SNRValues, }) } + // One bounded candidate query for the entire page, including all hash widths. + regions := make([]string, 0, len(allIATAs)) + for iata := range allIATAs { + regions = append(regions, iata) + } + hashes := make([][]byte, 0, len(allHashes)) + for _, h := range allHashes { + hashes = append(hashes, h) + } + candidates, err := s.traceCandidates(ctx, regions, hashes) + if err != nil { + return nil, err + } + for i := range items { + if items[i].Quality.Status != "supported" { + continue + } + for _, hop := range traceRoute(&payloads[i], rows[i].Iatas, candidates) { + if hop.Confidence == "ambiguous" { + items[i].Quality.Add("ambiguous_prefix") + break + } + } + } return items, nil } @@ -125,76 +182,94 @@ func (s *Store) GetTraceByTag(ctx context.Context, tag string) (*api.TraceDetail } packet.RawPath = rawPath } - if len(r.Iatas) > 0 { + packet.Quality = tracequality.Assess(r.RawPayload, int(r.PayloadVersion), int(r.RouteType), len(parsed.SNRValues)) + if r.InvalidSnrPath { + packet.Quality.Add("invalid_snr_path") + } + stored := tracequality.Stored(parsed.Raw, parsed.Flags, parsed.PathHashes, parsed.SNRValues) + for _, reason := range stored.Reasons { + packet.Quality.Add(reason) + } + if len(r.Iatas) > 0 && packet.Quality.Status == "supported" { packet.ResolvedRoute = s.resolveTraceRoute(ctx, &parsed, r.Iatas) } + for _, hop := range packet.ResolvedRoute { + if hop.Confidence == "ambiguous" { + packet.Quality.Add("ambiguous_prefix") + } + } detail.Packets = append(detail.Packets, packet) } return detail, nil } +// Candidates remain separated by hearing region until a trace is assembled. +// Never replace an ambiguous region with a convenient singleton from another. +type traceCandidateMap map[string]map[string][]api.ResolvedPathEntry + +func (s *Store) traceCandidates(ctx context.Context, iatas []string, hashes [][]byte) (traceCandidateMap, error) { + out := traceCandidateMap{} + if len(iatas) == 0 || len(hashes) == 0 { + return out, nil + } + rows, err := s.q.ResolveTraceHashes(ctx, sqlc.ResolveTraceHashesParams{Iatas: iatas, Hashes: hashes}) + if err != nil { + return nil, err + } + for _, r := range rows { + if out[r.Iata] == nil { + out[r.Iata] = map[string][]api.ResolvedPathEntry{} + } + h := hex.EncodeToString(r.Hash) + out[r.Iata][h] = append(out[r.Iata][h], api.ResolvedPathEntry{NodeID: r.NodeID, Name: r.Name, Latitude: r.Latitude, Longitude: r.Longitude, PublicKey: r.PublicKey}) + } + return out, nil +} + +func (s *Store) ResolveTracePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + candidates, err := s.traceCandidates(ctx, []string{iata}, hashes) + return candidates[iata], err +} + func (s *Store) resolveTraceRoute(ctx context.Context, payload *tracePayload, iatas []string) []api.ResolvedHop { - if payload == nil || len(payload.PathHashes) == 0 { + if payload == nil || tracequality.Stored(payload.Raw, payload.Flags, payload.PathHashes, payload.SNRValues).Status != "supported" { return nil } - hashSize := int(1 << (payload.Flags & 0x03)) hashes := make([][]byte, 0, len(payload.PathHashes)) for _, h := range payload.PathHashes { - b, err := hex.DecodeString(h) - if err == nil { - hashes = append(hashes, b) - } - } - confidenceRank := map[string]int{"none": 0, "ambiguous": 1, "high": 2} - type hopResult struct { - confidence string - entries []api.ResolvedPathEntry + b, _ := hex.DecodeString(h) + hashes = append(hashes, b) } - merged := make([]hopResult, len(hashes)) - for i := range merged { - merged[i] = hopResult{confidence: "none"} + candidates, err := s.traceCandidates(ctx, iatas, hashes) + if err != nil { + return nil } - for _, iata := range iatas { - resolved, err := s.ResolvePathHashes(ctx, iata, hashes) - if err != nil { - continue - } - for i, hash := range hashes { - key := hex.EncodeToString(hash[:hashSize]) - entries := resolved[key] - var confidence string - switch len(entries) { - case 0: - confidence = "none" - case 1: - confidence = "high" - default: - confidence = "ambiguous" - } - if confidenceRank[confidence] > confidenceRank[merged[i].confidence] { - merged[i] = hopResult{confidence: confidence, entries: entries} + return traceRoute(payload, iatas, candidates) +} + +func traceRoute(payload *tracePayload, iatas []string, candidates traceCandidateMap) []api.ResolvedHop { + route := make([]api.ResolvedHop, 0, len(payload.PathHashes)) + for i, h := range payload.PathHashes { + hop := api.ResolvedHop{Confidence: "none", Nodes: []api.ResolvedNode{}} + seen := map[uuid.UUID]bool{} + for _, iata := range iatas { + for _, e := range candidates[iata][h] { + if seen[e.NodeID] { + continue + } + seen[e.NodeID] = true + hop.Nodes = append(hop.Nodes, api.ResolvedNode{ID: e.NodeID, Name: e.Name, Latitude: e.Latitude, Longitude: e.Longitude, PublicKey: hex.EncodeToString(e.PublicKey)}) } } - } - route := make([]api.ResolvedHop, 0, len(hashes)) - for i, hr := range merged { - hop := api.ResolvedHop{ - Confidence: hr.confidence, - Nodes: make([]api.ResolvedNode, 0, len(hr.entries)), + if len(hop.Nodes) == 1 { + hop.Confidence = "high" + } else if len(hop.Nodes) > 1 { + hop.Confidence = "ambiguous" } if i < len(payload.SNRValues) { snr := payload.SNRValues[i] hop.SNR = &snr } - for _, e := range hr.entries { - hop.Nodes = append(hop.Nodes, api.ResolvedNode{ - ID: e.NodeID, - Name: e.Name, - Latitude: e.Latitude, - Longitude: e.Longitude, - PublicKey: hex.EncodeToString(e.PublicKey), - }) - } route = append(route, hop) } return route diff --git a/db/traces_integration_test.go b/db/traces_integration_test.go index 51305443..1d671afb 100644 --- a/db/traces_integration_test.go +++ b/db/traces_integration_test.go @@ -59,7 +59,7 @@ INSERT INTO node_short_ids (node_id,iata,prefix_4) VALUES ('00000000-0000-0000-0000-000000000001','YVR','\xaa000000'), ('00000000-0000-0000-0000-000000000002','YYJ','\xaa010000'); INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,trace_tag,parsed_payload,first_heard_at,last_heard_at) -SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,1,'\x00','\x00','\x01020304', +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,2,'\x010203040000000000aa','\x26','\x01020304', CASE WHEN i=100 THEN '{}'::jsonb ELSE '{"flags":0,"pathHashes":["aa"],"snrValues":[12.5]}'::jsonb END, '2026-01-01 00:00:00+00'::timestamptz+i*interval '1 second', '2026-01-01 00:00:00+00'::timestamptz+i*interval '1 second'+interval '1 second' FROM generate_series(1,100) i; @@ -89,7 +89,7 @@ ANALYZE packets; ANALYZE packet_observations; ANALYZE nodes; ANALYZE node_short_ anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) for i, p := range detail.Packets { number := i + 1 - if p.PacketHash != fmt.Sprintf("%064x", number) || p.RouteType != 1 || p.FirstHeardAt != anchor.Add(time.Duration(number)*time.Second).UnixMilli() || p.LastHeardAt != p.FirstHeardAt+1000 { + if p.PacketHash != fmt.Sprintf("%064x", number) || p.RouteType != 2 || p.FirstHeardAt != anchor.Add(time.Duration(number)*time.Second).UnixMilli() || p.LastHeardAt != p.FirstHeardAt+1000 { t.Fatalf("wrong packet fields or order at %d", number) } if number == 100 { @@ -109,7 +109,10 @@ ANALYZE packets; ANALYZE packet_observations; ANALYZE nodes; ANALYZE node_short_ } want := "YVR relay" if number == 1 { - want = "YYJ relay" + if len(p.ResolvedRoute) != 1 || p.ResolvedRoute[0].Confidence != "ambiguous" || len(p.ResolvedRoute[0].Nodes) != 2 || p.Quality.Status != "ambiguous" { + t.Fatal("conflicting regions must retain both candidates") + } + continue } if len(p.ResolvedRoute) != 1 || p.ResolvedRoute[0].Confidence != "high" || len(p.ResolvedRoute[0].Nodes) != 1 || p.ResolvedRoute[0].Nodes[0].Name == nil || *p.ResolvedRoute[0].Nodes[0].Name != want { t.Fatalf("wrong region precedence/resolution at %d", number) @@ -122,8 +125,8 @@ ANALYZE packets; ANALYZE packet_observations; ANALYZE nodes; ANALYZE node_short_ if counter.observations != 0 { t.Errorf("extra observation queries: %d; want 0", counter.observations) } - if counter.queries != 100 { - t.Errorf("got %d queries; want one packet query and 99 unchanged path-resolution queries", counter.queries) + if counter.queries != 99 { + t.Errorf("got %d queries; want one packet query and 98 batched path-resolution queries", counter.queries) } counter.queries, counter.observations = 0, 0 missing, err := store.GetTraceByTag(ctx, "00000000") diff --git a/db/traces_test.go b/db/traces_test.go index 0e3d4b48..ef0a4410 100644 --- a/db/traces_test.go +++ b/db/traces_test.go @@ -99,7 +99,7 @@ func TestGetTraceByTag_WithPacket(t *testing.T) { firstHeard := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} lastHeard := pgtype.Timestamptz{Time: time.UnixMilli(1700000001000), Valid: true} // aabbccdd is valid hex for the packet hash - parsedPayload := []byte(`{"pathHashes":["aabb"],"snrValues":[15.0],"flags":0}`) + parsedPayload := []byte(`{"pathHashes":["aabb"],"snrValues":[15.0],"flags":1}`) scopeName := "default" mock.EXPECT(). @@ -107,7 +107,8 @@ func TestGetTraceByTag_WithPacket(t *testing.T) { Return([]sqlc.GetPacketsByTraceTagRow{ { PacketHashHex: "aabbccdd", - RouteType: 1, + RouteType: 2, + RawPayload: []byte{0, 0, 0, 0, 0, 0, 0, 0, 1, 0xaa, 0xbb}, ScopeName: &scopeName, FirstHeardAt: firstHeard, LastHeardAt: lastHeard, @@ -117,11 +118,11 @@ func TestGetTraceByTag_WithPacket(t *testing.T) { }, nil) mock.EXPECT(). - ResolvePathHashesP2(gomock.Any(), sqlc.ResolvePathHashesP2Params{ - Iata: "YVR", - Column2: [][]byte{{0xaa, 0xbb}}, + ResolveTraceHashes(gomock.Any(), sqlc.ResolveTraceHashesParams{ + Iatas: []string{"YVR"}, + Hashes: [][]byte{{0xaa, 0xbb}}, }). - Return([]sqlc.ResolvePathHashesP2Row{}, nil) + Return([]sqlc.ResolveTraceHashesRow{}, nil) store := &Store{q: mock} detail, err := store.GetTraceByTag(context.Background(), "trace-001") diff --git a/docs/docs.go b/docs/docs.go index 93eaf49f..9665b7c8 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4583,6 +4583,9 @@ const docTemplate = `{ "description": "matched transport scope name e.g. \"#bc\"", "type": "string" }, + "traceQuality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "transportCodes": { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketTransportCodes" } @@ -4714,6 +4717,13 @@ const docTemplate = `{ }, "sourceBroker": { "type": "string" + }, + "traceQuality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, + "wirePathBytes": { + "description": "Exact on-air TRACE SNR bytes, before the display path substitution.", + "type": "string" } } }, @@ -5663,6 +5673,9 @@ const docTemplate = `{ "description": "hex-encoded packet hash", "type": "string" }, + "quality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "rawPath": { "description": "hops as received in the packet", "type": "array", @@ -5717,6 +5730,9 @@ const docTemplate = `{ "type": "string" } }, + "quality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "snrValues": { "description": "SNR per hop from the most complete observation", "type": "array", @@ -5828,6 +5844,21 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality": { + "type": "object", + "properties": { + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "description": "supported, suspect, or ambiguous; supported is not authenticated", + "type": "string" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 8d6c0357..1a204f32 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -4581,6 +4581,9 @@ "description": "matched transport scope name e.g. \"#bc\"", "type": "string" }, + "traceQuality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "transportCodes": { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketTransportCodes" } @@ -4712,6 +4715,13 @@ }, "sourceBroker": { "type": "string" + }, + "traceQuality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, + "wirePathBytes": { + "description": "Exact on-air TRACE SNR bytes, before the display path substitution.", + "type": "string" } } }, @@ -5661,6 +5671,9 @@ "description": "hex-encoded packet hash", "type": "string" }, + "quality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "rawPath": { "description": "hops as received in the packet", "type": "array", @@ -5715,6 +5728,9 @@ "type": "string" } }, + "quality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "snrValues": { "description": "SNR per hop from the most complete observation", "type": "array", @@ -5826,6 +5842,21 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality": { + "type": "object", + "properties": { + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "description": "supported, suspect, or ambiguous; supported is not authenticated", + "type": "string" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 726cdce3..8fda64ea 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -788,6 +788,8 @@ definitions: scope: description: matched transport scope name e.g. "#bc" type: string + traceQuality: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality' transportCodes: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketTransportCodes' type: object @@ -887,6 +889,11 @@ definitions: type: number sourceBroker: type: string + traceQuality: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality' + wirePathBytes: + description: Exact on-air TRACE SNR bytes, before the display path substitution. + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.PacketObservationSummary: properties: @@ -1530,6 +1537,8 @@ definitions: packetHash: description: hex-encoded packet hash type: string + quality: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality' rawPath: description: hops as received in the packet items: @@ -1569,6 +1578,8 @@ definitions: items: type: string type: array + quality: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality' snrValues: description: SNR per hop from the most complete observation items: @@ -1642,6 +1653,16 @@ definitions: url: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality: + properties: + reasons: + items: + type: string + type: array + status: + description: supported, suspect, or ambiguous; supported is not authenticated + type: string + type: object internal_api_handlers.APIError: properties: code: diff --git a/internal/api/packets.go b/internal/api/packets.go index e5f47fe8..f2fccb76 100644 --- a/internal/api/packets.go +++ b/internal/api/packets.go @@ -3,6 +3,8 @@ package api +import "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" + import ( "encoding/hex" "encoding/json" @@ -53,19 +55,21 @@ type PacketPathLength struct { // PacketObservationDetail is a full observation including radio settings and resolved path. type PacketObservationDetail struct { - ID int64 `json:"id"` - ObserverID uuid.UUID `json:"observerId"` - ObserverName *string `json:"observerName,omitempty"` - IATA string `json:"iata"` - HeardAt int64 `json:"heardAt"` // epoch ms - PathLength PacketPathLength `json:"pathLength"` - PathBytes *string `json:"pathBytes,omitempty"` // hex-encoded accumulated path hashes - RSSI *int16 `json:"rssi,omitempty"` - SNR *float32 `json:"snr,omitempty"` - PropagationTimeMs *int32 `json:"propagationTimeMs"` // ms since first observation; 0 for first - Radio *PacketRadio `json:"radio,omitempty"` - SourceBroker string `json:"sourceBroker"` - ResolvedPath []ResolvedHop `json:"resolvedPath"` // per-observation resolved path hashes + WirePathBytes *string `json:"wirePathBytes,omitempty"` // Exact on-air TRACE SNR bytes, before the display path substitution. + TraceQuality *tracequality.Quality `json:"traceQuality,omitempty"` + ID int64 `json:"id"` + ObserverID uuid.UUID `json:"observerId"` + ObserverName *string `json:"observerName,omitempty"` + IATA string `json:"iata"` + HeardAt int64 `json:"heardAt"` // epoch ms + PathLength PacketPathLength `json:"pathLength"` + PathBytes *string `json:"pathBytes,omitempty"` // hex-encoded accumulated path hashes + RSSI *int16 `json:"rssi,omitempty"` + SNR *float32 `json:"snr,omitempty"` + PropagationTimeMs *int32 `json:"propagationTimeMs"` // ms since first observation; 0 for first + Radio *PacketRadio `json:"radio,omitempty"` + SourceBroker string `json:"sourceBroker"` + ResolvedPath []ResolvedHop `json:"resolvedPath"` // per-observation resolved path hashes // ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type // carries one. Prefer the snapshot captured at ingest; legacy observations without a // snapshot use the current node registry. Endpoint matching itself is unchanged: @@ -134,6 +138,7 @@ type PacketTransportCodes struct { // Packet is the full packet representation including all observations and resolved paths. type Packet struct { + TraceQuality *tracequality.Quality `json:"traceQuality,omitempty"` PacketHash string `json:"packetHash"` Header PacketHeader `json:"header"` TransportCodes *PacketTransportCodes `json:"transportCodes,omitempty"` diff --git a/internal/api/traces.go b/internal/api/traces.go index b08fd7c3..f4a70d05 100644 --- a/internal/api/traces.go +++ b/internal/api/traces.go @@ -3,29 +3,33 @@ package api +import "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" + // TraceTagSummary is a single trace tag with aggregate metadata. type TraceTagSummary struct { - TraceTag string `json:"traceTag"` // hex-encoded 4-byte tag - FirstHeardAt int64 `json:"firstHeardAt"` // epoch ms - LastHeardAt int64 `json:"lastHeardAt"` // epoch ms - PacketCount int64 `json:"packetCount"` // number of packets with this trace tag - IATACount int64 `json:"iataCount"` // number of distinct IATAs where heard - TraceType string `json:"traceType"` // TRACE or PING - PathHashes []string `json:"pathHashes"` // hops from the most complete observation - SNRValues []float32 `json:"snrValues"` // SNR per hop from the most complete observation + Quality tracequality.Quality `json:"quality"` + TraceTag string `json:"traceTag"` // hex-encoded 4-byte tag + FirstHeardAt int64 `json:"firstHeardAt"` // epoch ms + LastHeardAt int64 `json:"lastHeardAt"` // epoch ms + PacketCount int64 `json:"packetCount"` // number of packets with this trace tag + IATACount int64 `json:"iataCount"` // number of distinct IATAs where heard + TraceType string `json:"traceType"` // TRACE or PING + PathHashes []string `json:"pathHashes"` // hops from the most complete observation + SNRValues []float32 `json:"snrValues"` // SNR per hop from the most complete observation } // TracePacket is a single packet within a trace series, including its // resolved route derived from the trace path hashes. type TracePacket struct { - PacketHash string `json:"packetHash"` // hex-encoded packet hash - RouteType int16 `json:"routeType"` // numeric route type - RouteTypeName string `json:"routeTypeName"` // human-readable route type - Scope *string `json:"scope,omitempty"` // transport scope name, if known - FirstHeardAt int64 `json:"firstHeardAt"` // epoch ms - LastHeardAt int64 `json:"lastHeardAt"` // epoch ms - RawPath []RawHop `json:"rawPath"` // hops as received in the packet - ResolvedRoute []ResolvedHop `json:"resolvedRoute"` // hops resolved to known nodes + Quality tracequality.Quality `json:"quality"` + PacketHash string `json:"packetHash"` // hex-encoded packet hash + RouteType int16 `json:"routeType"` // numeric route type + RouteTypeName string `json:"routeTypeName"` // human-readable route type + Scope *string `json:"scope,omitempty"` // transport scope name, if known + FirstHeardAt int64 `json:"firstHeardAt"` // epoch ms + LastHeardAt int64 `json:"lastHeardAt"` // epoch ms + RawPath []RawHop `json:"rawPath"` // hops as received in the packet + ResolvedRoute []ResolvedHop `json:"resolvedRoute"` // hops resolved to known nodes } // RawHop is a single hop in a trace path as received in the packet. diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 6c3abb72..46d7f830 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -158,6 +158,7 @@ type DB interface { // ResolvePathHashes returns a list of node UUIDs for the given path hash prefixes and IATA. ResolvePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) + ResolveTracePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) // ResolveEndpointHashes matches one-byte logical endpoints, including companions. ResolveEndpointHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index cffa4654..0c45b48a 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -266,6 +266,9 @@ func (s *stubDB) GetObserverScopes(_ context.Context, _ uuid.UUID) ([]string, er func (s *stubDB) ResolvePathHashes(_ context.Context, _ string, _ [][]byte) (map[string][]api.ResolvedPathEntry, error) { return nil, nil } +func (s *stubDB) ResolveTracePathHashes(_ context.Context, _ string, _ [][]byte) (map[string][]api.ResolvedPathEntry, error) { + return nil, nil +} func (s *stubDB) ResolveEndpointHashes(_ context.Context, _ string, _ [][]byte) (map[string][]api.ResolvedPathEntry, error) { return nil, nil diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index e8754925..2a4b84eb 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -18,6 +18,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/lora" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" "github.com/google/uuid" "github.com/meshcore-go/meshcore-go" ) @@ -202,13 +203,14 @@ type parsedGroupEnvelope struct { // is the accumulated per-hop SNR sequence carried in the packet header's // path field (a protocol-level repurposing distinct from the payload). type parsedTrace struct { - Raw string `json:"raw"` - Type string `json:"type"` - TraceTag string `json:"traceTag"` - AuthCode uint32 `json:"authCode"` - Flags byte `json:"flags"` - PathHashes []string `json:"pathHashes"` - SNRValues []float32 `json:"snrValues"` + Quality tracequality.Quality `json:"quality"` + Raw string `json:"raw"` + Type string `json:"type"` + TraceTag string `json:"traceTag"` + AuthCode uint32 `json:"authCode"` + Flags byte `json:"flags"` + PathHashes []string `json:"pathHashes"` + SNRValues []float32 `json:"snrValues"` } // parsedAck is the parsed form of an ACK payload (§2.12). Checksum is the @@ -557,7 +559,10 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ hashes = append(hashes, hex.EncodeToString(h)) rawHashes = append(rawHashes, h) } - traceRawHashes = rawHashes + quality := tracequality.Assess(packet.Payload, int(packet.PayloadVer()), int(packet.RouteType()), int(packet.PathLength)) + if quality.Status == "supported" { + traceRawHashes = rawHashes + } // SNR values are in packet.Path, one signed int8 per consumed hop snrValues := make([]float32, 0, len(packet.Path)) for _, b := range packet.Path { @@ -568,6 +573,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ traceType = "PING" } pt := parsedTrace{ + Quality: quality, Raw: hex.EncodeToString(packet.Payload), Type: traceType, TraceTag: hex.EncodeToString(uint32ToBytes(trace.Tag)), @@ -585,8 +591,8 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ // originator isn't in PathHashes) and is skipped. Either side // resolving ambiguously (>1 candidate) or not at all (0 // candidates) skips that specific pair. - if len(rawHashes) >= 2 { - resolved, rErr := w.db.ResolvePathHashes(ctx, iata, rawHashes) + if quality.Status == "supported" && len(rawHashes) >= 2 { + resolved, rErr := w.db.ResolveTracePathHashes(ctx, iata, rawHashes) if rErr == nil { for i := 1; i < len(rawHashes); i++ { if i >= len(snrValues) { @@ -822,12 +828,20 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if packet.PayloadType() == meshcore.PayloadTypeTrace { hashes = traceRawHashes } - resolved, err := w.db.ResolvePathHashes(ctx, iata, hashes) + var resolved map[string][]api.ResolvedPathEntry + if packet.PayloadType() == meshcore.PayloadTypeTrace { + resolved, err = w.db.ResolveTracePathHashes(ctx, iata, hashes) + } else { + resolved, err = w.db.ResolvePathHashes(ctx, iata, hashes) + } if err != nil { w.log.Error("path resolution failed", "error", err) } var resolvedIDs []uuid.UUID for _, entries := range resolved { + if len(entries) != 1 { + continue + } for _, e := range entries { resolvedIDs = append(resolvedIDs, e.NodeID) } @@ -846,13 +860,31 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ nodeIDs = append(nodeIDs, entries[0].NodeID) hashPrefixes = append(hashPrefixes, hash) } - if allHigh && len(nodeIDs) > 1 { + if allHigh && len(nodeIDs) > 1 && packet.PayloadType() != meshcore.PayloadTypeTrace { if err := w.db.UpsertKnownRoute(ctx, nodeIDs, hashPrefixes, iata, int32(len(nodeIDs))); err != nil { w.log.Error("failed to upsert known route", "error", err) } } } - w.runCapabilityDetection(ctx, packet.PayloadType(), packet.PathHashSize(), resolvedIDs) + hashSize := packet.PathHashSize() + if packet.PayloadType() == meshcore.PayloadTypeTrace { + hashSize = 1 + if len(traceRawHashes) > 0 { + hashSize = uint8(len(traceRawHashes[0])) + } + // Only consumed, uniquely resolved hops demonstrate trace capability. + resolvedIDs = nil + for i, hash := range traceRawHashes { + if i >= len(packet.Path) { + break + } + entries := resolved[hex.EncodeToString(hash)] + if len(entries) == 1 { + resolvedIDs = append(resolvedIDs, entries[0].NodeID) + } + } + } + w.runCapabilityDetection(ctx, packet.PayloadType(), hashSize, resolvedIDs) if inserted || repeat { if inserted { diff --git a/internal/ingest/trace_quality_test.go b/internal/ingest/trace_quality_test.go new file mode 100644 index 00000000..89c36b1a --- /dev/null +++ b/internal/ingest/trace_quality_test.go @@ -0,0 +1,75 @@ +package ingest + +import ( + "context" + "encoding/hex" + "encoding/json" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +type traceSafetyDB struct { + stubDB + saved UpsertPacketParams + resolutions, neighbors, routes int +} + +func (s *traceSafetyDB) UpsertPacket(_ context.Context, p UpsertPacketParams) (bool, error) { + s.saved = p + return true, nil +} +func (s *traceSafetyDB) ResolveTracePathHashes(_ context.Context, _ string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + if len(hashes) > 0 { + s.resolutions++ + } + out := map[string][]api.ResolvedPathEntry{} + for i, h := range hashes { + id := uuid.UUID{} + id[15] = byte(i + 1) + out[hex.EncodeToString(h)] = []api.ResolvedPathEntry{{NodeID: id}} + } + return out, nil +} +func (s *traceSafetyDB) UpsertNodeNeighbor(_ context.Context, _, _ uuid.UUID, _ string, _ *float32, _ *string) error { + s.neighbors++ + return nil +} +func (s *traceSafetyDB) UpsertKnownRoute(_ context.Context, _ []uuid.UUID, _ [][]byte, _ string, _ int32) error { + s.routes++ + return nil +} + +func TestTraceQualityDoesNotPromoteMalformedOrUnvisitedHops(t *testing.T) { + for _, bad := range []bool{false, true} { + w, _ := newTestWorker() + db := &traceSafetyDB{} + w.db = db + payload, _ := (&meshcore.Trace{Tag: 7, Flags: 1, PathHashes: []byte{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}}).ToBytes() + version := byte(0) + if bad { + version = 1 + } + pkt := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeDirect, meshcore.PayloadTypeTrace, version), PathLength: 2, Path: []byte{0, 252}, Payload: payload} + w.handlePacket(context.Background(), "YKF", "0102", packetEnvelope(t, pkt)) + if len(db.saved.RawPayload) == 0 || len(db.traceHearings) != 1 { + t.Fatal("raw evidence lost") + } + var parsed parsedTrace + if err := json.Unmarshal(db.saved.ParsedPayload, &parsed); err != nil { + t.Fatal(err) + } + if bad { + if parsed.Quality.Status != "suspect" || db.neighbors != 0 || db.resolutions != 0 || len(db.setCapabilityCalls) != 0 { + t.Fatalf("malformed trace promoted: %+v", db) + } + } else if parsed.Quality.Status != "supported" || db.neighbors != 1 || len(db.setCapabilityCalls) != 2 { + t.Fatalf("valid consumed links/capabilities missing: %+v", db) + } + if db.routes != 0 { + t.Fatal("planned route promoted as observed") + } + } +} diff --git a/internal/tracequality/quality.go b/internal/tracequality/quality.go new file mode 100644 index 00000000..e8857533 --- /dev/null +++ b/internal/tracequality/quality.go @@ -0,0 +1,96 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package tracequality checks TRACE structure, not authenticity or RF plausibility. +package tracequality + +import "encoding/hex" + +type Quality struct { + Status string `json:"status"` // supported, suspect, or ambiguous; supported is not authenticated + Reasons []string `json:"reasons"` +} + +func (q *Quality) Add(reason string) { + for _, r := range q.Reasons { + if r == reason { + return + } + } + q.Reasons = append(q.Reasons, reason) + if reason == "ambiguous_prefix" && q.Status != "suspect" { + q.Status = "ambiguous" + } else if reason != "ambiguous_prefix" { + q.Status = "suspect" + } +} + +// Assess uses TRACE's power-of-two payload hash widths, NOT the ordinary +// packet path's 1/2/3-byte widths. The path field contains consumed-hop SNRs. +// -1 means a header/path field is unavailable in a retained summary. +// Mesh.cpp onRecvPacket/sendDirect: a trace uses direct routing, flags bits +// 0..1, and fewer than 64 SNR bytes. Partial probes and return loops are valid. +func Assess(raw []byte, version, route, pathLength int) Quality { + q := Quality{Status: "supported", Reasons: []string{}} + if version > 0 { + q.Add("unsupported_version") + } + if route >= 0 && route != 2 && route != 3 { + q.Add("non_direct_trace") + } + if len(raw) < 10 { + q.Add("incomplete_payload") + return q + } + if len(raw) > 184 { + q.Add("payload_too_large") + } + if raw[8]&0xfc != 0 { + q.Add("unsupported_flags") + } + size := 1 << (raw[8] & 3) + if (len(raw)-9)%size != 0 { + q.Add("misaligned_hashes") + } + count := (len(raw) - 9) / size + if count > 63 { + q.Add("too_many_hops") + } + if pathLength > 63 || pathLength > count { + q.Add("invalid_snr_path") + } + return q +} + +// Stored also checks old parsed rows, without rewriting or deleting the evidence. +func Stored(raw string, flags byte, hashes []string, snrs []float32) Quality { + b, err := hex.DecodeString(raw) + if err != nil { + return Quality{Status: "suspect", Reasons: []string{"incomplete_payload"}} + } + if raw == "" { // Older exports may contain only decoded fields. + b = make([]byte, 9) + b[8] = flags + for _, h := range hashes { + v, e := hex.DecodeString(h) + if e != nil { + return Quality{Status: "suspect", Reasons: []string{"misaligned_hashes"}} + } + b = append(b, v...) + } + } + q := Assess(b, -1, -1, len(snrs)) + size := 1 << (flags & 3) + if len(b) >= 9 && b[8] != flags { + q.Add("misaligned_hashes") + } + for _, h := range hashes { + if len(h) != 2*size { + q.Add("misaligned_hashes") + } + } + if len(b) >= 9 && len(hashes) != (len(b)-9)/size { + q.Add("misaligned_hashes") + } + return q +} diff --git a/internal/tracequality/quality_test.go b/internal/tracequality/quality_test.go new file mode 100644 index 00000000..f811ae30 --- /dev/null +++ b/internal/tracequality/quality_test.go @@ -0,0 +1,51 @@ +package tracequality + +import ( + "bytes" + "slices" + "testing" +) + +func TestTraceFormat(t *testing.T) { + for flags := byte(0); flags < 4; flags++ { + raw := make([]byte, 9) + raw[8] = flags + raw = append(raw, bytes.Repeat([]byte{0xaa}, 3*(1< Date: Sun, 4 Oct 2026 00:05:18 -0400 Subject: [PATCH 13/23] test(traces): account for quality in legacy pagination fixtures --- db/trace_pagination_integration_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/trace_pagination_integration_test.go b/db/trace_pagination_integration_test.go index 90ed19d5..dc123a43 100644 --- a/db/trace_pagination_integration_test.go +++ b/db/trace_pagination_integration_test.go @@ -16,6 +16,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" "github.com/jackc/pgx/v5" ) @@ -89,7 +90,7 @@ func TestTraceTagPaginationPostgres(t *testing.T) { paths := map[string][]string{"a": {"aa", "bb", "cc"}, "b": {"bb", "11", "22", "33"}, "c": {"cc"}, "d": {"dd"}} snrs := map[string][]float32{"a": {1, 2, 3}, "b": {2, 3, 4, 5}, "c": {3}, "d": {4}} summary := func(tag string, first, last int, count, iatas int64, kind string) api.TraceTagSummary { - return api.TraceTagSummary{TraceTag: strings.Repeat(tag, 8), FirstHeardAt: at(first).UnixMilli(), LastHeardAt: at(last).UnixMilli(), PacketCount: count, IATACount: iatas, TraceType: kind, PathHashes: paths[tag], SNRValues: snrs[tag]} + return api.TraceTagSummary{Quality: tracequality.Quality{Status: "suspect", Reasons: []string{"non_direct_trace"}}, TraceTag: strings.Repeat(tag, 8), FirstHeardAt: at(first).UnixMilli(), LastHeardAt: at(last).UnixMilli(), PacketCount: count, IATACount: iatas, TraceType: kind, PathHashes: paths[tag], SNRValues: snrs[tag]} } a, b, c, d := summary("a", 1, 40, 3, 2, "TRACE"), summary("b", 4, 30, 2, 1, "TRACE"), summary("c", 6, 20, 1, 1, "TRACE"), summary("d", 7, 15, 1, 1, "PING") for _, tc := range []struct { From 84ac3c8781ed19f99e7d996e07e8b671dc6efc92 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 4 Oct 2026 01:03:08 -0400 Subject: [PATCH 14/23] feat(topology): serve bounded route links and identify decoded Public traffic --- db/queries/queries.sql | 15 +++++ db/sqlc/mock/querier.go | 15 +++++ db/sqlc/querier.go | 2 + db/sqlc/queries.sql.go | 53 +++++++++++++++++ db/topology.go | 25 ++++++++ db/topology_integration_test.go | 54 ++++++++++++++++++ docs/docs.go | 69 +++++++++++++++++++++++ docs/swagger.json | 69 +++++++++++++++++++++++ docs/swagger.yaml | 48 ++++++++++++++++ internal/api/handlers/routes.go | 1 + internal/api/handlers/stub_reader_test.go | 4 ++ internal/api/handlers/topology.go | 44 +++++++++++++++ internal/api/handlers/topology_test.go | 22 ++++++++ internal/api/reader.go | 2 + internal/api/topology.go | 14 +++++ internal/cache/cache_test.go | 4 ++ internal/cache/reader.go | 9 +++ internal/ingest/channel_scope_test.go | 34 +++++++++++ internal/ingest/side_effects.go | 2 + 19 files changed, 486 insertions(+) create mode 100644 db/topology.go create mode 100644 db/topology_integration_test.go create mode 100644 internal/api/handlers/topology.go create mode 100644 internal/api/handlers/topology_test.go create mode 100644 internal/api/topology.go diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 9f513b13..e31d9ecb 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1853,3 +1853,18 @@ JOIN node_short_ids si ON si.iata = ANY(@iatas::bpchar[]) AND JOIN nodes n ON n.id=si.node_id WHERE substring(n.public_key FROM 1 FOR octet_length(h.hash))=h.hash ORDER BY si.iata, h.hash, n.id; + +-- name: GetTopologyLinks :many +-- Reduce route history to unique adjacent pairs before transferring it to a browser. +WITH pairs AS ( + SELECT h.node_id AS a, r.node_ids[(h.ord+1)::int] AS b + FROM known_routes r CROSS JOIN LATERAL unnest(r.node_ids) WITH ORDINALITY h(node_id,ord) + WHERE r.last_seen >= @since::timestamptz AND r.last_seen < @until::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]),0)=0 OR r.iata=ANY(@iatas::bpchar[])) + AND h.ord < cardinality(r.node_ids) +) +SELECT LEAST(a,b)::uuid AS from_id, GREATEST(a,b)::uuid AS to_id +FROM pairs WHERE a IS NOT NULL AND b IS NOT NULL AND a<>b +GROUP BY LEAST(a,b), GREATEST(a,b) +ORDER BY LEAST(a,b), GREATEST(a,b) +LIMIT @link_limit::int; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 6813cc5e..0fc75cb6 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -1137,6 +1137,21 @@ func (mr *MockQuerierMockRecorder) GetTopNodes(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopNodes", reflect.TypeOf((*MockQuerier)(nil).GetTopNodes), ctx, arg) } +// GetTopologyLinks mocks base method. +func (m *MockQuerier) GetTopologyLinks(ctx context.Context, arg db.GetTopologyLinksParams) ([]db.GetTopologyLinksRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTopologyLinks", ctx, arg) + ret0, _ := ret[0].([]db.GetTopologyLinksRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTopologyLinks indicates an expected call of GetTopologyLinks. +func (mr *MockQuerierMockRecorder) GetTopologyLinks(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopologyLinks", reflect.TypeOf((*MockQuerier)(nil).GetTopologyLinks), ctx, arg) +} + // GetTransportScopeByName mocks base method. func (m *MockQuerier) GetTransportScopeByName(ctx context.Context, name string) (int32, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 5912fa44..c6338c18 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -156,6 +156,8 @@ type Querier interface { // Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; // node_id is NULL once the node row is gone. iata is a representative one. GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]GetTopNodesRow, error) + // Reduce route history to unique adjacent pairs before transferring it to a browser. + GetTopologyLinks(ctx context.Context, arg GetTopologyLinksParams) ([]GetTopologyLinksRow, error) GetTransportScopeByName(ctx context.Context, name string) (int32, error) GetTransportScopes(ctx context.Context) ([]GetTransportScopesRow, error) // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 92a6a176..fcc503fc 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2237,6 +2237,59 @@ func (q *Queries) GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]Get return items, nil } +const getTopologyLinks = `-- name: GetTopologyLinks :many +WITH pairs AS ( + SELECT h.node_id AS a, r.node_ids[(h.ord+1)::int] AS b + FROM known_routes r CROSS JOIN LATERAL unnest(r.node_ids) WITH ORDINALITY h(node_id,ord) + WHERE r.last_seen >= $2::timestamptz AND r.last_seen < $3::timestamptz + AND (COALESCE(cardinality($4::bpchar[]),0)=0 OR r.iata=ANY($4::bpchar[])) + AND h.ord < cardinality(r.node_ids) +) +SELECT LEAST(a,b)::uuid AS from_id, GREATEST(a,b)::uuid AS to_id +FROM pairs WHERE a IS NOT NULL AND b IS NOT NULL AND a<>b +GROUP BY LEAST(a,b), GREATEST(a,b) +ORDER BY LEAST(a,b), GREATEST(a,b) +LIMIT $1::int +` + +type GetTopologyLinksParams struct { + LinkLimit int32 `json:"link_limit"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetTopologyLinksRow struct { + FromID uuid.UUID `json:"from_id"` + ToID uuid.UUID `json:"to_id"` +} + +// Reduce route history to unique adjacent pairs before transferring it to a browser. +func (q *Queries) GetTopologyLinks(ctx context.Context, arg GetTopologyLinksParams) ([]GetTopologyLinksRow, error) { + rows, err := q.db.Query(ctx, getTopologyLinks, + arg.LinkLimit, + arg.Since, + arg.Until, + arg.Iatas, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetTopologyLinksRow{} + for rows.Next() { + var i GetTopologyLinksRow + if err := rows.Scan(&i.FromID, &i.ToID); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getTransportScopeByName = `-- name: GetTransportScopeByName :one SELECT id FROM transport_scopes WHERE name = $1 ` diff --git a/db/topology.go b/db/topology.go new file mode 100644 index 00000000..24c45cc8 --- /dev/null +++ b/db/topology.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "context" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "time" +) + +func (s *Store) GetTopologyLinks(ctx context.Context, iatas []string, since, until time.Time) (*api.TopologyLinks, error) { + const limit = 100000 + rows, err := s.q.GetTopologyLinks(ctx, sqlc.GetTopologyLinksParams{Iatas: iatas, Since: pgtype.Timestamptz{Time: since, Valid: true}, Until: pgtype.Timestamptz{Time: until, Valid: true}, LinkLimit: limit + 1}) + if err != nil { + return nil, err + } + result := &api.TopologyLinks{Links: make([][2]uuid.UUID, 0, min(len(rows), limit)), Capped: len(rows) > limit, Since: since.UnixMilli(), Until: until.UnixMilli()} + for _, r := range rows[:min(len(rows), limit)] { + result.Links = append(result.Links, [2]uuid.UUID{r.FromID, r.ToID}) + } + return result, nil +} diff --git a/db/topology_integration_test.go b/db/topology_integration_test.go new file mode 100644 index 00000000..58b4c579 --- /dev/null +++ b/db/topology_integration_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "context" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "testing" + "time" +) + +func TestTopologyLinksPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + if _, err = tx.Exec(ctx, "CREATE TEMP TABLE known_routes (LIKE public.known_routes INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + ids := []uuid.UUID{uuid.New(), uuid.New(), uuid.New()} + now := time.Now().UTC().Truncate(time.Minute) + // Thousands of duplicate/reversed routes transfer only their unique adjacent pairs. + _, err = tx.Exec(ctx, `INSERT INTO known_routes (path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) + SELECT decode(md5(i::text),'hex'),CASE WHEN i%2=0 THEN $1::uuid[] ELSE $2::uuid[] END,ARRAY['\xaa'::bytea,'\xbb'::bytea], + 'YKF',2,$3,$3 FROM generate_series(1,1000) i`, ids[:2], []uuid.UUID{ids[1], ids[0]}, now) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO known_routes(path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) VALUES + (decode(md5('other'),'hex'),$1,ARRAY['\xbb'::bytea,'\xcc'::bytea],'YOW',2,$3,$3), + (decode(md5('gap'),'hex'),$2,ARRAY['\xaa'::bytea,'\x00'::bytea,'\xcc'::bytea],'YKF',3,$3,$3), + (decode(md5('old'),'hex'),$1,ARRAY['\xbb'::bytea,'\xcc'::bytea],'YKF',2,$3-interval '25 hours',$3-interval '25 hours')`, ids[1:], []*uuid.UUID{&ids[0], nil, &ids[2]}, now) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + result, err := store.GetTopologyLinks(ctx, []string{"YKF"}, now.Add(-24*time.Hour), now.Add(time.Minute)) + if err != nil || result.Capped || len(result.Links) != 1 { + t.Fatalf("scoped: %+v %v", result, err) + } + all, err := store.GetTopologyLinks(ctx, nil, now.Add(-24*time.Hour), now.Add(time.Minute)) + if err != nil || len(all.Links) != 2 { + t.Fatalf("all regions: %+v %v", all, err) + } + if result.Since != now.Add(-24*time.Hour).UnixMilli() { + t.Fatal("window changed") + } +} diff --git a/docs/docs.go b/docs/docs.go index 9665b7c8..7f1d1abb 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2179,6 +2179,52 @@ const docTemplate = `{ } } }, + "/routes/topology": { + "get": { + "description": "Returns up to 100000 undirected node-ID pairs, deduplicated across known routes. Minute-aligned window includes the current minute. Existing rate limits apply; no per-node lookup or route pagination is needed. Capped is true when more links exist.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Unique adjacent route links for a topology snapshot", + "parameters": [ + { + "type": "string", + "description": "History window: 15m (default), 1h, or 24h", + "name": "window", + "in": "query" + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/routes/{iata}/{pathKey}/observations": { "get": { "description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.", @@ -5642,6 +5688,29 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks": { + "type": "object", + "properties": { + "capped": { + "type": "boolean" + }, + "links": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "since": { + "type": "integer" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.TraceDetail": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 1a204f32..b40537be 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2177,6 +2177,52 @@ } } }, + "/routes/topology": { + "get": { + "description": "Returns up to 100000 undirected node-ID pairs, deduplicated across known routes. Minute-aligned window includes the current minute. Existing rate limits apply; no per-node lookup or route pagination is needed. Capped is true when more links exist.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Unique adjacent route links for a topology snapshot", + "parameters": [ + { + "type": "string", + "description": "History window: 15m (default), 1h, or 24h", + "name": "window", + "in": "query" + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/routes/{iata}/{pathKey}/observations": { "get": { "description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.", @@ -5640,6 +5686,29 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks": { + "type": "object", + "properties": { + "capped": { + "type": "boolean" + }, + "links": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "since": { + "type": "integer" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.TraceDetail": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 8fda64ea..99b75549 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1515,6 +1515,21 @@ definitions: senderName: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks: + properties: + capped: + type: boolean + links: + items: + items: + type: string + type: array + type: array + since: + type: integer + until: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TraceDetail: properties: packets: @@ -3233,6 +3248,39 @@ paths: summary: Search known routes by source and destination hash tags: - Routes + /routes/topology: + get: + description: Returns up to 100000 undirected node-ID pairs, deduplicated across + known routes. Minute-aligned window includes the current minute. Existing + rate limits apply; no per-node lookup or route pagination is needed. Capped + is true when more links exist. + parameters: + - description: 'History window: 15m (default), 1h, or 24h' + in: query + name: window + type: string + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Unique adjacent route links for a topology snapshot + tags: + - Routes /scope-catalogues: get: description: Discovered sources only. Counts describe the source region, not diff --git a/internal/api/handlers/routes.go b/internal/api/handlers/routes.go index 86a7c861..48a147bf 100644 --- a/internal/api/handlers/routes.go +++ b/internal/api/handlers/routes.go @@ -20,6 +20,7 @@ import ( func RoutesRouter(reader api.Reader) http.Handler { r := chi.NewRouter() r.Get("/", listKnownRoutes(reader)) + r.Get("/topology", getTopologyLinks(reader)) r.Get("/cross", searchCrossIATARoutes(reader)) r.Get("/search", searchKnownRoutes(reader)) r.Get("/{iata}/{pathKey}/observations", getRouteEvidence(reader)) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index efb5cb8f..4ed1ea0a 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -417,3 +417,7 @@ func (s stubReader) GetRouteEvidence(ctx context.Context, iata, key string, q ap } return nil, nil } + +func (s stubReader) GetTopologyLinks(_ context.Context, _ []string, _ time.Time, _ time.Time) (*api.TopologyLinks, error) { + return &api.TopologyLinks{}, nil +} diff --git a/internal/api/handlers/topology.go b/internal/api/handlers/topology.go new file mode 100644 index 00000000..b0fa1040 --- /dev/null +++ b/internal/api/handlers/topology.go @@ -0,0 +1,44 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "context" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "net/http" + "time" +) + +// getTopologyLinks godoc +// @Summary Unique adjacent route links for a topology snapshot +// @Description Returns up to 100000 undirected node-ID pairs, deduplicated across known routes. Minute-aligned window includes the current minute. Existing rate limits apply; no per-node lookup or route pagination is needed. Capped is true when more links exist. +// @Tags Routes +// @Produce json +// @Param window query string false "History window: 15m (default), 1h, or 24h" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Success 200 {object} api.TopologyLinks +// @Failure 400 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /routes/topology [get] +func getTopologyLinks(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + window := r.URL.Query().Get("window") + if window == "" { + window = "15m" + } + duration, ok := map[string]time.Duration{"15m": 15 * time.Minute, "1h": time.Hour, "24h": 24 * time.Hour}[window] + if !ok { + respondError(w, http.StatusBadRequest, "window must be 15m, 1h or 24h") + return + } + until := time.Now().UTC().Truncate(time.Minute).Add(time.Minute) + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + links, err := reader.GetTopologyLinks(ctx, parseIATAs(r), until.Add(-duration), until) + if err != nil { + respondError(w, http.StatusServiceUnavailable, "route snapshot unavailable; try again shortly") + return + } + respond(w, http.StatusOK, links) + } +} diff --git a/internal/api/handlers/topology_test.go b/internal/api/handlers/topology_test.go new file mode 100644 index 00000000..c4274dcf --- /dev/null +++ b/internal/api/handlers/topology_test.go @@ -0,0 +1,22 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "net/http/httptest" + "testing" +) + +func TestTopologyWindowValidation(t *testing.T) { + for _, window := range []string{"", "15m", "1h", "24h", "72h", "0", "garbage"} { + w := httptest.NewRecorder() + RoutesRouter(stubReader{}).ServeHTTP(w, httptest.NewRequest("GET", "/topology?window="+window, nil)) + want := 200 + if window == "72h" || window == "0" || window == "garbage" { + want = 400 + } + if w.Code != want { + t.Fatalf("%s: %d", window, w.Code) + } + } +} diff --git a/internal/api/reader.go b/internal/api/reader.go index dc71e63f..08a5f0ca 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -224,6 +224,8 @@ type Reader interface { // GetTraceByTag returns all packets for a given trace tag with resolved routes. GetTraceByTag(ctx context.Context, tag string) (*TraceDetail, error) + GetTopologyLinks(ctx context.Context, iatas []string, since, until time.Time) (*TopologyLinks, error) + // ListKnownRoutes returns known routes filtered by IATA and optional hop count. // A nonzero cursorID breaks ties within the cursor's millisecond. ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]KnownRoute, error) diff --git a/internal/api/topology.go b/internal/api/topology.go new file mode 100644 index 00000000..3fc85ce8 --- /dev/null +++ b/internal/api/topology.go @@ -0,0 +1,14 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import "github.com/google/uuid" + +// TopologyLinks is a bounded set of undirected, adjacent known-route segments. +// It does not imply radio reachability or join across missing path entries. +type TopologyLinks struct { + Links [][2]uuid.UUID `json:"links"` + Capped bool `json:"capped"` + Since int64 `json:"since"` + Until int64 `json:"until"` +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index b5dcc7f5..4a349d45 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -446,3 +446,7 @@ func TestCachedReader_InvalidateObserver(t *testing.T) { func (s *stubReader) GetRouteEvidence(_ context.Context, _, _ string, _ api.RouteEvidenceQuery) (*api.RouteEvidence, error) { return nil, nil } + +func (s *stubReader) GetTopologyLinks(_ context.Context, _ []string, _ time.Time, _ time.Time) (*api.TopologyLinks, error) { + return &api.TopologyLinks{}, nil +} diff --git a/internal/cache/reader.go b/internal/cache/reader.go index e491152d..6c4a201f 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -378,3 +378,12 @@ func (cr *CachedReader) SearchCrossIATARoutes(ctx context.Context, fromHash, fro func (cr *CachedReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { return cr.inner.ListTraceTags(ctx, iatas, scope, traceType, since, until, cursor, cursorTag, limit) } + +func (cr *CachedReader) GetTopologyLinks(ctx context.Context, iatas []string, since, until time.Time) (*api.TopologyLinks, error) { + regions := append([]string{}, iatas...) + sort.Strings(regions) + key := fmt.Sprintf("beacon:topology:%s:%d:%d", strings.Join(regions, ","), since.Unix(), until.Unix()) + return getOrSet(ctx, cr.c, key, 30*time.Second, func(ctx context.Context) (*api.TopologyLinks, error) { + return cr.inner.GetTopologyLinks(ctx, iatas, since, until) + }) +} diff --git a/internal/ingest/channel_scope_test.go b/internal/ingest/channel_scope_test.go index 3ee69334..e229ef23 100644 --- a/internal/ingest/channel_scope_test.go +++ b/internal/ingest/channel_scope_test.go @@ -5,6 +5,7 @@ package ingest import ( "context" + "encoding/hex" "encoding/json" "testing" "time" @@ -73,3 +74,36 @@ func TestChannelMessageScopeLive(t *testing.T) { }) } } + +func TestChannelPublicMarkerUsesDecryptionKey(t *testing.T) { + for _, public := range []bool{false, true} { + w, base := newTestWorker() + w.db = &scopeMessageDB{stubDB: base, next: &InsertedChannelMessage{ID: 1}} + key := make([]byte, 16) + if public { + key, _ = hex.DecodeString("8b3387e9c5cdea6ac9e5edbaa115cd72") + } + w.keys = &mapKeys{entries: map[byte][]keystore.Entry{0x11: {{Key: key, Name: "Public"}}}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "public-marker", hub.Scope{Events: []hub.EventType{hub.EventChannelMessage, hub.EventObserverStatus}}) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeGrpTxt, 0), Payload: encryptedGroupText(t, 0x11, key, "Fixture", "test")} + w.handlePayloadTypeSideEffects(ctx, packet, "YKF", []byte{1}, RadioSettings{}, nil, nil, nil, 0) + select { + case event := <-client.Send: + var got channelMessageEvent + if err := json.Unmarshal(event.Payload, &got); err != nil { + t.Fatal(err) + } + if got.IsPublic != public { + t.Fatal("channel name/hash used instead of actual decryption key") + } + case <-ctx.Done(): + t.Fatal("no channel event") + } + } +} diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 3299027f..95c8441a 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -48,6 +48,7 @@ type InsertedChannelMessage struct { // channelMessageEvent is the JSON payload for a channelMessage WS event. type channelMessageEvent struct { + IsPublic bool `json:"isPublic"` // True only when decryption used the documented default Public key, never inferred from a one-byte channel hash. ID int64 `json:"id"` ChannelID int `json:"channelId"` ChannelHash string `json:"channelHash"` // hex-encoded single byte @@ -223,6 +224,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc if result.Message != nil { evt := channelMessageEvent{ + IsPublic: hex.EncodeToString(result.Entry.Key) == "8b3387e9c5cdea6ac9e5edbaa115cd72", ID: result.Message.ID, Scope: result.Message.Scope, ScopeStatus: result.Message.ScopeStatus, From a9b8c60a97d8a9000d2b5fcadf10efebd0627e3e Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 4 Oct 2026 16:31:48 -0400 Subject: [PATCH 15/23] feat: stage current preview on dev for dev1 evaluation Port the qualified server preview at 84ac3c8781ed19f99e7d996e07e8b671dc6efc92 onto org dev c7209b70433b8b127a5b1062fdfb17d4a676245c. Preserve upstream changes and branch policies; keep the independent private Collector service separate. Preview-Version: 2.2.0-n30nex.1 Preview-Revision: 84ac3c8781ed19f99e7d996e07e8b671dc6efc92 --- cmd/beacon/main.go | 1 + db/nodes.go | 1 + db/nodes_test.go | 12 ++ db/packets.go | 23 ++- db/queries/queries.sql | 36 +++- db/route_evidence.go | 39 +++- db/route_evidence_integration_test.go | 71 ++++++- db/sqlc/mock/querier.go | 32 ++- db/sqlc/querier.go | 5 + db/sqlc/queries.sql.go | 162 +++++++++++++-- db/topology.go | 25 +++ db/topology_integration_test.go | 54 +++++ db/trace_pagination_integration_test.go | 3 +- db/trace_quality_integration_test.go | 77 ++++++++ db/traces.go | 179 ++++++++++++----- db/traces_integration_test.go | 13 +- db/traces_test.go | 13 +- docs/docs.go | 196 ++++++++++++++++++- docs/swagger.json | 196 ++++++++++++++++++- docs/swagger.yaml | 151 +++++++++++++- internal/api/handlers/route_evidence.go | 22 ++- internal/api/handlers/route_evidence_test.go | 25 +++ internal/api/handlers/routes.go | 1 + internal/api/handlers/scope_catalogues.go | 33 ++++ internal/api/handlers/stub_reader_test.go | 4 + internal/api/handlers/topology.go | 44 +++++ internal/api/handlers/topology_test.go | 22 +++ internal/api/packets.go | 31 +-- internal/api/reader.go | 2 + internal/api/route_evidence.go | 47 +++-- internal/api/route_evidence_test.go | 20 ++ internal/api/router/options.go | 1 + internal/api/router/router.go | 1 + internal/api/router/scope_catalogues_test.go | 28 +++ internal/api/topology.go | 14 ++ internal/api/traces.go | 36 ++-- internal/cache/cache_test.go | 4 + internal/cache/reader.go | 9 + internal/ingest/channel_scope_test.go | 34 ++++ internal/ingest/ingest.go | 1 + internal/ingest/ingest_test.go | 3 + internal/ingest/packet.go | 58 ++++-- internal/ingest/side_effects.go | 2 + internal/ingest/trace_quality_test.go | 75 +++++++ internal/meshmapper/catalogues.go | 75 +++++++ internal/meshmapper/catalogues_test.go | 103 ++++++++++ internal/meshmapper/scopes.go | 24 ++- internal/tracequality/quality.go | 96 +++++++++ internal/tracequality/quality_test.go | 51 +++++ 49 files changed, 1990 insertions(+), 165 deletions(-) create mode 100644 db/topology.go create mode 100644 db/topology_integration_test.go create mode 100644 db/trace_quality_integration_test.go create mode 100644 internal/api/handlers/scope_catalogues.go create mode 100644 internal/api/handlers/topology.go create mode 100644 internal/api/handlers/topology_test.go create mode 100644 internal/api/router/scope_catalogues_test.go create mode 100644 internal/api/topology.go create mode 100644 internal/ingest/trace_quality_test.go create mode 100644 internal/meshmapper/catalogues.go create mode 100644 internal/meshmapper/catalogues_test.go create mode 100644 internal/tracequality/quality.go create mode 100644 internal/tracequality/quality_test.go diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index e3af3490..2af8745c 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -424,6 +424,7 @@ func main() { // CachedReader. Only response projections receive the geographic annotation. reader = api.WithLocalBorders(reader, localBorders) r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, router.Options{ + ScopeCatalogues: scopeImporter, MaxConnsPerIP: resolved.MaxConnsPerIP, MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, WSAllowedOrigins: cfg.WebSocket.AllowedOrigins, diff --git a/db/nodes.go b/db/nodes.go index a602cb3d..cff12107 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -127,6 +127,7 @@ func (s *Store) ListNodes(ctx context.Context, nodeType int16, iatas []string, s Longitude: v.Longitude, IsObserver: v.IsObserver, ObserverID: nullableUUID(v.ObserverID), + DefaultScope: v.DefaultScopeName, KnownNeighborCount: v.KnownNeighborCount, NeighborIDs: v.NeighborIds, Stale: v.LastSeen.Valid && v.LastSeen.Time.Before(time.Now().Add(-s.staleThreshold)), diff --git a/db/nodes_test.go b/db/nodes_test.go index db9a0dd4..590c08cd 100644 --- a/db/nodes_test.go +++ b/db/nodes_test.go @@ -706,3 +706,15 @@ func TestDeleteOldNodes(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestListNodes_DefaultScopePreservesCaseAndMissingValue(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + scope := "#YOW" + mock.EXPECT().ListNodes(gomock.Any(), gomock.Any()).Return([]sqlc.ListNodesRow{{DefaultScopeName: &scope}, {}}, nil) + store := &Store{q: mock} + page, err := store.ListNodes(context.Background(), 0, nil, nil, nil, nil, "", "", "", 0, 10, false) + if err != nil || len(page.Items) != 2 || page.Items[0].DefaultScope == nil || *page.Items[0].DefaultScope != scope || page.Items[1].DefaultScope != nil { + t.Fatalf("scope metadata lost or invented: %+v %v", page, err) + } +} diff --git a/db/packets.go b/db/packets.go index 620acceb..a142ab6b 100644 --- a/db/packets.go +++ b/db/packets.go @@ -15,6 +15,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" "github.com/meshcore-go/meshcore-go" @@ -406,7 +407,15 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, // compute it once rather than per observation. var traceRawHashes [][]byte if row.PayloadType == int16(meshcore.PayloadTypeTrace) { - if trace, err := meshcore.TraceFromBytes(row.RawPayload); err == nil { + quality := tracequality.Assess(row.RawPayload, int(row.PayloadVersion), int(row.RouteType), -1) + for _, obs := range obsRows { + q := tracequality.Assess(row.RawPayload, int(row.PayloadVersion), int(row.RouteType), int(obs.PathLengthByte)) + for _, reason := range q.Reasons { + quality.Add(reason) + } + } + p.TraceQuality = &quality + if trace, err := meshcore.TraceFromBytes(row.RawPayload); err == nil && quality.Status == "supported" { hashSize := int(trace.PathHashSize()) for i := 0; i+hashSize <= len(trace.PathHashes); i += hashSize { traceRawHashes = append(traceRawHashes, trace.PathHashes[i:i+hashSize]) @@ -442,8 +451,11 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, obs.PropagationTimeMs = &prop resolvedPath := []api.ResolvedHop{} if row.PayloadType == int16(meshcore.PayloadTypeTrace) { + rawPath := hex.EncodeToString(v.PathBytes) + obs.WirePathBytes = &rawPath + obs.TraceQuality = p.TraceQuality if len(traceRawHashes) > 0 { - resolved, err := s.ResolvePathHashes(ctx, v.Iata, traceRawHashes) + resolved, err := s.ResolveTracePathHashes(ctx, v.Iata, traceRawHashes) if err != nil { slog.Error(fmt.Sprintf("store: path resolution failed for observation %d", v.ID), "component", "db", "error", err) } else { @@ -491,7 +503,7 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, } p.Observations = append(p.Observations, obs) } - if row.PayloadType == 9 && len(obsRows) > 0 { + if row.PayloadType == 9 && len(obsRows) > 0 && p.TraceQuality.Status == "supported" { iatas := make([]string, 0, len(obsRows)) seen := make(map[string]struct{}) for _, v := range obsRows { @@ -503,6 +515,11 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, var parsed tracePayload if err := json.Unmarshal(row.ParsedPayload, &parsed); err == nil { p.ResolvedRoute = s.resolveTraceRoute(ctx, &parsed, iatas) + for _, hop := range p.ResolvedRoute { + if hop.Confidence == "ambiguous" { + p.TraceQuality.Add("ambiguous_prefix") + } + } } } return p, nil diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 2a2c470b..e31d9ecb 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -455,6 +455,8 @@ WHERE p.packet_hash = $1; -- Return distinct observation IATAs in first-heard order for path resolution, -- without fetching full observations separately for every trace packet. SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, + p.raw_payload, p.payload_version, + EXISTS(SELECT 1 FROM packet_observations po WHERE po.packet_hash=p.packet_hash AND po.path_length_byte>=64) AS invalid_snr_path, p.route_type, p.first_heard_at, p.last_heard_at, @@ -1422,7 +1424,11 @@ SELECT WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, COALESCE(t.trace_type, '')::text AS trace_type, - t.best_payload + t.best_payload, + ARRAY(SELECT ti.iata FROM trace_iatas ti WHERE ti.trace_tag=t.trace_tag)::bpchar[] AS iatas, + EXISTS(SELECT 1 FROM packets p WHERE p.trace_tag=t.trace_tag AND p.payload_version<>0) AS unsupported_version, + EXISTS(SELECT 1 FROM packets p WHERE p.trace_tag=t.trace_tag AND p.route_type NOT IN (2,3)) AS non_direct, + EXISTS(SELECT 1 FROM packets p JOIN packet_observations po ON po.packet_hash=p.packet_hash WHERE p.trace_tag=t.trace_tag AND po.path_length_byte>=64) AS invalid_snr_path FROM trace_tags t WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) @@ -1834,3 +1840,31 @@ ON CONFLICT (country) DO UPDATE SET -- name: PruneZoneBoundaries :many -- Drops imports for IATAs no longer configured, so their manual border returns. DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY(@keep::text[])) RETURNING iata; + +-- name: ResolveTraceHashes :many +-- TRACE can terminate at companions. Preserve the full prefix (including eight +-- bytes), and return every candidate across heard regions instead of picking one. +SELECT h.hash::bytea AS hash, si.iata, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM unnest(@hashes::bytea[]) AS h(hash) +JOIN node_short_ids si ON si.iata = ANY(@iatas::bpchar[]) AND + ((octet_length(h.hash)=1 AND si.prefix_1=h.hash) OR + (octet_length(h.hash)=2 AND si.prefix_2=h.hash) OR + (octet_length(h.hash) IN (4,8) AND si.prefix_4=substring(h.hash FROM 1 FOR 4))) +JOIN nodes n ON n.id=si.node_id +WHERE substring(n.public_key FROM 1 FOR octet_length(h.hash))=h.hash +ORDER BY si.iata, h.hash, n.id; + +-- name: GetTopologyLinks :many +-- Reduce route history to unique adjacent pairs before transferring it to a browser. +WITH pairs AS ( + SELECT h.node_id AS a, r.node_ids[(h.ord+1)::int] AS b + FROM known_routes r CROSS JOIN LATERAL unnest(r.node_ids) WITH ORDINALITY h(node_id,ord) + WHERE r.last_seen >= @since::timestamptz AND r.last_seen < @until::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]),0)=0 OR r.iata=ANY(@iatas::bpchar[])) + AND h.ord < cardinality(r.node_ids) +) +SELECT LEAST(a,b)::uuid AS from_id, GREATEST(a,b)::uuid AS to_id +FROM pairs WHERE a IS NOT NULL AND b IS NOT NULL AND a<>b +GROUP BY LEAST(a,b), GREATEST(a,b) +ORDER BY LEAST(a,b), GREATEST(a,b) +LIMIT @link_limit::int; diff --git a/db/route_evidence.go b/db/route_evidence.go index e7017ea5..f31f50c0 100644 --- a/db/route_evidence.go +++ b/db/route_evidence.go @@ -39,6 +39,16 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap if c.IATA != iata || c.PathKey != key || !c.Since.Equal(query.Since) || !c.Until.Equal(query.Until) || c.ID <= 0 || c.HeardAt.Before(query.Since) || !c.HeardAt.Before(query.Until) { return nil, api.ErrRouteEvidenceInput } + if c.HashSize != 0 || c.PathBytes != "" { + if (query.HashSize != 0 || query.PathBytes != "") && (query.HashSize != c.HashSize || query.PathBytes != c.PathBytes) { + return nil, api.ErrRouteEvidenceInput + } + query.HashSize, query.PathBytes = c.HashSize, c.PathBytes + } + } + pinned := query.HashSize != 0 || query.PathBytes != "" + if pinned && !api.ValidRouteEvidencePath(query.HashSize, query.PathBytes) { + return nil, api.ErrRouteEvidenceInput } keyBytes, _ := hex.DecodeString(key) row, err := s.q.GetRouteEvidenceRoute(ctx, sqlc.GetRouteEvidenceRouteParams{Iata: iata, PathKey: keyBytes}) @@ -49,10 +59,35 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap if err != nil { return nil, err } + width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) + if pinned { + selected, _ := hex.DecodeString(query.PathBytes) + hops := len(selected) / int(query.HashSize) + if hops != len(row.NodeIds) || hops != int(row.HopCount) { + return nil, api.ErrRouteEvidenceInput + } + prefixes := make([][]byte, hops) + for i, id := range row.NodeIds { + prefixes[i] = selected[i*int(query.HashSize) : (i+1)*int(query.HashSize)] + // A changed representation must still belong to this node chain. + // Reuse the already-loaded keys; never widen the evidence query. + if !valid || width != query.HashSize || !bytes.Equal(path, selected) { + node := nodes[id] + if node == nil { + return nil, api.ErrRouteEvidenceInput + } + pubkey, err := hex.DecodeString(node.PublicKey) + if err != nil || !bytes.HasPrefix(pubkey, prefixes[i]) { + return nil, api.ErrRouteEvidenceInput + } + } + } + row.HashPrefix = prefixes + width, path, valid = query.HashSize, selected, true + } route := toKnownRoutes([]knownRouteRow{{ID: row.ID, NodeIds: row.NodeIds, HashPrefix: row.HashPrefix, Iata: row.Iata, HopCount: row.HopCount, FirstSeen: row.FirstSeen, LastSeen: row.LastSeen, ObservationCount: row.ObservationCount}}, nodes)[0] route.PathKey = hex.EncodeToString(row.PathKey) out := &api.RouteEvidence{Items: []api.RouteObservation{}, Route: route, WindowStart: query.Since.UnixMilli(), WindowEnd: query.Until.UnixMilli(), GeneratedAt: time.Now().UnixMilli(), MatchType: "saved_path_prefixes"} - width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) if !valid { return out, nil } @@ -86,7 +121,7 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap } if out.HasMore { last := rows[len(rows)-1] - cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID}).String() + cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID, HashSize: width, PathBytes: out.PathBytes}).String() out.NextPageCursor = &cursor } return out, nil diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index a84b534f..23abc34b 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -36,12 +36,13 @@ func TestRouteEvidencePostgres(t *testing.T) { applyBaseline(t, ctx, tx) _, err := tx.Exec(ctx, ` INSERT INTO observers(id,public_key,display_name) VALUES ('00000000-0000-0000-0000-000000000001','\x01','One'),('00000000-0000-0000-0000-000000000002','\x02','Two'); - INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa',2,'A'),('00000000-0000-0000-0000-000000000002','\xbb',2,'B'); + INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa0102',2,'A'),('00000000-0000-0000-0000-000000000002','\xbb0304',2,'B'); INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(1,10) i;`) if err != nil { t.Fatal(err) } + capture := &evidenceCapture{Tx: tx} store := &Store{q: sqlc.New(capture)} ids := []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000001"), uuid.MustParse("00000000-0000-0000-0000-000000000002")} @@ -80,6 +81,36 @@ func TestRouteEvidencePostgres(t *testing.T) { if err != nil { t.Fatal(err) } + if query.Cursor.HashSize != 1 || query.Cursor.PathBytes != "aabb" { + t.Fatalf("cursor did not pin the original bytes: %+v", query.Cursor) + } + // A new representation arrives while page one is open. New reads follow it, + // but pagination and copied links retain the exact original representation. + if _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(12,14) i`); err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,path_length_byte) + SELECT i,int4send(i),'00000000-0000-0000-0000-000000000001','YOW',$1,2,2,'\xaa01bb03',4,66 FROM generate_series(12,14) i`, at) + if err != nil { + t.Fatal(err) + } + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa, 1}, {0xbb, 3}}, "YOW", 2); err != nil { + t.Fatal(err) + } + freshQuery := api.RouteEvidenceQuery{Since: since, Until: until, Limit: 2} + fresh, err := store.GetRouteEvidence(ctx, "YOW", key, freshQuery) + if err != nil || fresh.HashSize != 2 || fresh.PathBytes != "aa01bb03" || len(fresh.Items) != 2 || fresh.Items[0].ID != 14 || fresh.Route.Hops[0].HashBytes != "aa01" { + t.Fatalf("new representation lost: %+v %v", fresh, err) + } + if fresh.Route.ID != first.Route.ID || fresh.Route.PathKey != first.Route.PathKey || fresh.Route.FirstSeen != first.Route.FirstSeen || fresh.Route.ObservationCount != first.Route.ObservationCount+1 { + t.Fatalf("route identity/counters changed: %+v", fresh.Route) + } + shared := freshQuery + shared.HashSize, shared.PathBytes = first.HashSize, first.PathBytes + copyPage, err := store.GetRouteEvidence(ctx, "YOW", key, shared) + if err != nil || len(copyPage.Items) != 2 || copyPage.Items[0].ID != 3 || copyPage.Route.Hops[0].HashBytes != "aa" { + t.Fatalf("shared representation changed: %+v %v", copyPage, err) + } second, err := store.GetRouteEvidence(ctx, "YOW", key, query) if err != nil { t.Fatal(err) @@ -90,6 +121,44 @@ func TestRouteEvidencePostgres(t *testing.T) { if _, err = json.Marshal(second); err != nil { t.Fatalf("non-finite JSON: %v", err) } + for _, prefixes := range [][][]byte{{{0xaa, 1, 2}, {0xbb, 3, 4}}, {{0xaa, 1, 2}, {0xbb, 3, 4}}} { + if err := store.UpsertKnownRoute(ctx, ids, prefixes, "YOW", 2); err != nil { + t.Fatal(err) + } + } + thirdWidth, err := store.GetRouteEvidence(ctx, "YOW", key, freshQuery) + if err != nil || thirdWidth.HashSize != 3 || thirdWidth.PathBytes != "aa0102bb0304" || len(thirdWidth.Items) != 0 || thirdWidth.Route.ID != first.Route.ID || thirdWidth.Route.ObservationCount != first.Route.ObservationCount+3 { + t.Fatalf("third width or repeated upsert: %+v %v", thirdWidth, err) + } + // Missing identities or mismatched bytes cannot make a pinned query search + // a different chain. Prefix collisions likewise never widen the SQL match. + wrong := shared + wrong.PathBytes = "aacc" + if _, err := store.GetRouteEvidence(ctx, "YOW", key, wrong); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatalf("unrelated path accepted: %v", err) + } + wrong.PathBytes = "aabbcc" + if _, err := store.GetRouteEvidence(ctx, "YOW", key, wrong); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatalf("wrong hop count accepted: %v", err) + } + _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000003','\xaa9988',2,'Collision'); + DELETE FROM nodes WHERE id='00000000-0000-0000-0000-000000000001'`) + if err != nil { + t.Fatal(err) + } + if _, err := store.GetRouteEvidence(ctx, "YOW", key, shared); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatalf("missing node substituted by colliding node: %v", err) + } + if _, err = tx.Exec(ctx, `INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa0102',2,'A')`); err != nil { + t.Fatal(err) + } + copyPage, err = store.GetRouteEvidence(ctx, "YOW", key, shared) + if err != nil || len(copyPage.Items) != 2 || copyPage.Items[0].ID != 3 { + t.Fatalf("collision broadened evidence: %+v %v", copyPage, err) + } + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } query.Cursor.IATA = "YVR" if _, err = store.GetRouteEvidence(ctx, "YOW", key, query); !errors.Is(err, api.ErrRouteEvidenceInput) { t.Fatal("cross-route cursor accepted") diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index b34664f2..0fc75cb6 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -source db/sqlc/querier.go -destination db/sqlc/mock/querier.go -package mockdb +// mockgen -source=db/sqlc/querier.go -destination=db/sqlc/mock/querier.go -package=mockdb // // Package mockdb is a generated GoMock package. @@ -1137,6 +1137,21 @@ func (mr *MockQuerierMockRecorder) GetTopNodes(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopNodes", reflect.TypeOf((*MockQuerier)(nil).GetTopNodes), ctx, arg) } +// GetTopologyLinks mocks base method. +func (m *MockQuerier) GetTopologyLinks(ctx context.Context, arg db.GetTopologyLinksParams) ([]db.GetTopologyLinksRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTopologyLinks", ctx, arg) + ret0, _ := ret[0].([]db.GetTopologyLinksRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTopologyLinks indicates an expected call of GetTopologyLinks. +func (mr *MockQuerierMockRecorder) GetTopologyLinks(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopologyLinks", reflect.TypeOf((*MockQuerier)(nil).GetTopologyLinks), ctx, arg) +} + // GetTransportScopeByName mocks base method. func (m *MockQuerier) GetTransportScopeByName(ctx context.Context, name string) (int32, error) { m.ctrl.T.Helper() @@ -1896,6 +1911,21 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } +// ResolveTraceHashes mocks base method. +func (m *MockQuerier) ResolveTraceHashes(ctx context.Context, arg db.ResolveTraceHashesParams) ([]db.ResolveTraceHashesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveTraceHashes", ctx, arg) + ret0, _ := ret[0].([]db.ResolveTraceHashesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveTraceHashes indicates an expected call of ResolveTraceHashes. +func (mr *MockQuerierMockRecorder) ResolveTraceHashes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveTraceHashes", reflect.TypeOf((*MockQuerier)(nil).ResolveTraceHashes), ctx, arg) +} + // RollAdvertHearings mocks base method. func (m *MockQuerier) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index f3c54d3c..c6338c18 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -156,6 +156,8 @@ type Querier interface { // Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; // node_id is NULL once the node row is gone. iata is a representative one. GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]GetTopNodesRow, error) + // Reduce route history to unique adjacent pairs before transferring it to a browser. + GetTopologyLinks(ctx context.Context, arg GetTopologyLinksParams) ([]GetTopologyLinksRow, error) GetTransportScopeByName(ctx context.Context, name string) (int32, error) GetTransportScopes(ctx context.Context) ([]GetTransportScopesRow, error) // ============================================================ @@ -314,6 +316,9 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + // TRACE can terminate at companions. Preserve the full prefix (including eight + // bytes), and return every candidate across heard regions instead of picking one. + ResolveTraceHashes(ctx context.Context, arg ResolveTraceHashesParams) ([]ResolveTraceHashesRow, error) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error // Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 46b13b50..fcc503fc 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1405,6 +1405,8 @@ func (q *Queries) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPa const getPacketsByTraceTag = `-- name: GetPacketsByTraceTag :many SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, + p.raw_payload, p.payload_version, + EXISTS(SELECT 1 FROM packet_observations po WHERE po.packet_hash=p.packet_hash AND po.path_length_byte>=64) AS invalid_snr_path, p.route_type, p.first_heard_at, p.last_heard_at, @@ -1425,14 +1427,17 @@ ORDER BY p.first_heard_at ASC ` type GetPacketsByTraceTagRow struct { - PacketHashHex string `json:"packet_hash_hex"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ParsedPayload []byte `json:"parsed_payload"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - Iatas []string `json:"iatas"` + PacketHashHex string `json:"packet_hash_hex"` + RawPayload []byte `json:"raw_payload"` + PayloadVersion int16 `json:"payload_version"` + InvalidSnrPath bool `json:"invalid_snr_path"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ParsedPayload []byte `json:"parsed_payload"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + Iatas []string `json:"iatas"` } // Return distinct observation IATAs in first-heard order for path resolution, @@ -1448,6 +1453,9 @@ func (q *Queries) GetPacketsByTraceTag(ctx context.Context, decode string) ([]Ge var i GetPacketsByTraceTagRow if err := rows.Scan( &i.PacketHashHex, + &i.RawPayload, + &i.PayloadVersion, + &i.InvalidSnrPath, &i.RouteType, &i.FirstHeardAt, &i.LastHeardAt, @@ -2229,6 +2237,59 @@ func (q *Queries) GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]Get return items, nil } +const getTopologyLinks = `-- name: GetTopologyLinks :many +WITH pairs AS ( + SELECT h.node_id AS a, r.node_ids[(h.ord+1)::int] AS b + FROM known_routes r CROSS JOIN LATERAL unnest(r.node_ids) WITH ORDINALITY h(node_id,ord) + WHERE r.last_seen >= $2::timestamptz AND r.last_seen < $3::timestamptz + AND (COALESCE(cardinality($4::bpchar[]),0)=0 OR r.iata=ANY($4::bpchar[])) + AND h.ord < cardinality(r.node_ids) +) +SELECT LEAST(a,b)::uuid AS from_id, GREATEST(a,b)::uuid AS to_id +FROM pairs WHERE a IS NOT NULL AND b IS NOT NULL AND a<>b +GROUP BY LEAST(a,b), GREATEST(a,b) +ORDER BY LEAST(a,b), GREATEST(a,b) +LIMIT $1::int +` + +type GetTopologyLinksParams struct { + LinkLimit int32 `json:"link_limit"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetTopologyLinksRow struct { + FromID uuid.UUID `json:"from_id"` + ToID uuid.UUID `json:"to_id"` +} + +// Reduce route history to unique adjacent pairs before transferring it to a browser. +func (q *Queries) GetTopologyLinks(ctx context.Context, arg GetTopologyLinksParams) ([]GetTopologyLinksRow, error) { + rows, err := q.db.Query(ctx, getTopologyLinks, + arg.LinkLimit, + arg.Since, + arg.Until, + arg.Iatas, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetTopologyLinksRow{} + for rows.Next() { + var i GetTopologyLinksRow + if err := rows.Scan(&i.FromID, &i.ToID); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getTransportScopeByName = `-- name: GetTransportScopeByName :one SELECT id FROM transport_scopes WHERE name = $1 ` @@ -4137,7 +4198,11 @@ SELECT WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, COALESCE(t.trace_type, '')::text AS trace_type, - t.best_payload + t.best_payload, + ARRAY(SELECT ti.iata FROM trace_iatas ti WHERE ti.trace_tag=t.trace_tag)::bpchar[] AS iatas, + EXISTS(SELECT 1 FROM packets p WHERE p.trace_tag=t.trace_tag AND p.payload_version<>0) AS unsupported_version, + EXISTS(SELECT 1 FROM packets p WHERE p.trace_tag=t.trace_tag AND p.route_type NOT IN (2,3)) AS non_direct, + EXISTS(SELECT 1 FROM packets p JOIN packet_observations po ON po.packet_hash=p.packet_hash WHERE p.trace_tag=t.trace_tag AND po.path_length_byte>=64) AS invalid_snr_path FROM trace_tags t WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) @@ -4165,13 +4230,17 @@ type ListTraceTagsParams struct { } type ListTraceTagsRow struct { - TraceTag string `json:"trace_tag"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - PacketCount int64 `json:"packet_count"` - IataCount int64 `json:"iata_count"` - TraceType string `json:"trace_type"` - BestPayload []byte `json:"best_payload"` + TraceTag string `json:"trace_tag"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + PacketCount int64 `json:"packet_count"` + IataCount int64 `json:"iata_count"` + TraceType string `json:"trace_type"` + BestPayload []byte `json:"best_payload"` + Iatas []string `json:"iatas"` + UnsupportedVersion bool `json:"unsupported_version"` + NonDirect bool `json:"non_direct"` + InvalidSnrPath bool `json:"invalid_snr_path"` } // ============================================================ @@ -4204,6 +4273,10 @@ func (q *Queries) ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([ &i.IataCount, &i.TraceType, &i.BestPayload, + &i.Iatas, + &i.UnsupportedVersion, + &i.NonDirect, + &i.InvalidSnrPath, ); err != nil { return nil, err } @@ -4856,6 +4929,63 @@ func (q *Queries) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashes return items, nil } +const resolveTraceHashes = `-- name: ResolveTraceHashes :many +SELECT h.hash::bytea AS hash, si.iata, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM unnest($1::bytea[]) AS h(hash) +JOIN node_short_ids si ON si.iata = ANY($2::bpchar[]) AND + ((octet_length(h.hash)=1 AND si.prefix_1=h.hash) OR + (octet_length(h.hash)=2 AND si.prefix_2=h.hash) OR + (octet_length(h.hash) IN (4,8) AND si.prefix_4=substring(h.hash FROM 1 FOR 4))) +JOIN nodes n ON n.id=si.node_id +WHERE substring(n.public_key FROM 1 FOR octet_length(h.hash))=h.hash +ORDER BY si.iata, h.hash, n.id +` + +type ResolveTraceHashesParams struct { + Hashes [][]byte `json:"hashes"` + Iatas []string `json:"iatas"` +} + +type ResolveTraceHashesRow struct { + Hash []byte `json:"hash"` + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` + PublicKey []byte `json:"public_key"` +} + +// TRACE can terminate at companions. Preserve the full prefix (including eight +// bytes), and return every candidate across heard regions instead of picking one. +func (q *Queries) ResolveTraceHashes(ctx context.Context, arg ResolveTraceHashesParams) ([]ResolveTraceHashesRow, error) { + rows, err := q.db.Query(ctx, resolveTraceHashes, arg.Hashes, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ResolveTraceHashesRow{} + for rows.Next() { + var i ResolveTraceHashesRow + if err := rows.Scan( + &i.Hash, + &i.Iata, + &i.NodeID, + &i.Name, + &i.Latitude, + &i.Longitude, + &i.PublicKey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const saveChannelCatalogue = `-- name: SaveChannelCatalogue :exec INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) VALUES ($1, $2, $3::jsonb, $4::text, diff --git a/db/topology.go b/db/topology.go new file mode 100644 index 00000000..24c45cc8 --- /dev/null +++ b/db/topology.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "context" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "time" +) + +func (s *Store) GetTopologyLinks(ctx context.Context, iatas []string, since, until time.Time) (*api.TopologyLinks, error) { + const limit = 100000 + rows, err := s.q.GetTopologyLinks(ctx, sqlc.GetTopologyLinksParams{Iatas: iatas, Since: pgtype.Timestamptz{Time: since, Valid: true}, Until: pgtype.Timestamptz{Time: until, Valid: true}, LinkLimit: limit + 1}) + if err != nil { + return nil, err + } + result := &api.TopologyLinks{Links: make([][2]uuid.UUID, 0, min(len(rows), limit)), Capped: len(rows) > limit, Since: since.UnixMilli(), Until: until.UnixMilli()} + for _, r := range rows[:min(len(rows), limit)] { + result.Links = append(result.Links, [2]uuid.UUID{r.FromID, r.ToID}) + } + return result, nil +} diff --git a/db/topology_integration_test.go b/db/topology_integration_test.go new file mode 100644 index 00000000..58b4c579 --- /dev/null +++ b/db/topology_integration_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "context" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "testing" + "time" +) + +func TestTopologyLinksPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + if _, err = tx.Exec(ctx, "CREATE TEMP TABLE known_routes (LIKE public.known_routes INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + ids := []uuid.UUID{uuid.New(), uuid.New(), uuid.New()} + now := time.Now().UTC().Truncate(time.Minute) + // Thousands of duplicate/reversed routes transfer only their unique adjacent pairs. + _, err = tx.Exec(ctx, `INSERT INTO known_routes (path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) + SELECT decode(md5(i::text),'hex'),CASE WHEN i%2=0 THEN $1::uuid[] ELSE $2::uuid[] END,ARRAY['\xaa'::bytea,'\xbb'::bytea], + 'YKF',2,$3,$3 FROM generate_series(1,1000) i`, ids[:2], []uuid.UUID{ids[1], ids[0]}, now) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO known_routes(path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) VALUES + (decode(md5('other'),'hex'),$1,ARRAY['\xbb'::bytea,'\xcc'::bytea],'YOW',2,$3,$3), + (decode(md5('gap'),'hex'),$2,ARRAY['\xaa'::bytea,'\x00'::bytea,'\xcc'::bytea],'YKF',3,$3,$3), + (decode(md5('old'),'hex'),$1,ARRAY['\xbb'::bytea,'\xcc'::bytea],'YKF',2,$3-interval '25 hours',$3-interval '25 hours')`, ids[1:], []*uuid.UUID{&ids[0], nil, &ids[2]}, now) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + result, err := store.GetTopologyLinks(ctx, []string{"YKF"}, now.Add(-24*time.Hour), now.Add(time.Minute)) + if err != nil || result.Capped || len(result.Links) != 1 { + t.Fatalf("scoped: %+v %v", result, err) + } + all, err := store.GetTopologyLinks(ctx, nil, now.Add(-24*time.Hour), now.Add(time.Minute)) + if err != nil || len(all.Links) != 2 { + t.Fatalf("all regions: %+v %v", all, err) + } + if result.Since != now.Add(-24*time.Hour).UnixMilli() { + t.Fatal("window changed") + } +} diff --git a/db/trace_pagination_integration_test.go b/db/trace_pagination_integration_test.go index 90ed19d5..dc123a43 100644 --- a/db/trace_pagination_integration_test.go +++ b/db/trace_pagination_integration_test.go @@ -16,6 +16,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" "github.com/jackc/pgx/v5" ) @@ -89,7 +90,7 @@ func TestTraceTagPaginationPostgres(t *testing.T) { paths := map[string][]string{"a": {"aa", "bb", "cc"}, "b": {"bb", "11", "22", "33"}, "c": {"cc"}, "d": {"dd"}} snrs := map[string][]float32{"a": {1, 2, 3}, "b": {2, 3, 4, 5}, "c": {3}, "d": {4}} summary := func(tag string, first, last int, count, iatas int64, kind string) api.TraceTagSummary { - return api.TraceTagSummary{TraceTag: strings.Repeat(tag, 8), FirstHeardAt: at(first).UnixMilli(), LastHeardAt: at(last).UnixMilli(), PacketCount: count, IATACount: iatas, TraceType: kind, PathHashes: paths[tag], SNRValues: snrs[tag]} + return api.TraceTagSummary{Quality: tracequality.Quality{Status: "suspect", Reasons: []string{"non_direct_trace"}}, TraceTag: strings.Repeat(tag, 8), FirstHeardAt: at(first).UnixMilli(), LastHeardAt: at(last).UnixMilli(), PacketCount: count, IATACount: iatas, TraceType: kind, PathHashes: paths[tag], SNRValues: snrs[tag]} } a, b, c, d := summary("a", 1, 40, 3, 2, "TRACE"), summary("b", 4, 30, 2, 1, "TRACE"), summary("c", 6, 20, 1, 1, "TRACE"), summary("d", 7, 15, 1, 1, "PING") for _, tc := range []struct { diff --git a/db/trace_quality_integration_test.go b/db/trace_quality_integration_test.go new file mode 100644 index 00000000..d623ab6d --- /dev/null +++ b/db/trace_quality_integration_test.go @@ -0,0 +1,77 @@ +package db + +import ( + "context" + "encoding/hex" + "os" + "testing" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +func TestTraceQualityPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx := context.Background() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(ctx) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + for _, name := range []string{"nodes", "node_short_ids"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+name+" (LIKE public."+name+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` + INSERT INTO nodes (id,public_key,node_type,name) VALUES + ('00000000-0000-0000-0000-000000000001',decode('aabbccdd11223344'||repeat('00',24),'hex'),1,'Companion'), + ('00000000-0000-0000-0000-000000000002',decode('aabbccdd55667788'||repeat('00',24),'hex'),2,'Repeater'); + INSERT INTO node_short_ids (node_id,iata,prefix_4) VALUES + ('00000000-0000-0000-0000-000000000001','YKF','\xaabbccdd'), + ('00000000-0000-0000-0000-000000000002','YKF','\xaabbccdd'), + ('00000000-0000-0000-0000-000000000001','YHM','\xaabbccdd');`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + for _, width := range []int{1, 2, 4, 8} { + full := "aabbccdd11223344" + h := full[:width*2] + b, _ := hex.DecodeString(h) + candidates, err := store.traceCandidates(ctx, []string{"YKF", "YHM"}, [][]byte{b}) + if err != nil { + t.Fatal(err) + } + flags := byte(0) + switch width { + case 2: + flags = 1 + case 4: + flags = 2 + case 8: + flags = 3 + } + route := traceRoute(&tracePayload{Flags: flags, PathHashes: []string{h}}, []string{"YKF", "YHM"}, candidates) + count := 2 + confidence := "ambiguous" + if width == 8 { + count = 1 + confidence = "high" + } + if len(route) != 1 || len(route[0].Nodes) != count || route[0].Confidence != confidence { + t.Fatalf("width %d: %+v", width, route) + } + if route[0].Nodes[0].PublicKey[:16] != "aabbccdd11223344" { + t.Fatal("companion excluded or eight-byte hash truncated") + } + } +} diff --git a/db/traces.go b/db/traces.go index 93ed18cd..cc933b11 100644 --- a/db/traces.go +++ b/db/traces.go @@ -8,6 +8,8 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" + "github.com/google/uuid" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -17,9 +19,11 @@ import ( ) type tracePayload struct { - PathHashes []string `json:"pathHashes"` - Flags byte `json:"flags"` - SNRValues []float32 `json:"snrValues"` + Raw string `json:"raw"` + Quality tracequality.Quality `json:"quality"` + PathHashes []string `json:"pathHashes"` + Flags byte `json:"flags"` + SNRValues []float32 `json:"snrValues"` } func (s *Store) RecordTrace(ctx context.Context, h ingest.TraceHearing) error { @@ -71,12 +75,41 @@ func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceT return nil, err } items := make([]api.TraceTagSummary, 0, len(rows)) + payloads := make([]tracePayload, 0, len(rows)) + allIATAs := map[string]bool{} + allHashes := map[string][]byte{} for _, r := range rows { var best tracePayload if len(r.BestPayload) > 0 { _ = json.Unmarshal(r.BestPayload, &best) } + quality := tracequality.Stored(best.Raw, best.Flags, best.PathHashes, best.SNRValues) + for _, reason := range best.Quality.Reasons { + if reason != "ambiguous_prefix" { + quality.Add(reason) + } + } + if r.UnsupportedVersion { + quality.Add("unsupported_version") + } + if r.InvalidSnrPath { + quality.Add("invalid_snr_path") + } + if r.NonDirect { + quality.Add("non_direct_trace") + } + payloads = append(payloads, best) + if quality.Status == "supported" { + for _, iata := range r.Iatas { + allIATAs[iata] = true + } + for _, h := range best.PathHashes { + b, _ := hex.DecodeString(h) + allHashes[h] = b + } + } items = append(items, api.TraceTagSummary{ + Quality: quality, TraceTag: r.TraceTag, FirstHeardAt: r.FirstHeardAt.Time.UnixMilli(), LastHeardAt: r.LastHeardAt.Time.UnixMilli(), @@ -87,6 +120,30 @@ func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceT SNRValues: best.SNRValues, }) } + // One bounded candidate query for the entire page, including all hash widths. + regions := make([]string, 0, len(allIATAs)) + for iata := range allIATAs { + regions = append(regions, iata) + } + hashes := make([][]byte, 0, len(allHashes)) + for _, h := range allHashes { + hashes = append(hashes, h) + } + candidates, err := s.traceCandidates(ctx, regions, hashes) + if err != nil { + return nil, err + } + for i := range items { + if items[i].Quality.Status != "supported" { + continue + } + for _, hop := range traceRoute(&payloads[i], rows[i].Iatas, candidates) { + if hop.Confidence == "ambiguous" { + items[i].Quality.Add("ambiguous_prefix") + break + } + } + } return items, nil } @@ -125,76 +182,94 @@ func (s *Store) GetTraceByTag(ctx context.Context, tag string) (*api.TraceDetail } packet.RawPath = rawPath } - if len(r.Iatas) > 0 { + packet.Quality = tracequality.Assess(r.RawPayload, int(r.PayloadVersion), int(r.RouteType), len(parsed.SNRValues)) + if r.InvalidSnrPath { + packet.Quality.Add("invalid_snr_path") + } + stored := tracequality.Stored(parsed.Raw, parsed.Flags, parsed.PathHashes, parsed.SNRValues) + for _, reason := range stored.Reasons { + packet.Quality.Add(reason) + } + if len(r.Iatas) > 0 && packet.Quality.Status == "supported" { packet.ResolvedRoute = s.resolveTraceRoute(ctx, &parsed, r.Iatas) } + for _, hop := range packet.ResolvedRoute { + if hop.Confidence == "ambiguous" { + packet.Quality.Add("ambiguous_prefix") + } + } detail.Packets = append(detail.Packets, packet) } return detail, nil } +// Candidates remain separated by hearing region until a trace is assembled. +// Never replace an ambiguous region with a convenient singleton from another. +type traceCandidateMap map[string]map[string][]api.ResolvedPathEntry + +func (s *Store) traceCandidates(ctx context.Context, iatas []string, hashes [][]byte) (traceCandidateMap, error) { + out := traceCandidateMap{} + if len(iatas) == 0 || len(hashes) == 0 { + return out, nil + } + rows, err := s.q.ResolveTraceHashes(ctx, sqlc.ResolveTraceHashesParams{Iatas: iatas, Hashes: hashes}) + if err != nil { + return nil, err + } + for _, r := range rows { + if out[r.Iata] == nil { + out[r.Iata] = map[string][]api.ResolvedPathEntry{} + } + h := hex.EncodeToString(r.Hash) + out[r.Iata][h] = append(out[r.Iata][h], api.ResolvedPathEntry{NodeID: r.NodeID, Name: r.Name, Latitude: r.Latitude, Longitude: r.Longitude, PublicKey: r.PublicKey}) + } + return out, nil +} + +func (s *Store) ResolveTracePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + candidates, err := s.traceCandidates(ctx, []string{iata}, hashes) + return candidates[iata], err +} + func (s *Store) resolveTraceRoute(ctx context.Context, payload *tracePayload, iatas []string) []api.ResolvedHop { - if payload == nil || len(payload.PathHashes) == 0 { + if payload == nil || tracequality.Stored(payload.Raw, payload.Flags, payload.PathHashes, payload.SNRValues).Status != "supported" { return nil } - hashSize := int(1 << (payload.Flags & 0x03)) hashes := make([][]byte, 0, len(payload.PathHashes)) for _, h := range payload.PathHashes { - b, err := hex.DecodeString(h) - if err == nil { - hashes = append(hashes, b) - } - } - confidenceRank := map[string]int{"none": 0, "ambiguous": 1, "high": 2} - type hopResult struct { - confidence string - entries []api.ResolvedPathEntry + b, _ := hex.DecodeString(h) + hashes = append(hashes, b) } - merged := make([]hopResult, len(hashes)) - for i := range merged { - merged[i] = hopResult{confidence: "none"} + candidates, err := s.traceCandidates(ctx, iatas, hashes) + if err != nil { + return nil } - for _, iata := range iatas { - resolved, err := s.ResolvePathHashes(ctx, iata, hashes) - if err != nil { - continue - } - for i, hash := range hashes { - key := hex.EncodeToString(hash[:hashSize]) - entries := resolved[key] - var confidence string - switch len(entries) { - case 0: - confidence = "none" - case 1: - confidence = "high" - default: - confidence = "ambiguous" - } - if confidenceRank[confidence] > confidenceRank[merged[i].confidence] { - merged[i] = hopResult{confidence: confidence, entries: entries} + return traceRoute(payload, iatas, candidates) +} + +func traceRoute(payload *tracePayload, iatas []string, candidates traceCandidateMap) []api.ResolvedHop { + route := make([]api.ResolvedHop, 0, len(payload.PathHashes)) + for i, h := range payload.PathHashes { + hop := api.ResolvedHop{Confidence: "none", Nodes: []api.ResolvedNode{}} + seen := map[uuid.UUID]bool{} + for _, iata := range iatas { + for _, e := range candidates[iata][h] { + if seen[e.NodeID] { + continue + } + seen[e.NodeID] = true + hop.Nodes = append(hop.Nodes, api.ResolvedNode{ID: e.NodeID, Name: e.Name, Latitude: e.Latitude, Longitude: e.Longitude, PublicKey: hex.EncodeToString(e.PublicKey)}) } } - } - route := make([]api.ResolvedHop, 0, len(hashes)) - for i, hr := range merged { - hop := api.ResolvedHop{ - Confidence: hr.confidence, - Nodes: make([]api.ResolvedNode, 0, len(hr.entries)), + if len(hop.Nodes) == 1 { + hop.Confidence = "high" + } else if len(hop.Nodes) > 1 { + hop.Confidence = "ambiguous" } if i < len(payload.SNRValues) { snr := payload.SNRValues[i] hop.SNR = &snr } - for _, e := range hr.entries { - hop.Nodes = append(hop.Nodes, api.ResolvedNode{ - ID: e.NodeID, - Name: e.Name, - Latitude: e.Latitude, - Longitude: e.Longitude, - PublicKey: hex.EncodeToString(e.PublicKey), - }) - } route = append(route, hop) } return route diff --git a/db/traces_integration_test.go b/db/traces_integration_test.go index 51305443..1d671afb 100644 --- a/db/traces_integration_test.go +++ b/db/traces_integration_test.go @@ -59,7 +59,7 @@ INSERT INTO node_short_ids (node_id,iata,prefix_4) VALUES ('00000000-0000-0000-0000-000000000001','YVR','\xaa000000'), ('00000000-0000-0000-0000-000000000002','YYJ','\xaa010000'); INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,trace_tag,parsed_payload,first_heard_at,last_heard_at) -SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,1,'\x00','\x00','\x01020304', +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,2,'\x010203040000000000aa','\x26','\x01020304', CASE WHEN i=100 THEN '{}'::jsonb ELSE '{"flags":0,"pathHashes":["aa"],"snrValues":[12.5]}'::jsonb END, '2026-01-01 00:00:00+00'::timestamptz+i*interval '1 second', '2026-01-01 00:00:00+00'::timestamptz+i*interval '1 second'+interval '1 second' FROM generate_series(1,100) i; @@ -89,7 +89,7 @@ ANALYZE packets; ANALYZE packet_observations; ANALYZE nodes; ANALYZE node_short_ anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) for i, p := range detail.Packets { number := i + 1 - if p.PacketHash != fmt.Sprintf("%064x", number) || p.RouteType != 1 || p.FirstHeardAt != anchor.Add(time.Duration(number)*time.Second).UnixMilli() || p.LastHeardAt != p.FirstHeardAt+1000 { + if p.PacketHash != fmt.Sprintf("%064x", number) || p.RouteType != 2 || p.FirstHeardAt != anchor.Add(time.Duration(number)*time.Second).UnixMilli() || p.LastHeardAt != p.FirstHeardAt+1000 { t.Fatalf("wrong packet fields or order at %d", number) } if number == 100 { @@ -109,7 +109,10 @@ ANALYZE packets; ANALYZE packet_observations; ANALYZE nodes; ANALYZE node_short_ } want := "YVR relay" if number == 1 { - want = "YYJ relay" + if len(p.ResolvedRoute) != 1 || p.ResolvedRoute[0].Confidence != "ambiguous" || len(p.ResolvedRoute[0].Nodes) != 2 || p.Quality.Status != "ambiguous" { + t.Fatal("conflicting regions must retain both candidates") + } + continue } if len(p.ResolvedRoute) != 1 || p.ResolvedRoute[0].Confidence != "high" || len(p.ResolvedRoute[0].Nodes) != 1 || p.ResolvedRoute[0].Nodes[0].Name == nil || *p.ResolvedRoute[0].Nodes[0].Name != want { t.Fatalf("wrong region precedence/resolution at %d", number) @@ -122,8 +125,8 @@ ANALYZE packets; ANALYZE packet_observations; ANALYZE nodes; ANALYZE node_short_ if counter.observations != 0 { t.Errorf("extra observation queries: %d; want 0", counter.observations) } - if counter.queries != 100 { - t.Errorf("got %d queries; want one packet query and 99 unchanged path-resolution queries", counter.queries) + if counter.queries != 99 { + t.Errorf("got %d queries; want one packet query and 98 batched path-resolution queries", counter.queries) } counter.queries, counter.observations = 0, 0 missing, err := store.GetTraceByTag(ctx, "00000000") diff --git a/db/traces_test.go b/db/traces_test.go index 0e3d4b48..ef0a4410 100644 --- a/db/traces_test.go +++ b/db/traces_test.go @@ -99,7 +99,7 @@ func TestGetTraceByTag_WithPacket(t *testing.T) { firstHeard := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} lastHeard := pgtype.Timestamptz{Time: time.UnixMilli(1700000001000), Valid: true} // aabbccdd is valid hex for the packet hash - parsedPayload := []byte(`{"pathHashes":["aabb"],"snrValues":[15.0],"flags":0}`) + parsedPayload := []byte(`{"pathHashes":["aabb"],"snrValues":[15.0],"flags":1}`) scopeName := "default" mock.EXPECT(). @@ -107,7 +107,8 @@ func TestGetTraceByTag_WithPacket(t *testing.T) { Return([]sqlc.GetPacketsByTraceTagRow{ { PacketHashHex: "aabbccdd", - RouteType: 1, + RouteType: 2, + RawPayload: []byte{0, 0, 0, 0, 0, 0, 0, 0, 1, 0xaa, 0xbb}, ScopeName: &scopeName, FirstHeardAt: firstHeard, LastHeardAt: lastHeard, @@ -117,11 +118,11 @@ func TestGetTraceByTag_WithPacket(t *testing.T) { }, nil) mock.EXPECT(). - ResolvePathHashesP2(gomock.Any(), sqlc.ResolvePathHashesP2Params{ - Iata: "YVR", - Column2: [][]byte{{0xaa, 0xbb}}, + ResolveTraceHashes(gomock.Any(), sqlc.ResolveTraceHashesParams{ + Iatas: []string{"YVR"}, + Hashes: [][]byte{{0xaa, 0xbb}}, }). - Return([]sqlc.ResolvePathHashesP2Row{}, nil) + Return([]sqlc.ResolveTraceHashesRow{}, nil) store := &Store{q: mock} detail, err := store.GetTraceByTag(context.Background(), "trace-001") diff --git a/docs/docs.go b/docs/docs.go index 6fc5abce..328e3d24 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2179,9 +2179,55 @@ const docTemplate = `{ } } }, + "/routes/topology": { + "get": { + "description": "Returns up to 100000 undirected node-ID pairs, deduplicated across known routes. Minute-aligned window includes the current minute. Existing rate limits apply; no per-node lookup or route pagination is needed. Capped is true when more links exist.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Unique adjacent route links for a topology snapshot", + "parameters": [ + { + "type": "string", + "description": "History window: 15m (default), 1h, or 24h", + "name": "window", + "in": "query" + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/routes/{iata}/{pathKey}/observations": { "get": { - "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.", "produces": [ "application/json" ], @@ -2222,9 +2268,23 @@ const docTemplate = `{ "name": "until", "in": "query" }, + { + "maximum": 3, + "minimum": 1, + "type": "integer", + "description": "Exact hash width (1–3); provide with pathBytes to pin a response's representation", + "name": "hashSize", + "in": "query" + }, + { + "type": "string", + "description": "Complete lower-case hex path from the response; provide with hashSize", + "name": "pathBytes", + "in": "query" + }, { "type": "string", - "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "description": "Opaque precise cursor from nextPageCursor; route, window and path are pinned", "name": "pageCursor", "in": "query" }, @@ -2265,6 +2325,29 @@ const docTemplate = `{ } } }, + "/scope-catalogues": { + "get": { + "description": "Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", + "produces": [ + "application/json" + ], + "tags": [ + "Scopes" + ], + "summary": "Cached MeshMapper regional scope catalogues", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue" + } + } + } + } + } + }, "/scopes": { "get": { "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", @@ -4546,6 +4629,9 @@ const docTemplate = `{ "description": "matched transport scope name e.g. \"#bc\"", "type": "string" }, + "traceQuality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "transportCodes": { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketTransportCodes" } @@ -4677,6 +4763,13 @@ const docTemplate = `{ }, "sourceBroker": { "type": "string" + }, + "traceQuality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, + "wirePathBytes": { + "description": "Exact on-air TRACE SNR bytes, before the display path substitution.", + "type": "string" } } }, @@ -5595,6 +5688,29 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks": { + "type": "object", + "properties": { + "capped": { + "type": "boolean" + }, + "links": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "since": { + "type": "integer" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.TraceDetail": { "type": "object", "properties": { @@ -5626,6 +5742,9 @@ const docTemplate = `{ "description": "hex-encoded packet hash", "type": "string" }, + "quality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "rawPath": { "description": "hops as received in the packet", "type": "array", @@ -5680,6 +5799,9 @@ const docTemplate = `{ "type": "string" } }, + "quality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "snrValues": { "description": "SNR per hop from the most complete observation", "type": "array", @@ -5736,6 +5858,76 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { + "type": "object", + "properties": { + "default": { + "type": "integer" + }, + "monitored": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "wardriving": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue": { + "type": "object", + "properties": { + "checkedAt": { + "type": "integer" + }, + "freshUntil": { + "type": "integer" + }, + "generatedAt": { + "type": "integer" + }, + "iata": { + "type": "string" + }, + "lastError": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "scoped": { + "type": "integer" + }, + "scopes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope" + } + }, + "url": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality": { + "type": "object", + "properties": { + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "description": "supported, suspect, or ambiguous; supported is not authenticated", + "type": "string" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 9deb9fdb..b7912233 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2177,9 +2177,55 @@ } } }, + "/routes/topology": { + "get": { + "description": "Returns up to 100000 undirected node-ID pairs, deduplicated across known routes. Minute-aligned window includes the current minute. Existing rate limits apply; no per-node lookup or route pagination is needed. Capped is true when more links exist.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Unique adjacent route links for a topology snapshot", + "parameters": [ + { + "type": "string", + "description": "History window: 15m (default), 1h, or 24h", + "name": "window", + "in": "query" + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/routes/{iata}/{pathKey}/observations": { "get": { - "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "description": "Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported.", "produces": [ "application/json" ], @@ -2220,9 +2266,23 @@ "name": "until", "in": "query" }, + { + "maximum": 3, + "minimum": 1, + "type": "integer", + "description": "Exact hash width (1–3); provide with pathBytes to pin a response's representation", + "name": "hashSize", + "in": "query" + }, + { + "type": "string", + "description": "Complete lower-case hex path from the response; provide with hashSize", + "name": "pathBytes", + "in": "query" + }, { "type": "string", - "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "description": "Opaque precise cursor from nextPageCursor; route, window and path are pinned", "name": "pageCursor", "in": "query" }, @@ -2263,6 +2323,29 @@ } } }, + "/scope-catalogues": { + "get": { + "description": "Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate.", + "produces": [ + "application/json" + ], + "tags": [ + "Scopes" + ], + "summary": "Cached MeshMapper regional scope catalogues", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue" + } + } + } + } + } + }, "/scopes": { "get": { "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", @@ -4544,6 +4627,9 @@ "description": "matched transport scope name e.g. \"#bc\"", "type": "string" }, + "traceQuality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "transportCodes": { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketTransportCodes" } @@ -4675,6 +4761,13 @@ }, "sourceBroker": { "type": "string" + }, + "traceQuality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, + "wirePathBytes": { + "description": "Exact on-air TRACE SNR bytes, before the display path substitution.", + "type": "string" } } }, @@ -5593,6 +5686,29 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks": { + "type": "object", + "properties": { + "capped": { + "type": "boolean" + }, + "links": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "since": { + "type": "integer" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.TraceDetail": { "type": "object", "properties": { @@ -5624,6 +5740,9 @@ "description": "hex-encoded packet hash", "type": "string" }, + "quality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "rawPath": { "description": "hops as received in the packet", "type": "array", @@ -5678,6 +5797,9 @@ "type": "string" } }, + "quality": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality" + }, "snrValues": { "description": "SNR per hop from the most complete observation", "type": "array", @@ -5734,6 +5856,76 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope": { + "type": "object", + "properties": { + "default": { + "type": "integer" + }, + "monitored": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "wardriving": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue": { + "type": "object", + "properties": { + "checkedAt": { + "type": "integer" + }, + "freshUntil": { + "type": "integer" + }, + "generatedAt": { + "type": "integer" + }, + "iata": { + "type": "string" + }, + "lastError": { + "type": "string" + }, + "repeaters": { + "type": "integer" + }, + "scoped": { + "type": "integer" + }, + "scopes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope" + } + }, + "url": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality": { + "type": "object", + "properties": { + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "status": { + "description": "supported, suspect, or ambiguous; supported is not authenticated", + "type": "string" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9844449c..9a14c74b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -788,6 +788,8 @@ definitions: scope: description: matched transport scope name e.g. "#bc" type: string + traceQuality: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality' transportCodes: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketTransportCodes' type: object @@ -887,6 +889,11 @@ definitions: type: number sourceBroker: type: string + traceQuality: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality' + wirePathBytes: + description: Exact on-air TRACE SNR bytes, before the display path substitution. + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.PacketObservationSummary: properties: @@ -1508,6 +1515,21 @@ definitions: senderName: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks: + properties: + capped: + type: boolean + links: + items: + items: + type: string + type: array + type: array + since: + type: integer + until: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TraceDetail: properties: packets: @@ -1530,6 +1552,8 @@ definitions: packetHash: description: hex-encoded packet hash type: string + quality: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality' rawPath: description: hops as received in the packet items: @@ -1569,6 +1593,8 @@ definitions: items: type: string type: array + quality: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality' snrValues: description: SNR per hop from the most complete observation items: @@ -1606,6 +1632,52 @@ definitions: requires_restart: type: boolean type: object + github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope: + properties: + default: + type: integer + monitored: + type: boolean + name: + type: string + repeaters: + type: integer + wardriving: + type: boolean + type: object + github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue: + properties: + checkedAt: + type: integer + freshUntil: + type: integer + generatedAt: + type: integer + iata: + type: string + lastError: + type: string + repeaters: + type: integer + scoped: + type: integer + scopes: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.CatalogueScope' + type: array + url: + type: string + type: object + github_com_MeshCore-Beacon_beacon-server_internal_tracequality.Quality: + properties: + reasons: + items: + type: string + type: array + status: + description: supported, suspect, or ambiguous; supported is not authenticated + type: string + type: object internal_api_handlers.APIError: properties: code: @@ -3018,11 +3090,13 @@ paths: - Routes /routes/{iata}/{pathKey}/observations: get: - description: Matches the saved IATA, complete path bytes and hash width. Excludes - TRACE and unclassified reports; other widths and search-result subsegments - are not included. Byte matches do not prove hop identities or delivery. The - route and its historical counter can outlive raw evidence. Cursors retain - full timestamp precision and pin route/window scope. + description: Matches one complete path representation in the saved IATA. Defaults + to the latest processed prefixes; hashSize/pathBytes pin a prior response. + Excludes TRACE and unclassified reports; widths are never combined. Pinned + prefixes must match the saved node chain or its current metadata. Byte matches + do not prove hop identities or delivery. The route counter can outlive raw + evidence. New cursors pin path, route, window and full timestamp precision; + legacy unpinned cursors remain supported. parameters: - description: Three-character IATA code in: path @@ -3048,7 +3122,20 @@ paths: in: query name: until type: integer - - description: Opaque precise cursor from nextPageCursor; window is pinned + - description: Exact hash width (1–3); provide with pathBytes to pin a response's + representation + in: query + maximum: 3 + minimum: 1 + name: hashSize + type: integer + - description: Complete lower-case hex path from the response; provide with + hashSize + in: query + name: pathBytes + type: string + - description: Opaque precise cursor from nextPageCursor; route, window and + path are pinned in: query name: pageCursor type: string @@ -3161,6 +3248,58 @@ paths: summary: Search known routes by source and destination hash tags: - Routes + /routes/topology: + get: + description: Returns up to 100000 undirected node-ID pairs, deduplicated across + known routes. Minute-aligned window includes the current minute. Existing + rate limits apply; no per-node lookup or route pagination is needed. Capped + is true when more links exist. + parameters: + - description: 'History window: 15m (default), 1h, or 24h' + in: query + name: window + type: string + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TopologyLinks' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Unique adjacent route links for a topology snapshot + tags: + - Routes + /scope-catalogues: + get: + description: Discovered sources only. Counts describe the source region, not + individual nodes or links. No upstream request is made. An empty list means + no configured importer; checkedAt=0 means no successful check. lastError or + a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence + are separate. + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_meshmapper.PublicScopeCatalogue' + type: array + summary: Cached MeshMapper regional scope catalogues + tags: + - Scopes /scopes: get: description: Without filters, lists every stored scope name, including imported diff --git a/internal/api/handlers/route_evidence.go b/internal/api/handlers/route_evidence.go index a194306c..3819cbec 100644 --- a/internal/api/handlers/route_evidence.go +++ b/internal/api/handlers/route_evidence.go @@ -17,7 +17,7 @@ import ( func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.RouteEvidenceQuery, error) { q := api.RouteEvidenceQuery{} values := r.URL.Query() - for _, name := range []string{"pageCursor", "since", "until", "range", "limit"} { + for _, name := range []string{"pageCursor", "since", "until", "range", "limit", "hashSize", "pathBytes"} { if len(values[name]) > 1 { return q, api.ErrRouteEvidenceInput } @@ -38,6 +38,7 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R q.Cursor = c q.Since = c.Since q.Until = c.Until + q.HashSize, q.PathBytes = c.HashSize, c.PathBytes } else { q.Until = now.UTC().Truncate(time.Millisecond) q.Since = q.Until.Add(-24 * time.Hour) @@ -67,6 +68,17 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R return q, api.ErrRouteEvidenceInput } } + if values.Has("hashSize") || values.Has("pathBytes") { + width, err := strconv.ParseInt(values.Get("hashSize"), 10, 16) + path := values.Get("pathBytes") + if err != nil || !api.ValidRouteEvidencePath(int16(width), path) || strconv.FormatInt(width, 10) != values.Get("hashSize") { + return q, api.ErrRouteEvidenceInput + } + if q.Cursor != nil && q.Cursor.HashSize != 0 && (q.Cursor.HashSize != int16(width) || q.Cursor.PathBytes != path) { + return q, api.ErrRouteEvidenceInput + } + q.HashSize, q.PathBytes = int16(width), path + } if !api.ValidRouteEvidenceWindow(q.Since, q.Until) || q.Until.After(now.Add(clockSkewTolerance)) { return q, api.ErrRouteEvidenceInput } @@ -76,7 +88,7 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R // getRouteEvidence godoc // // @Summary Get retained reports matching a full saved route prefix sequence -// @Description Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope. +// @Description Matches one complete path representation in the saved IATA. Defaults to the latest processed prefixes; hashSize/pathBytes pin a prior response. Excludes TRACE and unclassified reports; widths are never combined. Pinned prefixes must match the saved node chain or its current metadata. Byte matches do not prove hop identities or delivery. The route counter can outlive raw evidence. New cursors pin path, route, window and full timestamp precision; legacy unpinned cursors remain supported. // @Tags Routes // @Produce json // @Param iata path string true "Three-character IATA code" @@ -84,7 +96,9 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R // @Param range query string false "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor" // @Param since query int false "Window start epoch ms; provide with until (default last 24h)" // @Param until query int false "Exclusive window end epoch ms; maximum span 30d, end may be up to 5 minutes ahead of server time (clock skew tolerance)" -// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; window is pinned" +// @Param hashSize query int false "Exact hash width (1–3); provide with pathBytes to pin a response's representation" minimum(1) maximum(3) +// @Param pathBytes query string false "Complete lower-case hex path from the response; provide with hashSize" +// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; route, window and path are pinned" // @Param limit query int false "Default 50; positive, capped at 200" minimum(1) maximum(200) // @Success 200 {object} api.RouteEvidence // @Failure 400 {object} handlers.APIError @@ -100,7 +114,7 @@ func getRouteEvidence(reader api.Reader) http.HandlerFunc { } query, err := routeEvidenceQuery(r, iata, key, time.Now()) if err != nil { - respondError(w, 400, "invalid route evidence window, limit or page cursor") + respondError(w, 400, "invalid route evidence window, path, limit or page cursor") return } result, err := reader.GetRouteEvidence(r.Context(), iata, key, query) diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go index f2268d86..88f057df 100644 --- a/internal/api/handlers/route_evidence_test.go +++ b/internal/api/handlers/route_evidence_test.go @@ -85,3 +85,28 @@ func TestRouteEvidenceHandler(t *testing.T) { } } } + +func TestRouteEvidencePinnedQuery(t *testing.T) { + now := time.Now().UTC().Truncate(time.Millisecond) + q, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?hashSize=2&pathBytes=aa01bb02", nil), "YOW", evidenceKey, now) + if err != nil || q.HashSize != 2 || q.PathBytes != "aa01bb02" { + t.Fatalf("selected path: %+v %v", q, err) + } + c := api.RouteEvidenceCursor{IATA: "YOW", PathKey: evidenceKey, Since: q.Since, Until: q.Until, HeardAt: q.Since.Add(time.Hour), ID: 5, HashSize: q.HashSize, PathBytes: q.PathBytes} + encoded := url.QueryEscape(c.String()) + next, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+encoded, nil), "YOW", evidenceKey, now) + if err != nil || next.HashSize != 2 || next.PathBytes != q.PathBytes { + t.Fatalf("cursor path lost: %+v %v", next, err) + } + for _, query := range []string{ + "hashSize=1", "pathBytes=aabb", "hashSize=0&pathBytes=aabb", "hashSize=4&pathBytes=aa01bb02", + "hashSize=01&pathBytes=aabb", "hashSize=1&hashSize=2&pathBytes=aabb", + "hashSize=1&pathBytes=aabb&pathBytes=aabb", "hashSize=1&pathBytes=aa", "hashSize=1&pathBytes=AABB", + "pageCursor=" + encoded + "&hashSize=1&pathBytes=aabb", + "pageCursor=" + encoded + "&hashSize=2&pathBytes=aa02bb01", + } { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } +} diff --git a/internal/api/handlers/routes.go b/internal/api/handlers/routes.go index 86a7c861..48a147bf 100644 --- a/internal/api/handlers/routes.go +++ b/internal/api/handlers/routes.go @@ -20,6 +20,7 @@ import ( func RoutesRouter(reader api.Reader) http.Handler { r := chi.NewRouter() r.Get("/", listKnownRoutes(reader)) + r.Get("/topology", getTopologyLinks(reader)) r.Get("/cross", searchCrossIATARoutes(reader)) r.Get("/search", searchKnownRoutes(reader)) r.Get("/{iata}/{pathKey}/observations", getRouteEvidence(reader)) diff --git a/internal/api/handlers/scope_catalogues.go b/internal/api/handlers/scope_catalogues.go new file mode 100644 index 00000000..886a874a --- /dev/null +++ b/internal/api/handlers/scope_catalogues.go @@ -0,0 +1,33 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "net/http" + + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" +) + +// ScopeCatalogueSource supplies immutable, already-cached public metadata. +type ScopeCatalogueSource interface { + Catalogues() []meshmapper.PublicScopeCatalogue +} + +// ScopeCatalogues godoc +// @Summary Cached MeshMapper regional scope catalogues +// @Description Discovered sources only. Counts describe the source region, not individual nodes or links. No upstream request is made. An empty list means no configured importer; checkedAt=0 means no successful check. lastError or a past freshUntil marks stale metadata. Manual Beacon scopes and packet evidence are separate. +// @Tags Scopes +// @Produce json +// @Success 200 {array} meshmapper.PublicScopeCatalogue +// @Router /scope-catalogues [get] +func ScopeCatalogues(source ScopeCatalogueSource) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + items := []meshmapper.PublicScopeCatalogue{} + if source != nil { + items = source.Catalogues() + } + w.Header().Set("Cache-Control", "public, max-age=60") + respond(w, http.StatusOK, items) + } +} diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index efb5cb8f..4ed1ea0a 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -417,3 +417,7 @@ func (s stubReader) GetRouteEvidence(ctx context.Context, iata, key string, q ap } return nil, nil } + +func (s stubReader) GetTopologyLinks(_ context.Context, _ []string, _ time.Time, _ time.Time) (*api.TopologyLinks, error) { + return &api.TopologyLinks{}, nil +} diff --git a/internal/api/handlers/topology.go b/internal/api/handlers/topology.go new file mode 100644 index 00000000..b0fa1040 --- /dev/null +++ b/internal/api/handlers/topology.go @@ -0,0 +1,44 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "context" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "net/http" + "time" +) + +// getTopologyLinks godoc +// @Summary Unique adjacent route links for a topology snapshot +// @Description Returns up to 100000 undirected node-ID pairs, deduplicated across known routes. Minute-aligned window includes the current minute. Existing rate limits apply; no per-node lookup or route pagination is needed. Capped is true when more links exist. +// @Tags Routes +// @Produce json +// @Param window query string false "History window: 15m (default), 1h, or 24h" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Success 200 {object} api.TopologyLinks +// @Failure 400 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /routes/topology [get] +func getTopologyLinks(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + window := r.URL.Query().Get("window") + if window == "" { + window = "15m" + } + duration, ok := map[string]time.Duration{"15m": 15 * time.Minute, "1h": time.Hour, "24h": 24 * time.Hour}[window] + if !ok { + respondError(w, http.StatusBadRequest, "window must be 15m, 1h or 24h") + return + } + until := time.Now().UTC().Truncate(time.Minute).Add(time.Minute) + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + links, err := reader.GetTopologyLinks(ctx, parseIATAs(r), until.Add(-duration), until) + if err != nil { + respondError(w, http.StatusServiceUnavailable, "route snapshot unavailable; try again shortly") + return + } + respond(w, http.StatusOK, links) + } +} diff --git a/internal/api/handlers/topology_test.go b/internal/api/handlers/topology_test.go new file mode 100644 index 00000000..c4274dcf --- /dev/null +++ b/internal/api/handlers/topology_test.go @@ -0,0 +1,22 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "net/http/httptest" + "testing" +) + +func TestTopologyWindowValidation(t *testing.T) { + for _, window := range []string{"", "15m", "1h", "24h", "72h", "0", "garbage"} { + w := httptest.NewRecorder() + RoutesRouter(stubReader{}).ServeHTTP(w, httptest.NewRequest("GET", "/topology?window="+window, nil)) + want := 200 + if window == "72h" || window == "0" || window == "garbage" { + want = 400 + } + if w.Code != want { + t.Fatalf("%s: %d", window, w.Code) + } + } +} diff --git a/internal/api/packets.go b/internal/api/packets.go index e5f47fe8..f2fccb76 100644 --- a/internal/api/packets.go +++ b/internal/api/packets.go @@ -3,6 +3,8 @@ package api +import "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" + import ( "encoding/hex" "encoding/json" @@ -53,19 +55,21 @@ type PacketPathLength struct { // PacketObservationDetail is a full observation including radio settings and resolved path. type PacketObservationDetail struct { - ID int64 `json:"id"` - ObserverID uuid.UUID `json:"observerId"` - ObserverName *string `json:"observerName,omitempty"` - IATA string `json:"iata"` - HeardAt int64 `json:"heardAt"` // epoch ms - PathLength PacketPathLength `json:"pathLength"` - PathBytes *string `json:"pathBytes,omitempty"` // hex-encoded accumulated path hashes - RSSI *int16 `json:"rssi,omitempty"` - SNR *float32 `json:"snr,omitempty"` - PropagationTimeMs *int32 `json:"propagationTimeMs"` // ms since first observation; 0 for first - Radio *PacketRadio `json:"radio,omitempty"` - SourceBroker string `json:"sourceBroker"` - ResolvedPath []ResolvedHop `json:"resolvedPath"` // per-observation resolved path hashes + WirePathBytes *string `json:"wirePathBytes,omitempty"` // Exact on-air TRACE SNR bytes, before the display path substitution. + TraceQuality *tracequality.Quality `json:"traceQuality,omitempty"` + ID int64 `json:"id"` + ObserverID uuid.UUID `json:"observerId"` + ObserverName *string `json:"observerName,omitempty"` + IATA string `json:"iata"` + HeardAt int64 `json:"heardAt"` // epoch ms + PathLength PacketPathLength `json:"pathLength"` + PathBytes *string `json:"pathBytes,omitempty"` // hex-encoded accumulated path hashes + RSSI *int16 `json:"rssi,omitempty"` + SNR *float32 `json:"snr,omitempty"` + PropagationTimeMs *int32 `json:"propagationTimeMs"` // ms since first observation; 0 for first + Radio *PacketRadio `json:"radio,omitempty"` + SourceBroker string `json:"sourceBroker"` + ResolvedPath []ResolvedHop `json:"resolvedPath"` // per-observation resolved path hashes // ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type // carries one. Prefer the snapshot captured at ingest; legacy observations without a // snapshot use the current node registry. Endpoint matching itself is unchanged: @@ -134,6 +138,7 @@ type PacketTransportCodes struct { // Packet is the full packet representation including all observations and resolved paths. type Packet struct { + TraceQuality *tracequality.Quality `json:"traceQuality,omitempty"` PacketHash string `json:"packetHash"` Header PacketHeader `json:"header"` TransportCodes *PacketTransportCodes `json:"transportCodes,omitempty"` diff --git a/internal/api/reader.go b/internal/api/reader.go index dc71e63f..08a5f0ca 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -224,6 +224,8 @@ type Reader interface { // GetTraceByTag returns all packets for a given trace tag with resolved routes. GetTraceByTag(ctx context.Context, tag string) (*TraceDetail, error) + GetTopologyLinks(ctx context.Context, iatas []string, since, until time.Time) (*TopologyLinks, error) + // ListKnownRoutes returns known routes filtered by IATA and optional hop count. // A nonzero cursorID breaks ties within the cursor's millisecond. ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]KnownRoute, error) diff --git a/internal/api/route_evidence.go b/internal/api/route_evidence.go index d320ade5..490c91b2 100644 --- a/internal/api/route_evidence.go +++ b/internal/api/route_evidence.go @@ -50,16 +50,22 @@ type RouteEvidenceQuery struct { Until time.Time Cursor *RouteEvidenceCursor Limit int32 + // Optional exact representation copied from a prior response or cursor. + HashSize int16 + PathBytes string } -// RouteEvidenceCursor binds the precise (heard_at DESC,id DESC) boundary to its route/window. +// RouteEvidenceCursor pins the route, window and exact representation as well as +// the precise (heard_at DESC,id DESC) boundary. Legacy v1 cursors have no path pin. type RouteEvidenceCursor struct { - IATA string - PathKey string - Since time.Time - Until time.Time - HeardAt time.Time - ID int64 + IATA string + PathKey string + Since time.Time + Until time.Time + HeardAt time.Time + ID int64 + HashSize int16 + PathBytes string } func ValidRouteEvidenceKey(iata, key string) bool { @@ -79,20 +85,32 @@ func ValidRouteEvidenceWindow(since, until time.Time) bool { return !since.Before(time.Unix(0, 0)) && until.After(since) && until.Sub(since) <= MaxRouteEvidenceWindow } +func ValidRouteEvidencePath(width int16, path string) bool { + if width < 1 || width > 3 || len(path) < 4*int(width) || len(path) > 126*int(width) || len(path)%(2*int(width)) != 0 { + return false + } + _, err := hex.DecodeString(path) + return err == nil && path == strings.ToLower(path) +} + func (c RouteEvidenceCursor) String() string { - return fmt.Sprintf("v1:%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) + value := fmt.Sprintf("%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) + if c.HashSize != 0 || c.PathBytes != "" { + return fmt.Sprintf("v2:%s:%d:%s", value, c.HashSize, c.PathBytes) + } + return "v1:" + value } func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { - if len(raw) > 200 { + if len(raw) > 512 { return nil, ErrRouteEvidenceInput } p := strings.Split(raw, ":") - if len(p) != 7 || p[0] != "v1" || !ValidRouteEvidenceKey(p[1], p[2]) { + if !((len(p) == 7 && p[0] == "v1") || (len(p) == 9 && p[0] == "v2")) || !ValidRouteEvidenceKey(p[1], p[2]) { return nil, ErrRouteEvidenceInput } values := [4]int64{} - for i, s := range p[3:] { + for i, s := range p[3:7] { n, err := strconv.ParseInt(s, 10, 64) if err != nil || n < 0 || strconv.FormatInt(n, 10) != s { return nil, ErrRouteEvidenceInput @@ -104,6 +122,13 @@ func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { return nil, ErrRouteEvidenceInput } c := &RouteEvidenceCursor{IATA: p[1], PathKey: p[2], Since: time.UnixMilli(values[0]).UTC(), Until: time.UnixMilli(values[1]).UTC(), HeardAt: time.UnixMicro(values[2]).UTC(), ID: values[3]} + if p[0] == "v2" { + width, err := strconv.ParseInt(p[7], 10, 16) + if err != nil || strconv.FormatInt(width, 10) != p[7] || !ValidRouteEvidencePath(int16(width), p[8]) { + return nil, ErrRouteEvidenceInput + } + c.HashSize, c.PathBytes = int16(width), p[8] + } if !ValidRouteEvidenceWindow(c.Since, c.Until) || c.HeardAt.Before(c.Since) || !c.HeardAt.Before(c.Until) { return nil, ErrRouteEvidenceInput } diff --git a/internal/api/route_evidence_test.go b/internal/api/route_evidence_test.go index 42c9a73a..a025a440 100644 --- a/internal/api/route_evidence_test.go +++ b/internal/api/route_evidence_test.go @@ -23,3 +23,23 @@ func TestRouteEvidenceCursor(t *testing.T) { } } } + +func TestRouteEvidencePinnedCursor(t *testing.T) { + c := RouteEvidenceCursor{IATA: "YOW", PathKey: strings.Repeat("a", 32), Since: time.UnixMilli(1000).UTC(), Until: time.UnixMilli(3000).UTC(), HeardAt: time.UnixMicro(2000123).UTC(), ID: 9, HashSize: 3, PathBytes: strings.Repeat("aabbcc", 63)} + got, err := ParseRouteEvidenceCursor(c.String()) + if err != nil || *got != c || !strings.HasPrefix(c.String(), "v2:") { + t.Fatalf("pinned cursor round-trip: %+v %v", got, err) + } + for _, bad := range []string{ + strings.Replace(c.String(), ":3:", ":4:", 1), + strings.Replace(c.String(), ":3:", ":03:", 1), + c.String() + "00", + strings.TrimSuffix(c.String(), c.PathBytes) + "aabbcc", + strings.ToUpper(c.String()), + strings.TrimSuffix(c.String(), c.PathBytes) + "zzzzzzzzzzzz", + } { + if _, err := ParseRouteEvidenceCursor(bad); err == nil { + t.Errorf("accepted invalid pinned cursor %q", bad) + } + } +} diff --git a/internal/api/router/options.go b/internal/api/router/options.go index e838834e..a62fff0d 100644 --- a/internal/api/router/options.go +++ b/internal/api/router/options.go @@ -14,6 +14,7 @@ import ( // Options names the startup settings used by the router. Adding an optional // route does not change New's signature or require unrelated callers to change. type Options struct { + ScopeCatalogues handlers.ScopeCatalogueSource MaxConnsPerIP int MaxConnectsPerMinute int WSAllowedOrigins []string diff --git a/internal/api/router/router.go b/internal/api/router/router.go index d6bbbc97..7ce218b3 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -109,6 +109,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) r.Mount("/routes", handlers.RoutesRouter(reader)) r.Mount("/scopes", handlers.ScopesRouter(reader, opts.Scopes)) r.Mount("/stats", handlers.StatsRouter(reader, handlers.StatsOptions{Scopes: opts.Scopes, SeriesWindow: opts.RollupRetention})) + r.Get("/scope-catalogues", handlers.ScopeCatalogues(opts.ScopeCatalogues)) r.Mount("/traces", handlers.TracesRouter(reader)) }) diff --git a/internal/api/router/scope_catalogues_test.go b/internal/api/router/scope_catalogues_test.go new file mode 100644 index 00000000..5092e4c0 --- /dev/null +++ b/internal/api/router/scope_catalogues_test.go @@ -0,0 +1,28 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" +) + +func TestScopeCataloguesDisabledAndReadOnly(t *testing.T) { + var importer *meshmapper.Importer + r := New(nil, nil, nil, Options{ScopeCatalogues: importer}) + for _, method := range []string{http.MethodGet, http.MethodPost} { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(method, "/api/v1/scope-catalogues", nil)) + if method == http.MethodGet && (w.Code != 200 || strings.TrimSpace(w.Body.String()) != "[]" || w.Header().Get("Cache-Control") != "public, max-age=60") { + t.Fatal(w.Code, w.Body.String()) + } + if method == http.MethodPost && w.Code != 405 { + t.Fatal(w.Code) + } + } +} diff --git a/internal/api/topology.go b/internal/api/topology.go new file mode 100644 index 00000000..3fc85ce8 --- /dev/null +++ b/internal/api/topology.go @@ -0,0 +1,14 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import "github.com/google/uuid" + +// TopologyLinks is a bounded set of undirected, adjacent known-route segments. +// It does not imply radio reachability or join across missing path entries. +type TopologyLinks struct { + Links [][2]uuid.UUID `json:"links"` + Capped bool `json:"capped"` + Since int64 `json:"since"` + Until int64 `json:"until"` +} diff --git a/internal/api/traces.go b/internal/api/traces.go index b08fd7c3..f4a70d05 100644 --- a/internal/api/traces.go +++ b/internal/api/traces.go @@ -3,29 +3,33 @@ package api +import "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" + // TraceTagSummary is a single trace tag with aggregate metadata. type TraceTagSummary struct { - TraceTag string `json:"traceTag"` // hex-encoded 4-byte tag - FirstHeardAt int64 `json:"firstHeardAt"` // epoch ms - LastHeardAt int64 `json:"lastHeardAt"` // epoch ms - PacketCount int64 `json:"packetCount"` // number of packets with this trace tag - IATACount int64 `json:"iataCount"` // number of distinct IATAs where heard - TraceType string `json:"traceType"` // TRACE or PING - PathHashes []string `json:"pathHashes"` // hops from the most complete observation - SNRValues []float32 `json:"snrValues"` // SNR per hop from the most complete observation + Quality tracequality.Quality `json:"quality"` + TraceTag string `json:"traceTag"` // hex-encoded 4-byte tag + FirstHeardAt int64 `json:"firstHeardAt"` // epoch ms + LastHeardAt int64 `json:"lastHeardAt"` // epoch ms + PacketCount int64 `json:"packetCount"` // number of packets with this trace tag + IATACount int64 `json:"iataCount"` // number of distinct IATAs where heard + TraceType string `json:"traceType"` // TRACE or PING + PathHashes []string `json:"pathHashes"` // hops from the most complete observation + SNRValues []float32 `json:"snrValues"` // SNR per hop from the most complete observation } // TracePacket is a single packet within a trace series, including its // resolved route derived from the trace path hashes. type TracePacket struct { - PacketHash string `json:"packetHash"` // hex-encoded packet hash - RouteType int16 `json:"routeType"` // numeric route type - RouteTypeName string `json:"routeTypeName"` // human-readable route type - Scope *string `json:"scope,omitempty"` // transport scope name, if known - FirstHeardAt int64 `json:"firstHeardAt"` // epoch ms - LastHeardAt int64 `json:"lastHeardAt"` // epoch ms - RawPath []RawHop `json:"rawPath"` // hops as received in the packet - ResolvedRoute []ResolvedHop `json:"resolvedRoute"` // hops resolved to known nodes + Quality tracequality.Quality `json:"quality"` + PacketHash string `json:"packetHash"` // hex-encoded packet hash + RouteType int16 `json:"routeType"` // numeric route type + RouteTypeName string `json:"routeTypeName"` // human-readable route type + Scope *string `json:"scope,omitempty"` // transport scope name, if known + FirstHeardAt int64 `json:"firstHeardAt"` // epoch ms + LastHeardAt int64 `json:"lastHeardAt"` // epoch ms + RawPath []RawHop `json:"rawPath"` // hops as received in the packet + ResolvedRoute []ResolvedHop `json:"resolvedRoute"` // hops resolved to known nodes } // RawHop is a single hop in a trace path as received in the packet. diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index b5dcc7f5..4a349d45 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -446,3 +446,7 @@ func TestCachedReader_InvalidateObserver(t *testing.T) { func (s *stubReader) GetRouteEvidence(_ context.Context, _, _ string, _ api.RouteEvidenceQuery) (*api.RouteEvidence, error) { return nil, nil } + +func (s *stubReader) GetTopologyLinks(_ context.Context, _ []string, _ time.Time, _ time.Time) (*api.TopologyLinks, error) { + return &api.TopologyLinks{}, nil +} diff --git a/internal/cache/reader.go b/internal/cache/reader.go index e491152d..6c4a201f 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -378,3 +378,12 @@ func (cr *CachedReader) SearchCrossIATARoutes(ctx context.Context, fromHash, fro func (cr *CachedReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { return cr.inner.ListTraceTags(ctx, iatas, scope, traceType, since, until, cursor, cursorTag, limit) } + +func (cr *CachedReader) GetTopologyLinks(ctx context.Context, iatas []string, since, until time.Time) (*api.TopologyLinks, error) { + regions := append([]string{}, iatas...) + sort.Strings(regions) + key := fmt.Sprintf("beacon:topology:%s:%d:%d", strings.Join(regions, ","), since.Unix(), until.Unix()) + return getOrSet(ctx, cr.c, key, 30*time.Second, func(ctx context.Context) (*api.TopologyLinks, error) { + return cr.inner.GetTopologyLinks(ctx, iatas, since, until) + }) +} diff --git a/internal/ingest/channel_scope_test.go b/internal/ingest/channel_scope_test.go index 3ee69334..e229ef23 100644 --- a/internal/ingest/channel_scope_test.go +++ b/internal/ingest/channel_scope_test.go @@ -5,6 +5,7 @@ package ingest import ( "context" + "encoding/hex" "encoding/json" "testing" "time" @@ -73,3 +74,36 @@ func TestChannelMessageScopeLive(t *testing.T) { }) } } + +func TestChannelPublicMarkerUsesDecryptionKey(t *testing.T) { + for _, public := range []bool{false, true} { + w, base := newTestWorker() + w.db = &scopeMessageDB{stubDB: base, next: &InsertedChannelMessage{ID: 1}} + key := make([]byte, 16) + if public { + key, _ = hex.DecodeString("8b3387e9c5cdea6ac9e5edbaa115cd72") + } + w.keys = &mapKeys{entries: map[byte][]keystore.Entry{0x11: {{Key: key, Name: "Public"}}}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "public-marker", hub.Scope{Events: []hub.EventType{hub.EventChannelMessage, hub.EventObserverStatus}}) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeGrpTxt, 0), Payload: encryptedGroupText(t, 0x11, key, "Fixture", "test")} + w.handlePayloadTypeSideEffects(ctx, packet, "YKF", []byte{1}, RadioSettings{}, nil, nil, nil, 0) + select { + case event := <-client.Send: + var got channelMessageEvent + if err := json.Unmarshal(event.Payload, &got); err != nil { + t.Fatal(err) + } + if got.IsPublic != public { + t.Fatal("channel name/hash used instead of actual decryption key") + } + case <-ctx.Done(): + t.Fatal("no channel event") + } + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 6c3abb72..46d7f830 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -158,6 +158,7 @@ type DB interface { // ResolvePathHashes returns a list of node UUIDs for the given path hash prefixes and IATA. ResolvePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) + ResolveTracePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) // ResolveEndpointHashes matches one-byte logical endpoints, including companions. ResolveEndpointHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index cffa4654..0c45b48a 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -266,6 +266,9 @@ func (s *stubDB) GetObserverScopes(_ context.Context, _ uuid.UUID) ([]string, er func (s *stubDB) ResolvePathHashes(_ context.Context, _ string, _ [][]byte) (map[string][]api.ResolvedPathEntry, error) { return nil, nil } +func (s *stubDB) ResolveTracePathHashes(_ context.Context, _ string, _ [][]byte) (map[string][]api.ResolvedPathEntry, error) { + return nil, nil +} func (s *stubDB) ResolveEndpointHashes(_ context.Context, _ string, _ [][]byte) (map[string][]api.ResolvedPathEntry, error) { return nil, nil diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index e8754925..2a4b84eb 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -18,6 +18,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/lora" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/MeshCore-Beacon/beacon-server/internal/tracequality" "github.com/google/uuid" "github.com/meshcore-go/meshcore-go" ) @@ -202,13 +203,14 @@ type parsedGroupEnvelope struct { // is the accumulated per-hop SNR sequence carried in the packet header's // path field (a protocol-level repurposing distinct from the payload). type parsedTrace struct { - Raw string `json:"raw"` - Type string `json:"type"` - TraceTag string `json:"traceTag"` - AuthCode uint32 `json:"authCode"` - Flags byte `json:"flags"` - PathHashes []string `json:"pathHashes"` - SNRValues []float32 `json:"snrValues"` + Quality tracequality.Quality `json:"quality"` + Raw string `json:"raw"` + Type string `json:"type"` + TraceTag string `json:"traceTag"` + AuthCode uint32 `json:"authCode"` + Flags byte `json:"flags"` + PathHashes []string `json:"pathHashes"` + SNRValues []float32 `json:"snrValues"` } // parsedAck is the parsed form of an ACK payload (§2.12). Checksum is the @@ -557,7 +559,10 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ hashes = append(hashes, hex.EncodeToString(h)) rawHashes = append(rawHashes, h) } - traceRawHashes = rawHashes + quality := tracequality.Assess(packet.Payload, int(packet.PayloadVer()), int(packet.RouteType()), int(packet.PathLength)) + if quality.Status == "supported" { + traceRawHashes = rawHashes + } // SNR values are in packet.Path, one signed int8 per consumed hop snrValues := make([]float32, 0, len(packet.Path)) for _, b := range packet.Path { @@ -568,6 +573,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ traceType = "PING" } pt := parsedTrace{ + Quality: quality, Raw: hex.EncodeToString(packet.Payload), Type: traceType, TraceTag: hex.EncodeToString(uint32ToBytes(trace.Tag)), @@ -585,8 +591,8 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ // originator isn't in PathHashes) and is skipped. Either side // resolving ambiguously (>1 candidate) or not at all (0 // candidates) skips that specific pair. - if len(rawHashes) >= 2 { - resolved, rErr := w.db.ResolvePathHashes(ctx, iata, rawHashes) + if quality.Status == "supported" && len(rawHashes) >= 2 { + resolved, rErr := w.db.ResolveTracePathHashes(ctx, iata, rawHashes) if rErr == nil { for i := 1; i < len(rawHashes); i++ { if i >= len(snrValues) { @@ -822,12 +828,20 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if packet.PayloadType() == meshcore.PayloadTypeTrace { hashes = traceRawHashes } - resolved, err := w.db.ResolvePathHashes(ctx, iata, hashes) + var resolved map[string][]api.ResolvedPathEntry + if packet.PayloadType() == meshcore.PayloadTypeTrace { + resolved, err = w.db.ResolveTracePathHashes(ctx, iata, hashes) + } else { + resolved, err = w.db.ResolvePathHashes(ctx, iata, hashes) + } if err != nil { w.log.Error("path resolution failed", "error", err) } var resolvedIDs []uuid.UUID for _, entries := range resolved { + if len(entries) != 1 { + continue + } for _, e := range entries { resolvedIDs = append(resolvedIDs, e.NodeID) } @@ -846,13 +860,31 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ nodeIDs = append(nodeIDs, entries[0].NodeID) hashPrefixes = append(hashPrefixes, hash) } - if allHigh && len(nodeIDs) > 1 { + if allHigh && len(nodeIDs) > 1 && packet.PayloadType() != meshcore.PayloadTypeTrace { if err := w.db.UpsertKnownRoute(ctx, nodeIDs, hashPrefixes, iata, int32(len(nodeIDs))); err != nil { w.log.Error("failed to upsert known route", "error", err) } } } - w.runCapabilityDetection(ctx, packet.PayloadType(), packet.PathHashSize(), resolvedIDs) + hashSize := packet.PathHashSize() + if packet.PayloadType() == meshcore.PayloadTypeTrace { + hashSize = 1 + if len(traceRawHashes) > 0 { + hashSize = uint8(len(traceRawHashes[0])) + } + // Only consumed, uniquely resolved hops demonstrate trace capability. + resolvedIDs = nil + for i, hash := range traceRawHashes { + if i >= len(packet.Path) { + break + } + entries := resolved[hex.EncodeToString(hash)] + if len(entries) == 1 { + resolvedIDs = append(resolvedIDs, entries[0].NodeID) + } + } + } + w.runCapabilityDetection(ctx, packet.PayloadType(), hashSize, resolvedIDs) if inserted || repeat { if inserted { diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 3299027f..95c8441a 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -48,6 +48,7 @@ type InsertedChannelMessage struct { // channelMessageEvent is the JSON payload for a channelMessage WS event. type channelMessageEvent struct { + IsPublic bool `json:"isPublic"` // True only when decryption used the documented default Public key, never inferred from a one-byte channel hash. ID int64 `json:"id"` ChannelID int `json:"channelId"` ChannelHash string `json:"channelHash"` // hex-encoded single byte @@ -223,6 +224,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc if result.Message != nil { evt := channelMessageEvent{ + IsPublic: hex.EncodeToString(result.Entry.Key) == "8b3387e9c5cdea6ac9e5edbaa115cd72", ID: result.Message.ID, Scope: result.Message.Scope, ScopeStatus: result.Message.ScopeStatus, diff --git a/internal/ingest/trace_quality_test.go b/internal/ingest/trace_quality_test.go new file mode 100644 index 00000000..89c36b1a --- /dev/null +++ b/internal/ingest/trace_quality_test.go @@ -0,0 +1,75 @@ +package ingest + +import ( + "context" + "encoding/hex" + "encoding/json" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +type traceSafetyDB struct { + stubDB + saved UpsertPacketParams + resolutions, neighbors, routes int +} + +func (s *traceSafetyDB) UpsertPacket(_ context.Context, p UpsertPacketParams) (bool, error) { + s.saved = p + return true, nil +} +func (s *traceSafetyDB) ResolveTracePathHashes(_ context.Context, _ string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + if len(hashes) > 0 { + s.resolutions++ + } + out := map[string][]api.ResolvedPathEntry{} + for i, h := range hashes { + id := uuid.UUID{} + id[15] = byte(i + 1) + out[hex.EncodeToString(h)] = []api.ResolvedPathEntry{{NodeID: id}} + } + return out, nil +} +func (s *traceSafetyDB) UpsertNodeNeighbor(_ context.Context, _, _ uuid.UUID, _ string, _ *float32, _ *string) error { + s.neighbors++ + return nil +} +func (s *traceSafetyDB) UpsertKnownRoute(_ context.Context, _ []uuid.UUID, _ [][]byte, _ string, _ int32) error { + s.routes++ + return nil +} + +func TestTraceQualityDoesNotPromoteMalformedOrUnvisitedHops(t *testing.T) { + for _, bad := range []bool{false, true} { + w, _ := newTestWorker() + db := &traceSafetyDB{} + w.db = db + payload, _ := (&meshcore.Trace{Tag: 7, Flags: 1, PathHashes: []byte{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}}).ToBytes() + version := byte(0) + if bad { + version = 1 + } + pkt := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeDirect, meshcore.PayloadTypeTrace, version), PathLength: 2, Path: []byte{0, 252}, Payload: payload} + w.handlePacket(context.Background(), "YKF", "0102", packetEnvelope(t, pkt)) + if len(db.saved.RawPayload) == 0 || len(db.traceHearings) != 1 { + t.Fatal("raw evidence lost") + } + var parsed parsedTrace + if err := json.Unmarshal(db.saved.ParsedPayload, &parsed); err != nil { + t.Fatal(err) + } + if bad { + if parsed.Quality.Status != "suspect" || db.neighbors != 0 || db.resolutions != 0 || len(db.setCapabilityCalls) != 0 { + t.Fatalf("malformed trace promoted: %+v", db) + } + } else if parsed.Quality.Status != "supported" || db.neighbors != 1 || len(db.setCapabilityCalls) != 2 { + t.Fatalf("valid consumed links/capabilities missing: %+v", db) + } + if db.routes != 0 { + t.Fatal("planned route promoted as observed") + } + } +} diff --git a/internal/meshmapper/catalogues.go b/internal/meshmapper/catalogues.go new file mode 100644 index 00000000..76552291 --- /dev/null +++ b/internal/meshmapper/catalogues.go @@ -0,0 +1,75 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "encoding/json" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +// CatalogueScope is regional source metadata, not per-node forwarding evidence. +type CatalogueScope struct { + Name string `json:"name"` + Repeaters int `json:"repeaters"` + Default int `json:"default"` + Monitored bool `json:"monitored"` + Wardriving bool `json:"wardriving"` +} + +type PublicScopeCatalogue struct { + IATA string `json:"iata"` + URL string `json:"url"` + GeneratedAt int64 `json:"generatedAt"` + CheckedAt int64 `json:"checkedAt"` + FreshUntil int64 `json:"freshUntil"` + LastError string `json:"lastError,omitempty"` + Repeaters int `json:"repeaters"` + Scoped int `json:"scoped"` + Scopes []CatalogueScope `json:"scopes"` +} + +// Catalogues returns an immutable snapshot of discovered sources. No request +// triggers an upstream fetch or reads ingestion tables. Callers must not mutate it. +func (i *Importer) Catalogues() []PublicScopeCatalogue { + if i != nil { + if snapshot := i.catalogues.Load(); snapshot != nil { + return snapshot.([]PublicScopeCatalogue) + } + } + return []PublicScopeCatalogue{} +} + +func (i *Importer) publishCatalogues() { + result := make([]PublicScopeCatalogue, 0, len(i.sources)) + for _, s := range i.sources { + c := PublicScopeCatalogue{IATA: s.iata, URL: s.url, LastError: s.cache.LastError, Scopes: []CatalogueScope{}} + if !s.cache.CheckedAt.IsZero() { + c.CheckedAt = s.cache.CheckedAt.UnixMilli() + c.FreshUntil = s.cache.CheckedAt.Add(i.interval + 5*time.Minute).UnixMilli() + } + if len(s.cache.Payload) > 0 { + // Reuse the importer validation before exposing saved metadata. Invalid + // saved catalogues are omitted just as they are from the scope matcher. + if _, generated, err := decode(s.cache.Payload, s.iata); err == nil { + var payload struct { + Repeaters int + Scoped int + Scopes []CatalogueScope + } + if json.Unmarshal(s.cache.Payload, &payload) == nil { + c.GeneratedAt = generated.UnixMilli() + c.Repeaters, c.Scoped = payload.Repeaters, payload.Scoped + for _, scope := range payload.Scopes { + scope.Name = scopestore.FromName(scope.Name).Name + c.Scopes = append(c.Scopes, scope) + } + } + } + } + result = append(result, c) + } + i.catalogues.Store(result) +} diff --git a/internal/meshmapper/catalogues_test.go b/internal/meshmapper/catalogues_test.go new file mode 100644 index 00000000..baefa72d --- /dev/null +++ b/internal/meshmapper/catalogues_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { + status := http.StatusOK + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(status) + if status == http.StatusOK { + _, _ = w.Write(catalogue("yow", "YOW", "monitored-zero")) + } + })) + defer server.Close() + cfg := config.MeshMapperScopesConfig{Enabled: true} + store := newMemoryStore("YOW") + imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + if got := imp.Catalogues(); len(got) != 1 || got[0].CheckedAt != 0 || len(got[0].Scopes) != 0 { + t.Fatal(got) + } + imp.sources[0].url = server.URL + now := time.Now() + if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil { + t.Fatal(err) + } + imp.publishCatalogues() + old := imp.Catalogues() + if old[0].Scopes[0].Name != "#yow" || old[0].Scopes[1].Name != "#YOW" || len(old[0].Scopes) != 3 || old[0].Repeaters != 5 || old[0].Scoped != 3 || old[0].FreshUntil <= old[0].CheckedAt { + t.Fatal(old) + } + status = http.StatusServiceUnavailable + if err := imp.refresh(context.Background(), &imp.sources[0], now.Add(time.Minute)); err != nil { + t.Fatal(err) + } + imp.publishCatalogues() + current := imp.Catalogues()[0] + if current.LastError != "HTTP 503" || current.CheckedAt != old[0].CheckedAt || current.GeneratedAt != old[0].GeneratedAt || len(current.Scopes) != 3 || old[0].LastError != "" { + t.Fatal(current, old) + } + // Readers only access published immutable data; the race suite exercises concurrent refreshes. + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for range 100 { + _, _ = json.Marshal(imp.Catalogues()) + } + }() + for range 100 { + imp.publishCatalogues() + } + wg.Wait() + if calls != 2 { + t.Fatal("reading public metadata performed upstream requests", calls) + } + disabled, err := New(context.Background(), config.MeshMapperScopesConfig{}, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) + if err != nil || len(disabled.Catalogues()) != 0 { + t.Fatal("disabled importer exposed old membership") + } + store.rows[sourceKey{"YOW", server.URL}] = Cache{Payload: []byte(`{"scopes":"invalid"}`)} + restored, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) + if err != nil || len(restored.Catalogues()[0].Scopes) != 0 || restored.Catalogues()[0].LastError == "" { + t.Fatal("invalid saved metadata leaked") + } +} + +func TestPublicCataloguesIncludeDiscoveredAndUnlistedRegions(t *testing.T) { + ctx := context.Background() + store := newMemoryStore("YOW") + dir := NewDirectory(newZoneListMemory()) + dir.lists["CA"] = &zoneList{zones: map[string]zoneEntry{}, fetchedAt: time.Now()} + imp, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + before := imp.Catalogues() + store.iatas = append(store.iatas, "YKF") + if err := imp.Refresh(ctx); err != nil { + t.Fatal(err) + } + after := imp.Catalogues() + if len(before) != 1 || before[0].LastError != "" || len(after) != 2 || after[1].IATA != "YKF" || after[1].LastError != "not listed" { + t.Fatal(before, after) + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index 76770185..191b6b5f 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -14,6 +14,7 @@ import ( "slices" "strconv" "strings" + "sync/atomic" "time" "unicode" "unicode/utf8" @@ -60,16 +61,17 @@ type source struct { // Importer is owned by one background task; ScopeStore synchronizes its consumers. type Importer struct { - store Store - scopes *scopestore.ScopeStore - manual []scopestore.Entry - dir *Directory - scopesURL func(site string) (string, bool) - seen map[string]bool - sources []source - interval time.Duration - client *http.Client - onChange func(context.Context) + catalogues atomic.Value // immutable []PublicScopeCatalogue, off the ingest path + store Store + scopes *scopestore.ScopeStore + manual []scopestore.Entry + dir *Directory + scopesURL func(site string) (string, bool) + seen map[string]bool + sources []source + interval time.Duration + client *http.Client + onChange func(context.Context) } // SetCacheInvalidator is wired once at startup, before the background task starts. @@ -102,6 +104,7 @@ func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, di } i.track(iatas, latest) i.publish() + i.publishCatalogues() return i, nil } @@ -136,6 +139,7 @@ func (i *Importer) track(iatas []string, saved map[string]Catalogue) { // Refresh checks only one due source. The scheduler serializes calls every 15s. func (i *Importer) Refresh(ctx context.Context) (err error) { + defer i.publishCatalogues() parent := ctx defer func() { if parent.Err() != nil { diff --git a/internal/tracequality/quality.go b/internal/tracequality/quality.go new file mode 100644 index 00000000..e8857533 --- /dev/null +++ b/internal/tracequality/quality.go @@ -0,0 +1,96 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package tracequality checks TRACE structure, not authenticity or RF plausibility. +package tracequality + +import "encoding/hex" + +type Quality struct { + Status string `json:"status"` // supported, suspect, or ambiguous; supported is not authenticated + Reasons []string `json:"reasons"` +} + +func (q *Quality) Add(reason string) { + for _, r := range q.Reasons { + if r == reason { + return + } + } + q.Reasons = append(q.Reasons, reason) + if reason == "ambiguous_prefix" && q.Status != "suspect" { + q.Status = "ambiguous" + } else if reason != "ambiguous_prefix" { + q.Status = "suspect" + } +} + +// Assess uses TRACE's power-of-two payload hash widths, NOT the ordinary +// packet path's 1/2/3-byte widths. The path field contains consumed-hop SNRs. +// -1 means a header/path field is unavailable in a retained summary. +// Mesh.cpp onRecvPacket/sendDirect: a trace uses direct routing, flags bits +// 0..1, and fewer than 64 SNR bytes. Partial probes and return loops are valid. +func Assess(raw []byte, version, route, pathLength int) Quality { + q := Quality{Status: "supported", Reasons: []string{}} + if version > 0 { + q.Add("unsupported_version") + } + if route >= 0 && route != 2 && route != 3 { + q.Add("non_direct_trace") + } + if len(raw) < 10 { + q.Add("incomplete_payload") + return q + } + if len(raw) > 184 { + q.Add("payload_too_large") + } + if raw[8]&0xfc != 0 { + q.Add("unsupported_flags") + } + size := 1 << (raw[8] & 3) + if (len(raw)-9)%size != 0 { + q.Add("misaligned_hashes") + } + count := (len(raw) - 9) / size + if count > 63 { + q.Add("too_many_hops") + } + if pathLength > 63 || pathLength > count { + q.Add("invalid_snr_path") + } + return q +} + +// Stored also checks old parsed rows, without rewriting or deleting the evidence. +func Stored(raw string, flags byte, hashes []string, snrs []float32) Quality { + b, err := hex.DecodeString(raw) + if err != nil { + return Quality{Status: "suspect", Reasons: []string{"incomplete_payload"}} + } + if raw == "" { // Older exports may contain only decoded fields. + b = make([]byte, 9) + b[8] = flags + for _, h := range hashes { + v, e := hex.DecodeString(h) + if e != nil { + return Quality{Status: "suspect", Reasons: []string{"misaligned_hashes"}} + } + b = append(b, v...) + } + } + q := Assess(b, -1, -1, len(snrs)) + size := 1 << (flags & 3) + if len(b) >= 9 && b[8] != flags { + q.Add("misaligned_hashes") + } + for _, h := range hashes { + if len(h) != 2*size { + q.Add("misaligned_hashes") + } + } + if len(b) >= 9 && len(hashes) != (len(b)-9)/size { + q.Add("misaligned_hashes") + } + return q +} diff --git a/internal/tracequality/quality_test.go b/internal/tracequality/quality_test.go new file mode 100644 index 00000000..f811ae30 --- /dev/null +++ b/internal/tracequality/quality_test.go @@ -0,0 +1,51 @@ +package tracequality + +import ( + "bytes" + "slices" + "testing" +) + +func TestTraceFormat(t *testing.T) { + for flags := byte(0); flags < 4; flags++ { + raw := make([]byte, 9) + raw[8] = flags + raw = append(raw, bytes.Repeat([]byte{0xaa}, 3*(1< Date: Wed, 7 Oct 2026 05:54:03 -0400 Subject: [PATCH 16/23] Adapt preview catalogue tests to authenticated MeshMapper client --- internal/meshmapper/catalogues_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/meshmapper/catalogues_test.go b/internal/meshmapper/catalogues_test.go index baefa72d..14dd3d03 100644 --- a/internal/meshmapper/catalogues_test.go +++ b/internal/meshmapper/catalogues_test.go @@ -29,7 +29,7 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { defer server.Close() cfg := config.MeshMapperScopesConfig{Enabled: true} store := newMemoryStore("YOW") - imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) + imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory(), ""), scopestore.New(), nil) if err != nil { t.Fatal(err) } @@ -37,6 +37,7 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { t.Fatal(got) } imp.sources[0].url = server.URL + imp.client = server.Client() now := time.Now() if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil { t.Fatal(err) @@ -71,12 +72,12 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { if calls != 2 { t.Fatal("reading public metadata performed upstream requests", calls) } - disabled, err := New(context.Background(), config.MeshMapperScopesConfig{}, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) + disabled, err := New(context.Background(), config.MeshMapperScopesConfig{}, store, NewDirectory(newZoneListMemory(), ""), scopestore.New(), nil) if err != nil || len(disabled.Catalogues()) != 0 { t.Fatal("disabled importer exposed old membership") } store.rows[sourceKey{"YOW", server.URL}] = Cache{Payload: []byte(`{"scopes":"invalid"}`)} - restored, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopestore.New(), nil) + restored, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory(), ""), scopestore.New(), nil) if err != nil || len(restored.Catalogues()[0].Scopes) != 0 || restored.Catalogues()[0].LastError == "" { t.Fatal("invalid saved metadata leaked") } @@ -85,7 +86,7 @@ func TestPublicCataloguesFreshnessAndIsolation(t *testing.T) { func TestPublicCataloguesIncludeDiscoveredAndUnlistedRegions(t *testing.T) { ctx := context.Background() store := newMemoryStore("YOW") - dir := NewDirectory(newZoneListMemory()) + dir := NewDirectory(newZoneListMemory(), "") dir.lists["CA"] = &zoneList{zones: map[string]zoneEntry{}, fetchedAt: time.Now()} imp, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopestore.New(), nil) if err != nil { From 03698f7acb0100c54655fcf5fd4d128ac932fbf8 Mon Sep 17 00:00:00 2001 From: n30nex Date: Thu, 15 Oct 2026 04:37:59 -0400 Subject: [PATCH 17/23] Exercise upstream packet search and repeater queries against PostgreSQL --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 129bd37c..3d94fcf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(SearchPacketsByHashPrefix|MeshMapperRepeaters|TopAdvertisers|Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v + run: go test ./db -run '^Test(SearchPacketsByHashPrefix|MeshMapperRepeaters|TopAdvertisersSort|Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | From 527efa4d334c2f673651a89f49fcc7920b1e4317 Mon Sep 17 00:00:00 2001 From: n30nex Date: Mon, 19 Oct 2026 15:57:27 -0400 Subject: [PATCH 18/23] Backport upstream bigint cursor fix to preview Port the query casts and boundary regression from MeshCore-Beacon/beacon-server#233 (d5aacce). Keep the existing preview schema and API version. --- .github/workflows/ci.yml | 2 +- db/bigint_cursor_integration_test.go | 253 +++++++++++++++++++++++++++ db/queries/queries.sql | 8 +- db/sqlc/queries.sql.go | 24 +-- 4 files changed, 270 insertions(+), 17 deletions(-) create mode 100644 db/bigint_cursor_integration_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d94fcf4..e74b0e88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(SearchPacketsByHashPrefix|MeshMapperRepeaters|TopAdvertisersSort|Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v + run: go test ./db -run '^Test(BigintCursors|SearchPacketsByHashPrefix|MeshMapperRepeaters|TopAdvertisersSort|Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/db/bigint_cursor_integration_test.go b/db/bigint_cursor_integration_test.go new file mode 100644 index 00000000..5802145b --- /dev/null +++ b/db/bigint_cursor_integration_test.go @@ -0,0 +1,253 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "reflect" + "slices" + "testing" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +// The four statements before the ::bigint casts, verbatim from origin/dev. +const ( + legacyListObserverAdverts = `SELECT + po.id, + encode(po.packet_hash, 'hex') AS packet_hash_hex, + p.payload_type, + po.iata, + po.heard_at, + po.rssi, + po.snr, + po.hop_count, + n.name AS node_name, + COALESCE(encode(p.origin_pubkey, 'hex'), '')::text AS node_public_key +FROM packet_observations po +JOIN packets p ON p.packet_hash = po.packet_hash +LEFT JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE po.observer_id = $1 + AND p.payload_type = 4 + AND ($2 = 0 OR po.id > $2) +ORDER BY po.id ASC +LIMIT $3` + legacyListNodeObservations = `SELECT po.id, encode(po.packet_hash, 'hex') AS packet_hash_hex, + p.payload_type, po.iata, po.heard_at, po.rssi, po.snr, po.hop_count +FROM packet_observations po +JOIN packets p ON p.packet_hash = po.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE n.id = $1 + AND ($2 = 0 OR po.id < $2) +ORDER BY po.id DESC +LIMIT $3` + legacyGetObserverTelemetry = `SELECT id, reported_at, battery_voltage_mv, airtime_tx_secs, airtime_rx_secs, + noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors +FROM observer_telemetry +WHERE observer_id = $1 + AND ($2::timestamptz IS NULL OR reported_at >= $2) + AND ($3::timestamptz IS NULL OR reported_at <= $3) + AND ($4 = 0 OR id > $4) +ORDER BY reported_at ASC` + legacyListAllChannelMessages = `SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, +p.observation_count AS observation_count +FROM channel_messages cm +JOIN channels c ON c.id = cm.channel_id +JOIN packet_observations po ON po.packet_hash = cm.packet_hash +JOIN packets p ON p.packet_hash = cm.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id +WHERE ($1::timestamptz IS NULL OR cm.sent_at >= $1) + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR po.iata = ANY($2::bpchar[])) + AND ($3::text = '' OR ts.name = $3::text) + AND ($4 = 0 OR cm.id < $4) +ORDER BY cm.id DESC +LIMIT $5` +) + +const ( + cursorObserver = "00000000-0000-0000-0000-000000000106" + cursorNode = "00000000-0000-0000-0000-000000000201" +) + +// Ids 1-3 sit below 2^31; each sequence then hands out 2147483646..2147483651, crossing it as prod's will. +const bigintCursorFixture = ` +INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000106','\x06'); +INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000201','\x01',1,'Fixture'); +INSERT INTO channels(id,channel_hash,name) VALUES (7,'\x11','#fixture'); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,origin_pubkey,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT int4send(i),CASE WHEN i % 2 = 0 THEN 4 ELSE 5 END,0,1,'\x01','\x00','\x00',now(),now() FROM generate_series(1,9) i; +INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT i,int4send(i),'00000000-0000-0000-0000-000000000106','YOW',now(),0,1,0 FROM generate_series(1,3) i; +SELECT setval('packet_observations_id_seq', 2147483645); +INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT int4send(i),'00000000-0000-0000-0000-000000000106','YOW',now(),0,1,0 FROM generate_series(4,9) i; +INSERT INTO channel_messages(id,channel_id,packet_hash,sent_at) SELECT i,7,int4send(i),now() FROM generate_series(1,3) i; +SELECT setval('channel_messages_id_seq', 2147483645); +INSERT INTO channel_messages(channel_id,packet_hash,sent_at) SELECT 7,int4send(i),now() FROM generate_series(4,9) i; +INSERT INTO observer_telemetry(id,observer_id,reported_at) +SELECT i,'00000000-0000-0000-0000-000000000106',date_trunc('minute',now())-make_interval(mins=>60-i) FROM generate_series(1,3) i; +SELECT setval('observer_telemetry_id_seq', 2147483645); +INSERT INTO observer_telemetry(observer_id,reported_at) +SELECT '00000000-0000-0000-0000-000000000106',date_trunc('minute',now())-make_interval(mins=>60-i) FROM generate_series(4,9) i;` + +func getJSON(t *testing.T, ctx context.Context, h http.Handler, path string, out any) { + t.Helper() + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil).WithContext(ctx)) + if rec.Code != http.StatusOK { + t.Fatalf("GET %s: HTTP %d %s", path, rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), out); err != nil { + t.Fatal(err) + } +} + +// walkPages follows nextCursor at limit=1, so every id, those past 2^31 included, goes back as a cursor. +// base ends in "?" or "&". +func walkPages[T any](t *testing.T, ctx context.Context, h http.Handler, base string, id func(T) int64) []int64 { + t.Helper() + var ids []int64 + path := base + "limit=1" + for range 20 { + var page api.Page[T] + getJSON(t, ctx, h, path, &page) + for _, item := range page.Items { + ids = append(ids, id(item)) + } + if !page.HasMore { + return ids + } + path = fmt.Sprintf("%slimit=1&cursor=%d", base, *page.NextCursor) + } + t.Fatalf("%s: still paging after 20 pages", base) + return nil +} + +func queryInt64s(t *testing.T, ctx context.Context, tx pgx.Tx, sql string, args ...any) []int64 { + t.Helper() + rows, err := tx.Query(ctx, sql, args...) + if err != nil { + t.Fatal(err) + } + out, err := pgx.CollectRows(rows, pgx.RowTo[int64]) + if err != nil { + t.Fatal(err) + } + return out +} + +func legacyRows[T any](t *testing.T, ctx context.Context, tx pgx.Tx, sql string, args ...any) []T { + t.Helper() + rows, err := tx.Query(ctx, sql, args...) + if err != nil { + t.Fatal(err) + } + out, err := pgx.CollectRows(rows, pgx.RowToStructByPos[T]) + if err != nil { + t.Fatal(err) + } + return out +} + +func sameRows[T any](got, want []T) bool { + return len(got) == len(want) && (len(got) == 0 || reflect.DeepEqual(got, want)) +} + +// Observation ids pass 2^31 in 2026-11; every id cursor must keep paging past it. +func TestBigintCursorsPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, bigintCursorFixture); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + observers, nodes, messages, packets := handlers.ObserversRouter(store), handlers.NodesRouter(store), handlers.MessagesRouter(store), handlers.PacketsRouter(store) + + t.Run("observer adverts", func(t *testing.T) { + want := queryInt64s(t, ctx, tx, `SELECT po.id FROM packet_observations po JOIN packets p USING (packet_hash) WHERE p.payload_type = 4 ORDER BY po.id`) + got := walkPages(t, ctx, observers, "/"+cursorObserver+"/adverts?", func(a api.AdvertObservation) int64 { return a.ID }) + if !slices.Equal(got, want) || want[len(want)-1] <= math.MaxInt32+1 { + t.Fatalf("got %v, want %v", got, want) + } + var past api.Page[api.AdvertObservation] + getJSON(t, ctx, observers, fmt.Sprintf("/%s/adverts?cursor=%d", cursorObserver, int64(math.MaxInt64)), &past) + if len(past.Items) != 0 || past.HasMore { + t.Fatalf("cursor MaxInt64: %+v", past) + } + }) + t.Run("node observations", func(t *testing.T) { + want := queryInt64s(t, ctx, tx, `SELECT id FROM packet_observations ORDER BY id DESC`) + got := walkPages(t, ctx, nodes, "/"+cursorNode+"/observations?", func(o api.PacketObservationSummary) int64 { return o.ID }) + if !slices.Equal(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + }) + t.Run("messages", func(t *testing.T) { + want := queryInt64s(t, ctx, tx, `SELECT id FROM channel_messages ORDER BY id DESC`) + for name, base := range map[string]string{"all channels": "/?", "channel id": "/?channelID=7&", "channel hash": "/?channelHash=11&"} { + t.Run(name, func(t *testing.T) { + if got := walkPages(t, ctx, messages, base, func(m api.ChannelMessage) int64 { return m.ID }); !slices.Equal(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + }) + } + }) + t.Run("telemetry", func(t *testing.T) { + for _, after := range []int64{0, 2, math.MaxInt32, math.MaxInt32 + 1, math.MaxInt32 + 3} { + want := queryInt64s(t, ctx, tx, `SELECT (extract(epoch FROM reported_at) * 1000)::bigint FROM observer_telemetry WHERE id > $1 ORDER BY reported_at`, after) + var tel api.ObserverTelemetry + getJSON(t, ctx, observers, fmt.Sprintf("/%s/telemetry?afterId=%d", cursorObserver, after), &tel) + got := make([]int64, 0, len(tel.Points)) + for _, p := range tel.Points { + got = append(got, p.T) + } + if !slices.Equal(got, want) { + t.Fatalf("afterId %d: got %v, want %v", after, got, want) + } + } + }) + t.Run("ws backfill", func(t *testing.T) { + var ps []api.PacketSummary + getJSON(t, ctx, packets, fmt.Sprintf("/backfill?afterObservationId=%d", math.MaxInt32), &ps) + var ms []api.ChannelMessage + getJSON(t, ctx, messages, fmt.Sprintf("/backfill?afterId=%d", math.MaxInt32), &ms) + if len(ps) != 4 || len(ms) != 4 { + t.Fatalf("backfill past 2^31: %d packets, %d messages, want 4 and 4", len(ps), len(ms)) + } + }) + t.Run("same rows as today below 2^31", func(t *testing.T) { + q := sqlc.New(tx) + obs, node := uuid.MustParse(cursorObserver), uuid.MustParse(cursorNode) + for _, c := range []int32{-1, 0, 1, 2, 3, 2147483646, math.MaxInt32} { + for _, n := range []int32{1, 2, 50} { + adverts, err := q.ListObserverAdverts(ctx, sqlc.ListObserverAdvertsParams{ObserverID: obs, Column2: int64(c), Limit: n}) + if err != nil || !sameRows(adverts, legacyRows[sqlc.ListObserverAdvertsRow](t, ctx, tx, legacyListObserverAdverts, obs, c, n)) { + t.Fatalf("adverts cursor %d limit %d differ: %v", c, n, err) + } + nodeObs, err := q.ListNodeObservations(ctx, sqlc.ListNodeObservationsParams{ID: node, Column2: int64(c), Limit: n}) + if err != nil || !sameRows(nodeObs, legacyRows[sqlc.ListNodeObservationsRow](t, ctx, tx, legacyListNodeObservations, node, c, n)) { + t.Fatalf("node observations cursor %d limit %d differ: %v", c, n, err) + } + msgs, err := q.ListAllChannelMessages(ctx, sqlc.ListAllChannelMessagesParams{Column4: int64(c), Limit: n}) + if err != nil || !sameRows(msgs, legacyRows[sqlc.ListAllChannelMessagesRow](t, ctx, tx, legacyListAllChannelMessages, pgtype.Timestamptz{}, []string(nil), "", c, n)) { + t.Fatalf("messages cursor %d limit %d differ: %v", c, n, err) + } + } + tel, err := q.GetObserverTelemetry(ctx, sqlc.GetObserverTelemetryParams{ObserverID: obs, Column4: int64(c)}) + if err != nil || !sameRows(tel, legacyRows[sqlc.GetObserverTelemetryRow](t, ctx, tx, legacyGetObserverTelemetry, obs, pgtype.Timestamptz{}, pgtype.Timestamptz{}, c)) { + t.Fatalf("telemetry cursor %d differ: %v", c, err) + } + } + }) +} diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 35c04f85..04a9d1e6 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -235,7 +235,7 @@ FROM observer_telemetry WHERE observer_id = $1 AND ($2::timestamptz IS NULL OR reported_at >= $2) AND ($3::timestamptz IS NULL OR reported_at <= $3) - AND ($4 = 0 OR id > $4) + AND ($4::bigint = 0 OR id > $4::bigint) ORDER BY reported_at ASC; -- name: GetObserverActivityRaw :many @@ -329,7 +329,7 @@ JOIN packets p ON p.packet_hash = po.packet_hash LEFT JOIN nodes n ON n.public_key = p.origin_pubkey WHERE po.observer_id = $1 AND p.payload_type = 4 - AND ($2 = 0 OR po.id > $2) + AND ($2::bigint = 0 OR po.id > $2::bigint) ORDER BY po.id ASC LIMIT $3; @@ -953,7 +953,7 @@ FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash JOIN nodes n ON n.public_key = p.origin_pubkey WHERE n.id = $1 - AND ($2 = 0 OR po.id < $2) + AND ($2::bigint = 0 OR po.id < $2::bigint) ORDER BY po.id DESC LIMIT $3; @@ -1148,7 +1148,7 @@ LEFT JOIN transport_scopes ts ON ts.id = p.scope_id WHERE ($1::timestamptz IS NULL OR cm.sent_at >= $1) AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR po.iata = ANY($2::bpchar[])) AND ($3::text = '' OR ts.name = $3::text) - AND ($4 = 0 OR cm.id < $4) + AND ($4::bigint = 0 OR cm.id < $4::bigint) ORDER BY cm.id DESC LIMIT $5; diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 348bffa7..67f0bc94 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1294,7 +1294,7 @@ FROM observer_telemetry WHERE observer_id = $1 AND ($2::timestamptz IS NULL OR reported_at >= $2) AND ($3::timestamptz IS NULL OR reported_at <= $3) - AND ($4 = 0 OR id > $4) + AND ($4::bigint = 0 OR id > $4::bigint) ORDER BY reported_at ASC ` @@ -1302,7 +1302,7 @@ type GetObserverTelemetryParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Timestamptz `json:"column_3"` - Column4 interface{} `json:"column_4"` + Column4 int64 `json:"column_4"` } type GetObserverTelemetryRow struct { @@ -2589,7 +2589,7 @@ LEFT JOIN transport_scopes ts ON ts.id = p.scope_id WHERE ($1::timestamptz IS NULL OR cm.sent_at >= $1) AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR po.iata = ANY($2::bpchar[])) AND ($3::text = '' OR ts.name = $3::text) - AND ($4 = 0 OR cm.id < $4) + AND ($4::bigint = 0 OR cm.id < $4::bigint) ORDER BY cm.id DESC LIMIT $5 ` @@ -2598,7 +2598,7 @@ type ListAllChannelMessagesParams struct { Column1 pgtype.Timestamptz `json:"column_1"` Column2 []string `json:"column_2"` Column3 string `json:"column_3"` - Column4 interface{} `json:"column_4"` + Column4 int64 `json:"column_4"` Limit int32 `json:"limit"` } @@ -3220,15 +3220,15 @@ FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash JOIN nodes n ON n.public_key = p.origin_pubkey WHERE n.id = $1 - AND ($2 = 0 OR po.id < $2) + AND ($2::bigint = 0 OR po.id < $2::bigint) ORDER BY po.id DESC LIMIT $3 ` type ListNodeObservationsParams struct { - ID uuid.UUID `json:"id"` - Column2 interface{} `json:"column_2"` - Limit int32 `json:"limit"` + ID uuid.UUID `json:"id"` + Column2 int64 `json:"column_2"` + Limit int32 `json:"limit"` } type ListNodeObservationsRow struct { @@ -3484,15 +3484,15 @@ JOIN packets p ON p.packet_hash = po.packet_hash LEFT JOIN nodes n ON n.public_key = p.origin_pubkey WHERE po.observer_id = $1 AND p.payload_type = 4 - AND ($2 = 0 OR po.id > $2) + AND ($2::bigint = 0 OR po.id > $2::bigint) ORDER BY po.id ASC LIMIT $3 ` type ListObserverAdvertsParams struct { - ObserverID uuid.UUID `json:"observer_id"` - Column2 interface{} `json:"column_2"` - Limit int32 `json:"limit"` + ObserverID uuid.UUID `json:"observer_id"` + Column2 int64 `json:"column_2"` + Limit int32 `json:"limit"` } type ListObserverAdvertsRow struct { From 194c0c1f08dfbe98cf8910422e5ca90f2fff68bc Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 09:17:19 -0400 Subject: [PATCH 19/23] fix(db): bound boot backfill writes and retry smaller ranges --- db/backfill_batch.go | 68 +++++++++ db/backfill_batch_integration_test.go | 129 ++++++++++++++++++ db/backfill_batch_test.go | 47 +++++++ db/cmi.go | 27 +++- db/packet_sites.go | 42 ++++-- db/packet_sites_reconcile_integration_test.go | 16 ++- 6 files changed, 307 insertions(+), 22 deletions(-) create mode 100644 db/backfill_batch.go create mode 100644 db/backfill_batch_integration_test.go create mode 100644 db/backfill_batch_test.go diff --git a/db/backfill_batch.go b/db/backfill_batch.go new file mode 100644 index 00000000..15fe483c --- /dev/null +++ b/db/backfill_batch.go @@ -0,0 +1,68 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +const backfillStatementTimeout = 5 * time.Second + +// Only one boot backfill writes at a time. Separate maintenance connections do +// not isolate the disk/WAL used by live ingest. +var backfillWriteSlot = make(chan struct{}, 1) + +// backfillWrite bounds one idempotent write with a server-side timeout. A local +// transaction restores the setting on success or failure without releasing the +// session advisory lock. In particular, a canceled statement must be rolled back +// before the caller retries an FK race or a smaller range. +func backfillWrite(ctx context.Context, conn *pgxpool.Conn, timeout time.Duration, write func() (int64, error)) (int64, error) { + select { + case backfillWriteSlot <- struct{}{}: + defer func() { <-backfillWriteSlot }() + case <-ctx.Done(): + return 0, ctx.Err() + } + if err := ctx.Err(); err != nil { + return 0, err + } + tx, err := conn.Begin(ctx) + if err != nil { + return 0, err + } + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = tx.Rollback(cleanup) + }() + // Never relax an operator's smaller timeout. pg_settings reports milliseconds. + if _, err := tx.Exec(ctx, `SELECT set_config('statement_timeout', + CASE WHEN setting::bigint = 0 THEN ($1::bigint)::text + ELSE LEAST(setting::bigint, $1::bigint)::text END, true) + FROM pg_settings WHERE name = 'statement_timeout'`, max(1, timeout.Milliseconds())); err != nil { + return 0, err + } + n, err := write() + if err != nil { + return 0, err + } + if err := tx.Commit(ctx); err != nil { + return 0, err + } + return n, nil +} + +func backfillTimedOut(ctx context.Context, err error) bool { + return ctx.Err() == nil && isStatementTimeout(err) +} + +func backfillTimeout(limit time.Duration, deadline time.Time) time.Duration { + if limit <= 0 { + limit = backfillStatementTimeout + } + return max(time.Millisecond, min(limit, time.Until(deadline))) +} diff --git a/db/backfill_batch_integration_test.go b/db/backfill_batch_integration_test.go new file mode 100644 index 00000000..a9a7539e --- /dev/null +++ b/db/backfill_batch_integration_test.go @@ -0,0 +1,129 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5/pgconn" +) + +func TestBackfillWriteBoundsPostgres(t *testing.T) { + f := newPsFixture(t) + p := f.begin() + if _, err := p.conn.Exec(f.ctx, "SET statement_timeout = '2s'"); err != nil { + t.Fatal(err) + } + _, err := backfillWrite(f.ctx, p.conn, 20*time.Millisecond, func() (int64, error) { + _, err := p.conn.Exec(f.ctx, `INSERT INTO background_state(key, value) VALUES ('timeout-test', '{}'); + SELECT pg_sleep(1)`) + return 1, err + }) + if !backfillTimedOut(f.ctx, err) { + t.Fatalf("expected server statement timeout, got %v", err) + } + var count int + if err := p.conn.QueryRow(f.ctx, "SELECT count(*) FROM background_state WHERE key = 'timeout-test'").Scan(&count); err != nil || count != 0 { + t.Fatalf("canceled write persisted or session unusable: count %d, %v", count, err) + } + var timeout string + if err := p.conn.QueryRow(f.ctx, "SHOW statement_timeout").Scan(&timeout); err != nil || timeout != "2s" { + t.Fatalf("timeout leaked: %q, %v", timeout, err) + } + if other, ok, err := f.store.BeginPacketSites(f.ctx); err != nil || ok { + if other != nil { + other.Close(f.ctx) + } + t.Fatalf("timeout lost session advisory lock: acquired %v, %v", ok, err) + } + // A subsequent attempt commits normally on the same connection. + if _, err := backfillWrite(f.ctx, p.conn, time.Second, func() (int64, error) { + tag, err := p.conn.Exec(f.ctx, "INSERT INTO background_state(key,value) VALUES ('timeout-test','{}')") + return tag.RowsAffected(), err + }); err != nil { + t.Fatal(err) + } + // A larger requested limit must not relax an operator's smaller setting. + if _, err := p.conn.Exec(f.ctx, "SET statement_timeout = '10ms'"); err != nil { + t.Fatal(err) + } + _, err = backfillWrite(f.ctx, p.conn, time.Second, func() (int64, error) { + _, err := p.conn.Exec(f.ctx, "SELECT pg_sleep(1)") + return 0, err + }) + if !backfillTimedOut(f.ctx, err) { + t.Fatalf("operator timeout was relaxed: %v", err) + } +} + +func TestCmiBackfillShrinksPostgres(t *testing.T) { + f := newCmiFixture(t) + for _, id := range []int64{1, 600} { + h := f.packet(string(rune('a'+id)), 3) + f.obs(f.pool, h, "YUL", true) + f.unseeded(id, 7, h) + } + c := f.begin() + c.pause = -1 + c.backfillLimit = 50 * time.Millisecond + if _, err := c.Init(f.ctx); err != nil { + t.Fatal(err) + } + held, err := f.pool.Begin(f.ctx) + if err != nil { + t.Fatal(err) + } + defer held.Rollback(context.Background()) //nolint:errcheck + if _, err := held.Exec(f.ctx, "SELECT id FROM channel_messages WHERE id = 600 FOR UPDATE"); err != nil { + t.Fatal(err) + } + st, err := c.Backfill(f.ctx, time.Second) + if !backfillTimedOut(f.ctx, err) || st.Complete || st.BackfilledToID <= 0 || st.BackfilledToID >= 600 { + t.Fatalf("expected a committed smaller prefix before the locked message: %+v, %v", st, err) + } + var saved CmiState + if found, err := f.store.LoadState(f.ctx, cmiStateKey, &saved); err != nil || !found || saved.BackfilledToID != st.BackfilledToID { + t.Fatalf("saved watermark %+v, %v, %v", saved, found, err) + } + if err := held.Rollback(f.ctx); err != nil { + t.Fatal(err) + } + st, err = c.Backfill(f.ctx, time.Minute) + if err != nil || !st.Complete || st.BackfilledToID != 600 { + t.Fatalf("resume: %+v, %v", st, err) + } + if check := f.check(); check.Missing != 0 || check.Extra != 0 { + t.Fatalf("range skipped or duplicated after retry: %+v", check) + } +} + +func TestPacketSitesBackfillShrinksPostgres(t *testing.T) { + f := newPsFixture(t) + stream := f.packets(10) + f.unsited(stream[:30]...) + p := f.begin() + if _, err := p.Init(f.ctx); err != nil { + t.Fatal(err) + } + original := p.fillStmt + var canceled int + p.fillStmt = func(ctx context.Context, after, through int64) (int64, error) { + if through-after > 8 { + canceled++ + return 0, &pgconn.PgError{Code: "57014", Message: "canceling statement due to statement timeout"} + } + return original(ctx, after, through) + } + st, err := p.Backfill(f.ctx, time.Minute) + if err != nil || !st.Complete || canceled == 0 || p.backfillSpan > 8 { + t.Fatalf("backfill did not shrink and complete: %+v, span %d, canceled %d, %v", st, p.backfillSpan, canceled, err) + } + f.wantClean("after timeout retries") + if n, err := sqlc.New(f.pool).FillPacketSitesSlice(f.ctx, sqlc.FillPacketSitesSliceParams{AfterID: 0, ThroughID: f.maxID()}); err != nil || n != 0 { + t.Fatalf("retry left incomplete rows: wrote %d, %v", n, err) + } +} diff --git a/db/backfill_batch_test.go b/db/backfill_batch_test.go new file mode 100644 index 00000000..dfbfc2c8 --- /dev/null +++ b/db/backfill_batch_test.go @@ -0,0 +1,47 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" +) + +func TestBackfillTimeout(t *testing.T) { + if got := backfillTimeout(0, time.Now().Add(time.Minute)); got != backfillStatementTimeout { + t.Fatalf("default timeout %v", got) + } + if got := backfillTimeout(time.Second, time.Now().Add(-time.Second)); got != time.Millisecond { + t.Fatalf("expired budget disabled the PostgreSQL timeout: %v", got) + } + if got := backfillTimeout(time.Second, time.Now().Add(50*time.Millisecond)); got <= 0 || got > 50*time.Millisecond { + t.Fatalf("remaining timeout %v", got) + } + err := &pgconn.PgError{Code: "57014"} + if !backfillTimedOut(context.Background(), err) || backfillTimedOut(context.Background(), errors.New("transport failed")) { + t.Fatal("only a PostgreSQL query cancellation permits a smaller retry") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if backfillTimedOut(ctx, err) { + t.Fatal("a canceled caller must stop, not retry") + } +} + +func TestBackfillAdmissionCancellation(t *testing.T) { + backfillWriteSlot <- struct{}{} + defer func() { <-backfillWriteSlot }() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := backfillWrite(ctx, nil, time.Second, func() (int64, error) { + t.Fatal("canceled waiter ran") + return 0, nil + }); !errors.Is(err, context.Canceled) { + t.Fatalf("waiting for admission: %v", err) + } +} diff --git a/db/cmi.go b/db/cmi.go index 0e8243ec..eb18a53d 100644 --- a/db/cmi.go +++ b/db/cmi.go @@ -21,8 +21,10 @@ import ( const ( cmiStateKey = "cmi" - // cmiBackfillBatch is the message ids one fill covers, in the backfill and the reconcile's message part. + // cmiBackfillBatch is the message ids one fill of reconciliation covers. cmiBackfillBatch int64 = 10_000 + // Boot starts with smaller writes than steady-state reconciliation. + cmiBootBatch int64 = 1_000 // cmiObsSlice is the observation ids one fill of the reconcile's observation part covers. cmiObsSlice int64 = 500_000 // cmiFillAttempts bounds the retries of a fill that lost a message to a concurrent cascade (23503). @@ -54,6 +56,8 @@ type CmiSession struct { // fills and retries count fill statements and 23503 retries, for tests. fills, retries int pause time.Duration // between backfill batches; 0 = cmiBatchPause, < 0 = none + backfillSpan int64 + backfillLimit time.Duration } func (c *CmiSession) batchPause() time.Duration { @@ -162,10 +166,14 @@ func (c *CmiSession) fillMessages(ctx context.Context, after, through int64) (in }) } -// Backfill fills cmi oldest to newest in batches of cmiBackfillBatch message ids, jumping id gaps, until it -// reaches BackfillMaxID (then Complete) or budget is spent. Progress is saved after every batch. +// Backfill fills cmi oldest to newest, jumping id gaps, until it reaches BackfillMaxID (then Complete) or budget is +// spent. Boot writes start at cmiBootBatch and halve after a statement timeout. Progress is saved only after a +// committed batch; retrying a failed or interrupted range is idempotent. func (c *CmiSession) Backfill(ctx context.Context, budget time.Duration) (CmiState, error) { deadline := time.Now().Add(budget) + if c.backfillSpan <= 0 { + c.backfillSpan = cmiBootBatch + } for !c.state.Complete && time.Now().Before(deadline) { if err := ctx.Err(); err != nil { return c.state, err @@ -182,8 +190,17 @@ func (c *CmiSession) Backfill(ctx context.Context, budget time.Duration) (CmiSta slog.Info("cmi backfill complete", "component", "readmodel", "max_id", c.state.BackfillMaxID) break } - hi := min(next-1+cmiBackfillBatch, c.state.BackfillMaxID) - if _, err := c.fillMessages(ctx, next-1, hi); err != nil { + hi := min(next-1+c.backfillSpan, c.state.BackfillMaxID) + _, err = c.fill(ctx, func() (int64, error) { + return backfillWrite(ctx, c.conn, backfillTimeout(c.backfillLimit, deadline), func() (int64, error) { + return c.q.FillChannelMessageIATAsByMessages(ctx, sqlc.FillChannelMessageIATAsByMessagesParams{AfterID: next - 1, ThroughID: hi}) + }) + }) + if err != nil { + if backfillTimedOut(ctx, err) && hi-(next-1) > 1 { + c.backfillSpan = max(1, (hi-(next-1))/2) + continue + } return c.state, fmt.Errorf("cmi backfill (%d, %d]: %w", next-1, hi, err) } c.state.BackfilledToID = hi diff --git a/db/packet_sites.go b/db/packet_sites.go index 4a3f1d8e..11846571 100644 --- a/db/packet_sites.go +++ b/db/packet_sites.go @@ -23,7 +23,8 @@ import ( const ( packetSitesStateKey = "packet_sites" // packetSitesSlice is the observation ids one fill covers, in the backfill, the reconcile and the audit. - packetSitesSlice int64 = 500_000 + packetSitesSlice int64 = 500_000 + packetSitesBootSlice int64 = 50_000 // packetSitesFillAttempts bounds the retries of a fill that lost a packet to a concurrent cascade (23503). packetSitesFillAttempts = 3 // packetSitesBackfillPause separates backfill slices, so the one-off backfill leaves room for other work. @@ -45,12 +46,14 @@ type PacketSitesState struct { // PacketSitesSession holds the packet_sites lock on one maintenance connection and the state it read after taking // the lock; every statement and state write goes through it. type PacketSitesSession struct { - store *Store - conn *pgxpool.Conn - q *sqlc.Queries - state PacketSitesState - found bool // the state existed when the lock was taken - pause time.Duration + store *Store + conn *pgxpool.Conn + q *sqlc.Queries + state PacketSitesState + found bool // the state existed when the lock was taken + pause time.Duration + backfillSpan int64 + backfillLimit time.Duration // fillStmt runs one fill; tests replace it. fills lists the (after, through] ranges filled, and retries counts // 23503 retries, for tests. @@ -150,11 +153,14 @@ func (p *PacketSitesSession) fillWith(ctx context.Context, after, through int64, return 0, err } -// Backfill fills packet_sites from WriterFromID down, one slice of packetSitesSlice observation ids at a time, -// newest first, until it passes the oldest observation (then Complete) or budget is spent. Progress is saved after -// every slice, and a failed slice is retried from the same place on the next call. +// Backfill fills packet_sites from WriterFromID down, newest first, until it passes the oldest observation (then +// Complete) or budget is spent. Boot writes start at packetSitesBootSlice and halve after a statement timeout. +// Progress is saved only after a committed slice; a failed slice is retried from the same place. func (p *PacketSitesSession) Backfill(ctx context.Context, budget time.Duration) (PacketSitesState, error) { deadline := time.Now().Add(budget) + if p.backfillSpan <= 0 { + p.backfillSpan = packetSitesBootSlice + } for !p.state.Complete && time.Now().Before(deadline) { if err := ctx.Err(); err != nil { return p.state, err @@ -171,9 +177,19 @@ func (p *PacketSitesSession) Backfill(ctx context.Context, budget time.Duration) slog.Info("packet_sites backfill complete", "component", "readmodel", "writer_from_id", p.state.WriterFromID) break } - from := max(p.state.BackfilledDownToID-packetSitesSlice, lo-1) - if _, err := p.fill(ctx, from, p.state.BackfilledDownToID); err != nil { - return p.state, fmt.Errorf("packet_sites backfill (%d, %d]: %w", from, p.state.BackfilledDownToID, err) + top := p.state.BackfilledDownToID + from := max(top-p.backfillSpan, lo-1) + _, err = p.fillWith(ctx, from, top, func(ctx context.Context, after, through int64) (int64, error) { + return backfillWrite(ctx, p.conn, backfillTimeout(p.backfillLimit, deadline), func() (int64, error) { + return p.fillStmt(ctx, after, through) + }) + }) + if err != nil { + if backfillTimedOut(ctx, err) && top-from > 1 { + p.backfillSpan = max(1, (top-from)/2) + continue + } + return p.state, fmt.Errorf("packet_sites backfill (%d, %d]: %w", from, top, err) } p.state.BackfilledDownToID = from if err := p.save(ctx); err != nil { diff --git a/db/packet_sites_reconcile_integration_test.go b/db/packet_sites_reconcile_integration_test.go index 2ddac7aa..502fff1a 100644 --- a/db/packet_sites_reconcile_integration_test.go +++ b/db/packet_sites_reconcile_integration_test.go @@ -120,15 +120,23 @@ func TestPacketSitesInitAndBackfillPostgres(t *testing.T) { if err != nil || !st.Complete { t.Fatalf("Backfill = %+v, %v", st, err) } - // 1.2M ids from the oldest: three slices, none below the oldest observation. - if fmt.Sprint(p.fills) != "[[3000700000 3001200000] [3000200000 3000700000] [3000000000 3000200000]]" { - t.Fatalf("fills %v", p.fills) + // The entire interval is covered once in smaller, contiguous boot slices. + end := hi + for _, r := range p.fills { + if r[1] != end || r[0] >= r[1] || r[1]-r[0] > packetSitesBootSlice { + t.Fatalf("unbounded or noncontiguous fills %v", p.fills) + } + end = r[0] + } + if end != 3_000_000_000 { + t.Fatalf("backfill stopped at %d", end) } f.wantClean("after backfill") if n, err := sqlc.New(f.pool).FillPacketSitesSlice(f.ctx, sqlc.FillPacketSitesSliceParams{AfterID: 0, ThroughID: f.maxID()}); err != nil || n != 0 { t.Fatalf("a second fill wrote %d, %v", n, err) } - if st, err := p.Backfill(f.ctx, time.Minute); err != nil || len(p.fills) != 3 || !st.Complete { + fills := len(p.fills) + if st, err := p.Backfill(f.ctx, time.Minute); err != nil || len(p.fills) != fills || !st.Complete { t.Fatalf("a complete backfill ran again: %+v, %v, %v", st, err, p.fills) } From 2f9396744837eb5ded07c619e92a2cc9612a072e Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 09:41:55 -0400 Subject: [PATCH 20/23] fix(api): encode JSON before committing response status --- internal/api/handlers/responses.go | 14 ++++++++++---- internal/api/handlers/responses_test.go | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/internal/api/handlers/responses.go b/internal/api/handlers/responses.go index f9363428..df844ca7 100644 --- a/internal/api/handlers/responses.go +++ b/internal/api/handlers/responses.go @@ -41,13 +41,19 @@ type APIError struct { Message string `json:"message"` // e.g. "channel not found" } -// respond writes a JSON response with the given status code and data. -// Logs if encoding fails since the header is already written at that point. +// respond encodes before committing the status, so an unsupported value cannot +// turn a serialization error into an empty success response. func respond(w http.ResponseWriter, status int, data any) { + body, err := json.Marshal(data) + if err != nil { + slog.Error("api: failed to encode response", "component", "api", "error", err) + status = http.StatusInternalServerError + body = []byte(`{"error":{"code":"internal_server_error","message":"Unable to return this response."}}`) + } w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) - if err := json.NewEncoder(w).Encode(data); err != nil { - slog.Error("api: failed to encode response", "component", "api", "error", err) + if _, err := w.Write(append(body, '\n')); err != nil { + slog.Error("api: failed to write response", "component", "api", "error", err) } } diff --git a/internal/api/handlers/responses_test.go b/internal/api/handlers/responses_test.go index 4450f1db..3204e05a 100644 --- a/internal/api/handlers/responses_test.go +++ b/internal/api/handlers/responses_test.go @@ -5,11 +5,31 @@ package handlers import ( "encoding/json" + "math" "net/http" "net/http/httptest" "testing" ) +func TestRespond_EncodingFailureIsJSONError(t *testing.T) { + for name, value := range map[string]any{ + "nonfinite signal": map[string]float64{"snr": math.NaN()}, + "unsupported type": make(chan int), + } { + t.Run(name, func(t *testing.T) { + w := httptest.NewRecorder() + respond(w, http.StatusOK, value) + if w.Code != http.StatusInternalServerError || w.Header().Get("Content-Type") != "application/json" { + t.Fatalf("status %d, headers %v", w.Code, w.Header()) + } + var body map[string]APIError + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || body["error"].Code != "internal_server_error" { + t.Fatalf("invalid error response %q: %v", w.Body.String(), err) + } + }) + } +} + func TestRespond_StatusAndContentType(t *testing.T) { w := httptest.NewRecorder() respond(w, http.StatusOK, map[string]string{"hello": "world"}) From 28fce241ab7b4685858f33146beb1cb31e08e2c2 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 11:35:42 -0400 Subject: [PATCH 21/23] test: align preview public-message fixture with updated ingest --- internal/ingest/channel_scope_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ingest/channel_scope_test.go b/internal/ingest/channel_scope_test.go index d0a7d2b1..2f7c8412 100644 --- a/internal/ingest/channel_scope_test.go +++ b/internal/ingest/channel_scope_test.go @@ -92,7 +92,7 @@ func TestChannelPublicMarkerUsesDecryptionKey(t *testing.T) { w.hub.AddScope(client, "public-marker", hub.Scope{Events: []hub.EventType{hub.EventChannelMessage, hub.EventObserverStatus}}) waitForSummarySubscriber(t, ctx, w.hub, client) packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeGrpTxt, 0), Payload: encryptedGroupText(t, 0x11, key, "Fixture", "test")} - w.handlePayloadTypeSideEffects(ctx, packet, "YKF", []byte{1}, RadioSettings{}, nil, nil, nil, 0) + w.handlePayloadTypeSideEffects(ctx, packet, "YKF", []byte{1}, RadioSettings{}, nil, nil, nil, 0, false) select { case event := <-client.Send: var got channelMessageEvent From f7b8064c710057099166aaf2b1c322d4afc22d46 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 11:35:43 -0400 Subject: [PATCH 22/23] fix(cache): reuse entries filled before a late singleflight miss --- internal/cache/cache.go | 12 ++++++++++-- internal/cache/cache_test.go | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 90cf83f5..23297a19 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -120,7 +120,7 @@ func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Durati } else { slog.DebugContext(ctx, "cache miss", "component", "cache") } - raw, err = fetchShared(ctx, c, key, ttl, func(ctx context.Context) (any, error) { return fetch(ctx) }) + raw, err = fetchShared(ctx, c, key, ttl, fetch) if err != nil { return zero, err } @@ -137,10 +137,18 @@ const sharedFetchTimeout = 30 * time.Second // fetchShared runs one fetch per key across concurrent misses and returns its JSON. The fetch // gets its own bounded context so one caller disconnecting can't fail the others, and each // caller still stops waiting when its own context ends. -func fetchShared(ctx context.Context, c *Client, key string, ttl time.Duration, fetch func(context.Context) (any, error)) ([]byte, error) { +func fetchShared[T any](ctx context.Context, c *Client, key string, ttl time.Duration, fetch func(context.Context) (T, error)) ([]byte, error) { ch := c.sf.DoChan(key, func() (any, error) { fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), sharedFetchTimeout) defer cancel() + // A previous flight may have filled the key after this caller's miss. + // Recheck inside the group so late callers do not start another DB read. + if data, err := c.rdb.Get(fctx, key).Bytes(); err == nil { + var cached T + if json.Unmarshal(data, &cached) == nil { + return data, nil + } + } val, err := fetch(fctx) if err != nil { return nil, err diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 0adf41a6..5314ee5a 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -24,6 +24,25 @@ func newTestClient(t *testing.T) (*Client, *miniredis.Miniredis) { return c, mr } +func TestSharedFetchRechecksCacheAfterLateMiss(t *testing.T) { + c, _ := newTestClient(t) + ctx := context.Background() + // A caller saw a miss, then another flight populated the key before it + // entered the group. That late caller must reuse the completed result. + if err := c.rdb.Set(ctx, "late-miss", `{"revision":7}`, time.Hour).Err(); err != nil { + t.Fatal(err) + } + calls := 0 + raw, err := fetchShared(ctx, c, "late-miss", time.Hour, func(context.Context) (any, error) { + calls++ + return &api.StatsSeries{Revision: 99}, nil + }) + var got api.StatsSeries + if err != nil || json.Unmarshal(raw, &got) != nil || got.Revision != 7 || calls != 0 { + t.Fatalf("late miss refetched: revision=%d calls=%d err=%v", got.Revision, calls, err) + } +} + // stubReader is a minimal api.Reader that returns preset values. type stubReader struct { iatas []api.IATA From 6df3ba57e420a1d851a94779352f2b2d11174782 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 20 Oct 2026 16:37:42 -0400 Subject: [PATCH 23/23] fix(db): keep batch size after a backfill cycle deadline --- .github/workflows/ci.yml | 268 +++++++++++++------------- db/backfill_batch.go | 8 +- db/backfill_batch_integration_test.go | 21 ++ db/cmi.go | 8 +- db/packet_sites.go | 7 +- 5 files changed, 174 insertions(+), 138 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4b381ae..b96dceee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,134 +1,134 @@ -# Copyright 2026 Beacon Contributors -# SPDX-License-Identifier: AGPL-3.0-or-later -name: CI - -on: - push: - branches: [main, dev, dev1] - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - services: - backup-postgres: - image: postgres:16-alpine - env: - POSTGRES_PASSWORD: backup-ci-only - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 5s - --health-timeout 5s - --health-retries 10 - # The fixture tier's baselines were written on PostgreSQL 16.15 under C collation: keep both fixed here. - planguard-postgres: - image: postgres:16.15-alpine - env: - POSTGRES_PASSWORD: planguard-ci-only - POSTGRES_DB: planguard - POSTGRES_INITDB_ARGS: --locale=C --encoding=UTF8 - ports: - - 5433:5432 - options: >- - --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache: true - - - name: Build - run: go build ./... - - - name: Format check - run: | - if [ -n "$(gofmt -l .)" ]; then - echo "The following files are not formatted:" - gofmt -l . - exit 1 - fi - - - name: Vet - run: go vet ./... - - - name: Test - run: go test ./... - - - name: Verify scope refresh concurrency - run: go test -race ./internal/meshmapper ./internal/scopestore && go test -race ./db -run '^TestChannelIDCache' - - - name: Verify backup startup and the read-only boot against PostgreSQL 16 - env: - BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./cmd/beacon -run '^Test(BackupConfiguration|ReadOnlyBootWritesNothing)Postgres$' -count=1 -v - - - name: Verify stats and endpoint queries against PostgreSQL 16 - env: - BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(SearchPacketsByHashPrefix|MeshMapperRepeaters|TopAdvertisersSort|TraceQuality|TopologyLinks|Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|KnownRoutesIndexSet|PacketsDomainChecks|Reconfirm[A-Za-z]*|BigintCursors|RouteSearchHopOrder|StatsNodeTypesSplit|ListKnownRoutes|ListKnownRoutesMatchesLegacy|KnownRoutesCursorTies|KnownRoutesUnanalyzedRegion|KnownRoutesHopStatementsStayCustom|KnownRoutePaginationTies|RouteRetentionCutoffsExact|DeleteExpiredRoutes|DeleteGraceRoutesSlice|GraceWatermarkHeldByLockedRow|OpenPoolsSettings|PlanRouterClassCUnnamed|APIStoreHopStatementsUnnamed|PgStatStatementsMigrationGuarded|DBStatements|DBStatementsNoExtension|PgStatStatementsMigrationNoCreatableSchema|IndexHealthBaselineOnReindex|UnusedIndexes|PgActivity|RollHourSetsWorkMem|SmallTableIndexSet|NodeReadersAfterIndexDrops|ObserverChannelReadersAfterIndexDrops|UpsertChannelWriteThrough|UpsertChannelHashOnlyWriteThrough|ReadOnlyPoolsRejectWrites|ListPacketsGlobalMatchesLegacy|PacketsGlobalCursorTies|ListPacketsWindowInvertedRows|PacketsLatestObserverTie|NodesLegacyCursorMatchesToday|NodesPageCursorTies|ObserversLegacyCursorMatchesToday|ObserversNextCursorFromLastSeen|ObserverAdvertsPartialIndex|AdvertsWaitForPayloadTypeBackfill|StatsProbeMatchesToday|TraceTagsKeysetMatchesToday|TelemetrySinceMatchesCatchAll|TraceTagPagination|TraceTagPaginationTies|TraceSummarySurvivesMissedHearing|CmiMigrationsIndexSet|InsertChannelMessageSeed[A-Za-z]*|AddChannelMessageIATA|CmiSeedAndAddRace|CmiInitAndBackfill|CmiMessageRangeReconcile|CmiObservationRangeReconcile|CmiBackfillRetries23503|WaitForPreBootWriters|ChannelMessagesByIATAsMatchesLegacy|ChannelMessagesZeroHitRegion|CmiTrustLifecycle|CmiFirstVerification|PacketSitesMigrationsIndexSet|InsertObservationMaintainsPacketSites|InsertObservationResendWritesNothing|PacketSitesRaceMaxMerge|InsertObservationSiteFallback|PacketSitesInitAndBackfill|FillPacketSitesSliceIdempotent|FillPacketSitesSliceRetries23503|PacketSitesReconcile|PacketSitesAudit|PacketSitesAuditTakesNewestHearing|PacketsBySitesMatchesTruth|PacketsBySitesCursorTies|PacketsBySitesManyBogusSites|PacketsBySitesSaturatedPagesFull|PacketsBySitesMatchesLegacy|PacketSitesTrustLifecycle|PacketsBySitesBounded|PacketsBySitesFallbackKeepsPageCursor|SweepPacketSitesFrontsPlan|IndexHealthPacketSitesFreshBaseline|PacketSitesFirstVerification|PacedBackfillSlices|FillSessionsPlanCustom|FillAnalyzeFailureIsIgnored|FillAnalyzeOnlyNeverAnalyzed|HygieneIndexSet|HygieneReadersAfterIndexDrops|KnownRoutesColumns|SequenceUse|SlowPoolIsolation|SlowPoolSize|SequenceUseMixedCaseSchema|ObservationsIndexSet|ReadersAfterIndexDrops|BackfillWriteBounds|CmiBackfillShrinks|PacketSitesBackfillShrinks)Postgres$' -count=1 -v - - - name: Verify the channel message site rule against PostgreSQL 16 - env: - BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./internal/ingest -run '^Test(CmiAddOnDecryptFailure|CmiCleanIngestReconcilesNothing)Postgres$' -count=1 -v - - - name: Verify route retention task against PostgreSQL 16 - env: - BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./internal/background -run '^Test(RouteRetentionTask|PayloadTypeBackfill|PayloadTypeBackfillProbe|CmiBootRepair|CmiReadOnlyTrust|PacketSitesRollbackRollForward|PacketSitesTrustDropsOnBypass|PacketSitesReadOnlyTrust|PacedFillsEarnTrust)Postgres$' -count=1 -v - - # Builds the fixture once; the planguard tests below reuse it (PLANGUARD_FIXTURE_DSN) after checking that its - # marker records a build of the same files. - - name: Planguard fixture tier (plain EXPLAIN on a migrated, skewed fixture) - run: go run ./cmd/planguard run --tier fixture --dsn 'postgres://postgres:planguard-ci-only@127.0.0.1:5433/planguard?sslmode=disable' --out "$RUNNER_TEMP/planguard" - - - name: Verify planguard against PostgreSQL 16 - env: - BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - PLANGUARD_FIXTURE_DSN: postgres://postgres:planguard-ci-only@127.0.0.1:5433/planguard?sslmode=disable - run: go test ./internal/planguard -timeout 20m -run '^Test(RunnerReadOnly|RunnerRecoversAfterError|UnresolvedParamsSkip|GenericModeShowsParams|CorpusStatementsPrepareClean|BuildFixtureThroughMigrations|BuildFixtureRefusesRealDatabase|BuildFixtureRebuildsStaleInputs|RunDiffSnapshot|RunDiffFindsReadmodelsSkip|StatementScannerMatches|PlanguardProdTier|PlanguardProdCancel|RequiresIndexSkips|RetiredIndexesDropped)Postgres$' -count=1 -v - - - name: Plan policy up to date - run: go run ./cmd/planguard gen-policy && git diff --exit-code db/planpolicy - - - name: Verify backup command against PostgreSQL 16 - run: | - CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup - CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup.test" ./internal/backup - CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup-http.test" ./internal/api/router - docker run --rm --network host \ - -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ - -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable \ - -e BEACON_BACKUP_TEST_POSTGRES=1 -e BEACON_BACKUP_TEST_BINARY=/beacon-backup \ - -v "$RUNNER_TEMP/beacon-backup:/beacon-backup:ro" \ - -v "$RUNNER_TEMP/backup.test:/backup.test:ro" \ - postgres:16-alpine /backup.test -test.run '^TestExportPostgres$' -test.v - docker run --rm --network host \ - -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ - -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable -e BEACON_BACKUP_TEST_POSTGRES=1 \ - -v "$RUNNER_TEMP/backup-http.test:/backup-http.test:ro" \ - postgres:16-alpine /backup-http.test -test.run '^TestBackupDownloadPostgres$' -test.v - - - name: Install govulncheck - run: go install golang.org/x/vuln/cmd/govulncheck@latest - - - name: Vulnerability check - run: govulncheck ./... - - - name: Install swag - run: go install github.com/swaggo/swag/cmd/swag@latest - - - name: Swagger docs up to date - run: | - cp docs/swagger.json docs/swagger.json.bak - swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency - if ! diff -q docs/swagger.json docs/swagger.json.bak > /dev/null; then - echo "Swagger docs are out of date. Run swag init and commit the result." - rm docs/swagger.json.bak - exit 1 - fi - rm docs/swagger.json.bak +# Copyright 2026 Beacon Contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +name: CI + +on: + push: + branches: [main, dev, dev1] + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + services: + backup-postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: backup-ci-only + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + # The fixture tier's baselines were written on PostgreSQL 16.15 under C collation: keep both fixed here. + planguard-postgres: + image: postgres:16.15-alpine + env: + POSTGRES_PASSWORD: planguard-ci-only + POSTGRES_DB: planguard + POSTGRES_INITDB_ARGS: --locale=C --encoding=UTF8 + ports: + - 5433:5432 + options: >- + --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build + run: go build ./... + + - name: Format check + run: | + if [ -n "$(gofmt -l .)" ]; then + echo "The following files are not formatted:" + gofmt -l . + exit 1 + fi + + - name: Vet + run: go vet ./... + + - name: Test + run: go test ./... + + - name: Verify scope refresh concurrency + run: go test -race ./internal/meshmapper ./internal/scopestore && go test -race ./db -run '^TestChannelIDCache' + + - name: Verify backup startup and the read-only boot against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./cmd/beacon -run '^Test(BackupConfiguration|ReadOnlyBootWritesNothing)Postgres$' -count=1 -v + + - name: Verify stats and endpoint queries against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./db -run '^Test(SearchPacketsByHashPrefix|MeshMapperRepeaters|TopAdvertisersSort|TraceQuality|TopologyLinks|Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|KnownRoutesIndexSet|PacketsDomainChecks|Reconfirm[A-Za-z]*|BigintCursors|RouteSearchHopOrder|StatsNodeTypesSplit|ListKnownRoutes|ListKnownRoutesMatchesLegacy|KnownRoutesCursorTies|KnownRoutesUnanalyzedRegion|KnownRoutesHopStatementsStayCustom|KnownRoutePaginationTies|RouteRetentionCutoffsExact|DeleteExpiredRoutes|DeleteGraceRoutesSlice|GraceWatermarkHeldByLockedRow|OpenPoolsSettings|PlanRouterClassCUnnamed|APIStoreHopStatementsUnnamed|PgStatStatementsMigrationGuarded|DBStatements|DBStatementsNoExtension|PgStatStatementsMigrationNoCreatableSchema|IndexHealthBaselineOnReindex|UnusedIndexes|PgActivity|RollHourSetsWorkMem|SmallTableIndexSet|NodeReadersAfterIndexDrops|ObserverChannelReadersAfterIndexDrops|UpsertChannelWriteThrough|UpsertChannelHashOnlyWriteThrough|ReadOnlyPoolsRejectWrites|ListPacketsGlobalMatchesLegacy|PacketsGlobalCursorTies|ListPacketsWindowInvertedRows|PacketsLatestObserverTie|NodesLegacyCursorMatchesToday|NodesPageCursorTies|ObserversLegacyCursorMatchesToday|ObserversNextCursorFromLastSeen|ObserverAdvertsPartialIndex|AdvertsWaitForPayloadTypeBackfill|StatsProbeMatchesToday|TraceTagsKeysetMatchesToday|TelemetrySinceMatchesCatchAll|TraceTagPagination|TraceTagPaginationTies|TraceSummarySurvivesMissedHearing|CmiMigrationsIndexSet|InsertChannelMessageSeed[A-Za-z]*|AddChannelMessageIATA|CmiSeedAndAddRace|CmiInitAndBackfill|CmiMessageRangeReconcile|CmiObservationRangeReconcile|CmiBackfillRetries23503|WaitForPreBootWriters|ChannelMessagesByIATAsMatchesLegacy|ChannelMessagesZeroHitRegion|CmiTrustLifecycle|CmiFirstVerification|PacketSitesMigrationsIndexSet|InsertObservationMaintainsPacketSites|InsertObservationResendWritesNothing|PacketSitesRaceMaxMerge|InsertObservationSiteFallback|PacketSitesInitAndBackfill|FillPacketSitesSliceIdempotent|FillPacketSitesSliceRetries23503|PacketSitesReconcile|PacketSitesAudit|PacketSitesAuditTakesNewestHearing|PacketsBySitesMatchesTruth|PacketsBySitesCursorTies|PacketsBySitesManyBogusSites|PacketsBySitesSaturatedPagesFull|PacketsBySitesMatchesLegacy|PacketSitesTrustLifecycle|PacketsBySitesBounded|PacketsBySitesFallbackKeepsPageCursor|SweepPacketSitesFrontsPlan|IndexHealthPacketSitesFreshBaseline|PacketSitesFirstVerification|PacedBackfillSlices|FillSessionsPlanCustom|FillAnalyzeFailureIsIgnored|FillAnalyzeOnlyNeverAnalyzed|HygieneIndexSet|HygieneReadersAfterIndexDrops|KnownRoutesColumns|SequenceUse|SlowPoolIsolation|SlowPoolSize|SequenceUseMixedCaseSchema|ObservationsIndexSet|ReadersAfterIndexDrops|BackfillWriteBounds|CmiBackfillShrinks|PacketSitesBackfillShrinks|PacketSitesBackfillBudgetDoesNotShrink)Postgres$' -count=1 -v + + - name: Verify the channel message site rule against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./internal/ingest -run '^Test(CmiAddOnDecryptFailure|CmiCleanIngestReconcilesNothing)Postgres$' -count=1 -v + + - name: Verify route retention task against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./internal/background -run '^Test(RouteRetentionTask|PayloadTypeBackfill|PayloadTypeBackfillProbe|CmiBootRepair|CmiReadOnlyTrust|PacketSitesRollbackRollForward|PacketSitesTrustDropsOnBypass|PacketSitesReadOnlyTrust|PacedFillsEarnTrust)Postgres$' -count=1 -v + + # Builds the fixture once; the planguard tests below reuse it (PLANGUARD_FIXTURE_DSN) after checking that its + # marker records a build of the same files. + - name: Planguard fixture tier (plain EXPLAIN on a migrated, skewed fixture) + run: go run ./cmd/planguard run --tier fixture --dsn 'postgres://postgres:planguard-ci-only@127.0.0.1:5433/planguard?sslmode=disable' --out "$RUNNER_TEMP/planguard" + + - name: Verify planguard against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + PLANGUARD_FIXTURE_DSN: postgres://postgres:planguard-ci-only@127.0.0.1:5433/planguard?sslmode=disable + run: go test ./internal/planguard -timeout 20m -run '^Test(RunnerReadOnly|RunnerRecoversAfterError|UnresolvedParamsSkip|GenericModeShowsParams|CorpusStatementsPrepareClean|BuildFixtureThroughMigrations|BuildFixtureRefusesRealDatabase|BuildFixtureRebuildsStaleInputs|RunDiffSnapshot|RunDiffFindsReadmodelsSkip|StatementScannerMatches|PlanguardProdTier|PlanguardProdCancel|RequiresIndexSkips|RetiredIndexesDropped)Postgres$' -count=1 -v + + - name: Plan policy up to date + run: go run ./cmd/planguard gen-policy && git diff --exit-code db/planpolicy + + - name: Verify backup command against PostgreSQL 16 + run: | + CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup + CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup.test" ./internal/backup + CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup-http.test" ./internal/api/router + docker run --rm --network host \ + -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ + -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable \ + -e BEACON_BACKUP_TEST_POSTGRES=1 -e BEACON_BACKUP_TEST_BINARY=/beacon-backup \ + -v "$RUNNER_TEMP/beacon-backup:/beacon-backup:ro" \ + -v "$RUNNER_TEMP/backup.test:/backup.test:ro" \ + postgres:16-alpine /backup.test -test.run '^TestExportPostgres$' -test.v + docker run --rm --network host \ + -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ + -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable -e BEACON_BACKUP_TEST_POSTGRES=1 \ + -v "$RUNNER_TEMP/backup-http.test:/backup-http.test:ro" \ + postgres:16-alpine /backup-http.test -test.run '^TestBackupDownloadPostgres$' -test.v + + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + + - name: Vulnerability check + run: govulncheck ./... + + - name: Install swag + run: go install github.com/swaggo/swag/cmd/swag@latest + + - name: Swagger docs up to date + run: | + cp docs/swagger.json docs/swagger.json.bak + swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency + if ! diff -q docs/swagger.json docs/swagger.json.bak > /dev/null; then + echo "Swagger docs are out of date. Run swag init and commit the result." + rm docs/swagger.json.bak + exit 1 + fi + rm docs/swagger.json.bak diff --git a/db/backfill_batch.go b/db/backfill_batch.go index 15fe483c..8e1d0060 100644 --- a/db/backfill_batch.go +++ b/db/backfill_batch.go @@ -60,9 +60,13 @@ func backfillTimedOut(ctx context.Context, err error) bool { return ctx.Err() == nil && isStatementTimeout(err) } -func backfillTimeout(limit time.Duration, deadline time.Time) time.Duration { +func backfillMaxTimeout(limit time.Duration) time.Duration { if limit <= 0 { limit = backfillStatementTimeout } - return max(time.Millisecond, min(limit, time.Until(deadline))) + return limit +} + +func backfillTimeout(limit time.Duration, deadline time.Time) time.Duration { + return max(time.Millisecond, min(backfillMaxTimeout(limit), time.Until(deadline))) } diff --git a/db/backfill_batch_integration_test.go b/db/backfill_batch_integration_test.go index a9a7539e..6dafa0e0 100644 --- a/db/backfill_batch_integration_test.go +++ b/db/backfill_batch_integration_test.go @@ -127,3 +127,24 @@ func TestPacketSitesBackfillShrinksPostgres(t *testing.T) { t.Fatalf("retry left incomplete rows: wrote %d, %v", n, err) } } + +func TestPacketSitesBackfillBudgetDoesNotShrinkPostgres(t *testing.T) { + f := newPsFixture(t) + stream := f.packets(10) + f.unsited(stream[:30]...) + p := f.begin() + if _, err := p.Init(f.ctx); err != nil { + t.Fatal(err) + } + before := p.state.BackfilledDownToID + p.backfillLimit = time.Second + var calls int + p.fillStmt = func(context.Context, int64, int64) (int64, error) { + calls++ + return 0, &pgconn.PgError{Code: "57014", Message: "canceling statement due to statement timeout"} + } + st, err := p.Backfill(f.ctx, 100*time.Millisecond) + if err != nil || calls != 1 || p.backfillSpan != packetSitesBootSlice || st.BackfilledDownToID != before { + t.Fatalf("a budget-limited attempt shrank the next cycle: calls=%d span=%d state=%+v err=%v", calls, p.backfillSpan, st, err) + } +} diff --git a/db/cmi.go b/db/cmi.go index 263a4564..39c3b9c0 100644 --- a/db/cmi.go +++ b/db/cmi.go @@ -207,12 +207,18 @@ func (c *CmiSession) Backfill(ctx context.Context, budget time.Duration) (CmiSta c.progress.Start(c.state.BackfilledToID) } hi := min(next-1+c.backfillSpan, c.state.BackfillMaxID) + timeout := backfillTimeout(c.backfillLimit, deadline) _, err = c.fill(ctx, func() (int64, error) { - return backfillWrite(ctx, c.conn, backfillTimeout(c.backfillLimit, deadline), func() (int64, error) { + return backfillWrite(ctx, c.conn, timeout, func() (int64, error) { return c.q.FillChannelMessageIATAsByMessages(ctx, sqlc.FillChannelMessageIATAsByMessagesParams{AfterID: next - 1, ThroughID: hi}) }) }) if err != nil { + // A short remaining cycle is not evidence that the batch is too large. + // Leave its watermark and size intact for the next full budget. + if backfillTimedOut(ctx, err) && timeout < backfillMaxTimeout(c.backfillLimit) { + return c.state, nil + } if backfillTimedOut(ctx, err) && hi-(next-1) > 1 { c.backfillSpan = max(1, (hi-(next-1))/2) continue diff --git a/db/packet_sites.go b/db/packet_sites.go index 8d63903f..e8314142 100644 --- a/db/packet_sites.go +++ b/db/packet_sites.go @@ -208,12 +208,17 @@ func (p *PacketSitesSession) Backfill(ctx context.Context, budget time.Duration) } top := p.state.BackfilledDownToID from := max(top-p.backfillSpan, lo-1) + timeout := backfillTimeout(p.backfillLimit, deadline) _, err = p.fillWith(ctx, from, top, func(ctx context.Context, after, through int64) (int64, error) { - return backfillWrite(ctx, p.conn, backfillTimeout(p.backfillLimit, deadline), func() (int64, error) { + return backfillWrite(ctx, p.conn, timeout, func() (int64, error) { return p.fillStmt(ctx, after, through) }) }) if err != nil { + // Do not carry a deadline-induced shrink into every later cycle. + if backfillTimedOut(ctx, err) && timeout < backfillMaxTimeout(p.backfillLimit) { + return p.state, nil + } if backfillTimedOut(ctx, err) && top-from > 1 { p.backfillSpan = max(1, (top-from)/2) continue