From 3f12bea4f3d808bae7c2a7291f83d39cfa0b124e Mon Sep 17 00:00:00 2001 From: dborup Date: Wed, 7 Oct 2026 12:05:04 +0200 Subject: [PATCH 1/5] fix(packets): exclude payload types before pagination (#242) --- .github/workflows/deploy.yml | 1 + cmd/server/db.go | 3 + cmd/server/openapi.go | 1 + cmd/server/packet_exclusions.go | 63 ++++++++ cmd/server/packet_exclusions_242_test.go | 177 +++++++++++++++++++++++ cmd/server/routes.go | 10 ++ cmd/server/store.go | 12 +- docs/api-spec.md | 21 ++- public/packets.js | 52 +++++-- test-issue-242-type-exclusions-e2e.js | 140 ++++++++++++++++++ test-issue-96-hide-control-e2e.js | 13 +- test-issue-96-hide-control.js | 5 +- test-packets.js | 18 +++ 13 files changed, 491 insertions(+), 25 deletions(-) create mode 100644 cmd/server/packet_exclusions.go create mode 100644 cmd/server/packet_exclusions_242_test.go create mode 100644 test-issue-242-type-exclusions-e2e.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 229aa62b7..58be4e8b4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -443,6 +443,7 @@ jobs: BASE_URL=http://localhost:13581 node test-issue-147-packets-url-obs-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-packet-detail-sender-hash-size-obs-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-96-hide-control-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node test-issue-242-type-exclusions-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-180-packets-url-modal-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-slideover-1168-munger-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-logo-pulse-1173-e2e.js 2>&1 | tee -a e2e-output.txt diff --git a/cmd/server/db.go b/cmd/server/db.go index 5c8a88737..d68d7d465 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -687,6 +687,7 @@ type PacketQuery struct { Limit int Offset int Type *int + ExcludeTypes packetTypeExclusions Route *int Observer string Hash string @@ -904,6 +905,7 @@ func parseDistinctIatasCSV(v interface{}) []string { func (db *DB) buildPacketWhere(q PacketQuery) ([]string, []interface{}) { var where []string var args []interface{} + where, args = q.ExcludeTypes.appendSQL(where, args, "payload_type") if q.Type != nil { where = append(where, "payload_type = ?") @@ -948,6 +950,7 @@ func (db *DB) buildPacketWhere(q PacketQuery) ([]string, []interface{}) { func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { var where []string var args []interface{} + where, args = q.ExcludeTypes.appendSQL(where, args, "t.payload_type") if q.Type != nil { where = append(where, "t.payload_type = ?") diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 240ced49b..ee1d5a985 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -61,6 +61,7 @@ func routeDescriptions() map[string]routeMeta { {Name: "sort", Description: "Sort field", Type: "string"}, {Name: "order", Description: "Sort order (asc/desc)", Type: "string"}, {Name: "type", Description: "Filter by packet type", Type: "string"}, + {Name: "excludeTypes", Description: "Comma-separated numeric payload types 0-15 excluded before pagination and total counting (raw and grouped). At most 16 entries and 64 characters; duplicates collapse. Empty means no exclusions; NULL payload types remain. Intersects with type; exclusion wins on overlap. Invalid/repeated parameters or combination with nodes return 400; single node is supported.", Type: "string"}, {Name: "observer", Description: "Filter by observer ID", Type: "string"}, {Name: "timeRange", Description: "Time range filter (e.g. 1h, 24h, 7d)", Type: "string"}, {Name: "search", Description: "Full-text search", Type: "string"}, diff --git a/cmd/server/packet_exclusions.go b/cmd/server/packet_exclusions.go new file mode 100644 index 000000000..c114f2f6c --- /dev/null +++ b/cmd/server/packet_exclusions.go @@ -0,0 +1,63 @@ +package main + +import ( + "fmt" + "net/url" + "strconv" + "strings" +) + +// Packet.h defines PH_TYPE_MASK=0x0F: include reserved wire codes, not just +// the currently named types. A mask bounds storage and canonicalizes duplicates. +type packetTypeExclusions uint16 + +func parsePacketTypeExclusions(values url.Values) (packetTypeExclusions, error) { + entries := values["excludeTypes"] + if len(entries) == 0 { + return 0, nil + } + if len(entries) != 1 || len(entries[0]) > 64 { + return 0, fmt.Errorf("excludeTypes must be one comma-separated list of at most 16 types (0-15)") + } + if entries[0] == "" { + return 0, nil + } + parts := strings.Split(entries[0], ",") + if len(parts) > 16 { + return 0, fmt.Errorf("excludeTypes accepts at most 16 entries") + } + var mask packetTypeExclusions + for _, part := range parts { + part = strings.TrimSpace(part) + if len(part) < 1 || len(part) > 2 || part[0] < '0' || part[0] > '9' { + return 0, fmt.Errorf("excludeTypes entries must be integers from 0 to 15") + } + n, err := strconv.Atoi(part) + if err != nil || n < 0 || n > 15 { + return 0, fmt.Errorf("excludeTypes entries must be integers from 0 to 15") + } + mask |= 1 << uint(n) + } + return mask, nil +} + +func (mask packetTypeExclusions) excludes(typ *int) bool { + return typ != nil && *typ >= 0 && *typ < 16 && mask&(1<