fix(packets): filter excluded types before pagination (#242)

.github/workflows/deploy.yml

@@ -444,6 +444,7 @@ jobs:
           BASE_URL=http://localhost:13581 node test-issue-165-grouped-hop-warn-e2e.js 2>&1 | tee -a e2e-output.txt
           BASE_URL=http://localhost:13581 node test-packet-detail-sender-hash-size-obs-e2e.js 2>&1 | tee -a e2e-output.txt
           BASE_URL=http://localhost:13581 node test-issue-96-hide-control-e2e.js 2>&1 | tee -a e2e-output.txt
+          BASE_URL=http://localhost:13581 node test-issue-242-type-exclusions-e2e.js 2>&1 | tee -a e2e-output.txt
           BASE_URL=http://localhost:13581 node test-issue-180-packets-url-modal-e2e.js 2>&1 | tee -a e2e-output.txt
           BASE_URL=http://localhost:13581 node test-slideover-1168-munger-e2e.js 2>&1 | tee -a e2e-output.txt
           BASE_URL=http://localhost:13581 node test-logo-pulse-1173-e2e.js 2>&1 | tee -a e2e-output.txt

cmd/server/db.go

@@ -687,6 +687,7 @@ type PacketQuery struct {
 	Limit              int
 	Offset             int
 	Type               *int
+	ExcludeTypes       packetTypeExclusions
 	Route              *int
 	Observer           string
 	Hash               string
@@ -918,6 +919,7 @@ func parseDistinctIatasCSV(v interface{}) []string {
 func (db *DB) buildPacketWhere(q PacketQuery) ([]string, []interface{}) {
 	var where []string
 	var args []interface{}
+	where, args = q.ExcludeTypes.appendSQL(where, args, "payload_type")
 
 	if q.Type != nil {
 		where = append(where, "payload_type = ?")
@@ -962,6 +964,7 @@ func (db *DB) buildPacketWhere(q PacketQuery) ([]string, []interface{}) {
 func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) {
 	var where []string
 	var args []interface{}
+	where, args = q.ExcludeTypes.appendSQL(where, args, "t.payload_type")
 
 	if q.Type != nil {
 		where = append(where, "t.payload_type = ?")

cmd/server/openapi.go

@@ -61,6 +61,7 @@ func routeDescriptions() map[string]routeMeta {
 				{Name: "sort", Description: "Sort field", Type: "string"},
 				{Name: "order", Description: "Sort order (asc/desc)", Type: "string"},
 				{Name: "type", Description: "Filter by packet type", Type: "string"},
+				{Name: "excludeTypes", Description: "Comma-separated numeric payload types 0-15 excluded before pagination and total counting (raw and grouped). At most 16 entries and 64 characters; duplicates collapse. Empty means no exclusions; NULL payload types remain. Intersects with type; exclusion wins on overlap. Invalid/repeated parameters or combination with nodes return 400; single node is supported.", Type: "string"},
 				{Name: "observer", Description: "Filter by observer ID", Type: "string"},
 				{Name: "timeRange", Description: "Time range filter (e.g. 1h, 24h, 7d)", Type: "string"},
 				{Name: "search", Description: "Full-text search", Type: "string"},

cmd/server/packet_exclusions.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+// Packet.h defines PH_TYPE_MASK=0x0F: include reserved wire codes, not just
+// the currently named types. A mask bounds storage and canonicalizes duplicates.
+type packetTypeExclusions uint16
+
+func parsePacketTypeExclusions(values url.Values) (packetTypeExclusions, error) {
+	entries := values["excludeTypes"]
+	if len(entries) == 0 {
+		return 0, nil
+	}
+	if len(entries) != 1 || len(entries[0]) > 64 {
+		return 0, fmt.Errorf("excludeTypes must be one comma-separated list of at most 16 types (0-15)")
+	}
+	if entries[0] == "" {
+		return 0, nil
+	}
+	parts := strings.Split(entries[0], ",")
+	if len(parts) > 16 {
+		return 0, fmt.Errorf("excludeTypes accepts at most 16 entries")
+	}
+	var mask packetTypeExclusions
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if len(part) < 1 || len(part) > 2 || part[0] < '0' || part[0] > '9' {
+			return 0, fmt.Errorf("excludeTypes entries must be integers from 0 to 15")
+		}
+		n, err := strconv.Atoi(part)
+		if err != nil || n < 0 || n > 15 {
+			return 0, fmt.Errorf("excludeTypes entries must be integers from 0 to 15")
+		}
+		mask |= 1 << uint(n)
+	}
+	return mask, nil
+}
+
+// Test mask first: the default (no exclusion) must not dereference every
+// tx's separately allocated PayloadType in the filter loop.
+func (mask packetTypeExclusions) excludes(typ *int) bool {
+	return mask != 0 && typ != nil && *typ >= 0 && *typ < 16 && mask&(1<<uint(*typ)) != 0
+}
+
+// Keep unknown/NULL payload types, matching the in-memory predicate. The
+// caller supplies a fixed SQL column name; all values remain parameters.
+func (mask packetTypeExclusions) appendSQL(where []string, args []interface{}, column string) ([]string, []interface{}) {
+	if mask == 0 {
+		return where, args
+	}
+	placeholders := make([]string, 0, 16)
+	for typ := 0; typ < 16; typ++ {
+		if mask&(1<<uint(typ)) != 0 {
+			placeholders = append(placeholders, "?")
+			args = append(args, typ)
+		}
+	}
+	where = append(where, "("+column+" IS NULL OR "+column+" NOT IN ("+strings.Join(placeholders, ",")+"))")
+	return where, args
+}

cmd/server/packet_exclusions_242_test.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+)
+
+// The newest page contains only CONTROL; exclusion must precede pagination.
+func TestPacketExclusionsBeforePagination(t *testing.T) {
+	for _, mode := range []string{"memory", "database", "fallback"} {
+		t.Run(mode, func(t *testing.T) {
+			db := setupTestDB(t)
+			defer db.Close()
+			for i, typ := range []interface{}{nil, 5, 11, 11, 11} {
+				_, err := db.conn.Exec(`INSERT INTO transmissions(raw_hex,hash,first_seen,payload_type) VALUES('AA',?,?,?)`, fmt.Sprintf("%016x", i+1), fmt.Sprintf("2026-01-01T00:00:%02dZ", i), typ)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			srv := NewServer(db, &Config{}, NewHub())
+			if mode != "database" {
+				srv.store = NewPacketStore(db, nil)
+				if err := srv.store.Load(); err != nil {
+					t.Fatal(err)
+				}
+				if !srv.store.WaitIndexesReady(5 * time.Second) {
+					t.Fatal("indexes")
+				}
+				if mode == "fallback" {
+					srv.store.oldestLoaded = "2026-01-02"
+				}
+			}
+			for _, grouped := range []bool{false, true} {
+				suffix := fmt.Sprintf("&groupByHash=%t", grouped)
+				if mode == "fallback" {
+					suffix += "&until=2026-01-01T12:00:00"
+				}
+				check := func(query string, total int, wantHash string) {
+					t.Helper()
+					w := httptest.NewRecorder()
+					srv.handlePackets(w, httptest.NewRequest("GET", "/api/packets?limit=1"+suffix+query, nil))
+					if w.Code != 200 {
+						t.Fatalf("%s: %d %s", query, w.Code, w.Body.String())
+					}
+					var got PacketResult
+					if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
+						t.Fatal(err)
+					}
+					if got.Total != total {
+						t.Fatalf("%s grouped=%t total=%d want %d", query, grouped, got.Total, total)
+					}
+					if wantHash == "" {
+						if len(got.Packets) != 0 {
+							t.Fatal("expected empty")
+						}
+						return
+					}
+					if len(got.Packets) != 1 || got.Packets[0]["hash"] != wantHash {
+						t.Fatalf("%s packets=%v want %s", query, got.Packets, wantHash)
+					}
+				}
+				check("", 5, "0000000000000005")
+				check("&excludeTypes=11", 2, "0000000000000002")
+				check("&excludeTypes=11&offset=1", 2, "0000000000000001")
+				check("&excludeTypes=11,5", 1, "0000000000000001")
+				check("&excludeTypes=11&type=11", 0, "")
+				check("&excludeTypes=11&type=5", 1, "0000000000000002")
+				check("&excludeTypes=11&hash=0000000000000005", 0, "")
+				check("", 5, "0000000000000005") // grouped cache must not leak exclusions
+			}
+		})
+	}
+}
+
+func TestPacketExclusionMaskAndIndexes(t *testing.T) {
+	left, err := parsePacketTypeExclusions(url.Values{"excludeTypes": {"11,5,11"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	right, err := parsePacketTypeExclusions(url.Values{"excludeTypes": {"5,11"}})
+	if err != nil || left != right {
+		t.Fatal("equivalent lists must share cache identity")
+	}
+	all, err := parsePacketTypeExclusions(url.Values{"excludeTypes": {"0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for typ := 0; typ < 16; typ++ {
+		if !all.excludes(&typ) {
+			t.Errorf("wire type %d not excluded", typ)
+		}
+	}
+	unknown := 99
+	if all.excludes(nil) || all.excludes(&unknown) {
+		t.Fatal("unknown types must remain")
+	}
+	db := setupTestDB(t)
+	defer db.Close()
+	store := NewPacketStore(db, nil)
+	pk := strings.Repeat("a", 64)
+	for i, typ := range []int{5, 11} {
+		typ := typ
+		tx := &StoreTx{ID: i + 1, Hash: fmt.Sprintf("%016x", i+1), FirstSeen: "2026-01-01", PayloadType: &typ, Observations: []*StoreObs{{ObserverID: "test-observer"}}}
+		store.packets = append(store.packets, tx)
+		store.byHash[tx.Hash] = tx
+		store.byNode[pk] = append(store.byNode[pk], tx)
+	}
+	for _, query := range []PacketQuery{{Hash: "0000000000000002"}, {Observer: "test-observer"}, {Node: pk}} {
+		query.ExcludeTypes = 1 << 11
+		got := store.filterPackets(query)
+		want := 1
+		if query.Hash != "" {
+			want = 0
+		}
+		if len(got) != want {
+			t.Errorf("index query %+v yielded %d rows want %d", query, len(got), want)
+		}
+	}
+}
+
+// Both cases include the complete 30K-row filter/sort/page path. Grouped
+// cache is deliberately invalidated per iteration to measure cold requests.
+func BenchmarkPacketExclusions30K(b *testing.B) {
+	store := NewPacketStore(nil, nil)
+	for i := 0; i < 30000; i++ {
+		typ := 5
+		if i%5 != 0 {
+			typ = 11
+		}
+		stamp := fmt.Sprintf("2026-01-01T%08d", i)
+		store.packets = append(store.packets, &StoreTx{ID: i + 1, Hash: fmt.Sprintf("%016x", i+1), FirstSeen: stamp, LatestSeen: stamp, PayloadType: &typ, PathJSON: "[]"})
+	}
+	for _, grouped := range []bool{false, true} {
+		for _, mask := range []packetTypeExclusions{0, 1 << 5, 1 << 11} {
+			b.Run(fmt.Sprintf("grouped=%t/exclusions=%d", grouped, mask), func(b *testing.B) {
+				q := PacketQuery{Limit: 50, ExcludeTypes: mask}
+				b.ReportAllocs()
+				for i := 0; i < b.N; i++ {
+					var result *PacketResult
+					if grouped {
+						store.groupedCacheTxs = nil
+						result = store.QueryGroupedPackets(q)
+					} else {
+						result = store.QueryPackets(q)
+					}
+					if len(result.Packets) != 50 {
+						b.Fatal("page size")
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestPacketExclusionsValidation(t *testing.T) {
+	srv, _ := setupTestServer(t)
+	defer srv.db.Close()
+	for _, query := range []string{"-1", "16", "CONTROL", "11,", "1,,2", "1.5", "%2B1", "1&excludeTypes=2", "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0", "11&nodes=abcd", strings.Repeat("1", 65)} {
+		w := httptest.NewRecorder()
+		srv.handlePackets(w, httptest.NewRequest("GET", "/api/packets?excludeTypes="+query, nil))
+		if w.Code != 400 {
+			t.Errorf("%s: got %d want400", query, w.Code)
+		}
+	}
+	for _, query := range []string{"", "0", "15", "11,11", "%2011%20,5", "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15"} {
+		w := httptest.NewRecorder()
+		srv.handlePackets(w, httptest.NewRequest("GET", "/api/packets?excludeTypes="+query, nil))
+		if w.Code != 200 {
+			t.Errorf("%s: got %d want200", query, w.Code)
+		}
+	}
+}

cmd/server/routes.go

@@ -1263,6 +1263,15 @@ func (s *Server) handlePerfReset(w http.ResponseWriter, r *http.Request) {
 // --- Packet Handlers ---
 
 func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
+	excludeTypes, parseErr := parsePacketTypeExclusions(r.URL.Query())
+	if parseErr != nil {
+		writeError(w, 400, parseErr.Error())
+		return
+	}
+	if excludeTypes != 0 && r.URL.Query().Get("nodes") != "" {
+		writeError(w, 400, "excludeTypes is not supported with nodes; use the single node filter")
+		return
+	}
 	// Multi-node filter: comma-separated pubkeys (Node.js parity)
 	if nodesParam := r.URL.Query().Get("nodes"); nodesParam != "" {
 		pubkeys := strings.Split(nodesParam, ",")
@@ -1300,6 +1309,7 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) {
 	}
 
 	q := PacketQuery{
+		ExcludeTypes:       excludeTypes,
 		Limit:              queryLimit(r, 50, s.cfg.ListLimits.PacketsMax),
 		Offset:             queryInt(r, "offset", 0),
 		Observer:           r.URL.Query().Get("observer"),

cmd/server/store.go

@@ -2347,6 +2347,9 @@ func (s *PacketStore) QueryGroupedPackets(q PacketQuery) *PacketResult {
 
 	// Cache key covers all filter dimensions. Empty key = no filters.
 	cacheKey := q.Since + "|" + q.Until + "|" + q.Region + "|" + q.Area + "|" + q.Node + "|" + q.Hash + "|" + q.Observer + "|" + q.Channel
+	if q.ExcludeTypes != 0 {
+		cacheKey += fmt.Sprintf("|x%d", q.ExcludeTypes)
+	}
 	if q.Type != nil {
 		cacheKey += fmt.Sprintf("|t%d", *q.Type)
 	}
@@ -3949,7 +3952,7 @@ func (s *PacketStore) MaxObservationID() int {
 // filterPackets applies PacketQuery filters to the in-memory packet list.
 func (s *PacketStore) filterPackets(q PacketQuery) []*StoreTx {
 	// Fast path: single-key index lookups
-	if q.Hash != "" && q.Type == nil && q.Route == nil && q.Observer == "" &&
+	if q.Hash != "" && q.Type == nil && q.ExcludeTypes == 0 && q.Route == nil && q.Observer == "" &&
 		q.Region == "" && q.Area == "" && q.Node == "" && q.Channel == "" && q.Since == "" && q.Until == "" {
 		h := strings.ToLower(q.Hash)
 		tx := s.byHash[h]
@@ -3958,7 +3961,7 @@ func (s *PacketStore) filterPackets(q PacketQuery) []*StoreTx {
 		}
 		return []*StoreTx{tx}
 	}
-	if q.Observer != "" && q.Type == nil && q.Route == nil &&
+	if q.Observer != "" && q.Type == nil && q.ExcludeTypes == 0 && q.Route == nil &&
 		q.Region == "" && q.Area == "" && q.Node == "" && q.Channel == "" && q.Hash == "" && q.Since == "" && q.Until == "" {
 		return s.transmissionsForObserver(q.Observer, nil)
 	}
@@ -4027,7 +4030,7 @@ func (s *PacketStore) filterPackets(q PacketQuery) []*StoreTx {
 	// Determine the source slice. Use index-based source when only node
 	// filter is active and an index exists.
 	source := s.packets
-	if hasNode && !hasType && !hasRoute && q.Observer == "" &&
+	if hasNode && !hasType && q.ExcludeTypes == 0 && !hasRoute && q.Observer == "" &&
 		filterHash == "" && !hasSince && !hasUntil && q.Region == "" && q.Area == "" && filterChannel == "" {
 		if indexed, ok := s.byNode[nodePK]; ok {
 			return indexed
@@ -4042,6 +4045,9 @@ func (s *PacketStore) filterPackets(q PacketQuery) []*StoreTx {
 		if hasType && (tx.PayloadType == nil || *tx.PayloadType != filterType) {
 			return false
 		}
+		if q.ExcludeTypes.excludes(tx.PayloadType) {
+			return false
+		}
 		if hasRoute && (tx.RouteType == nil || *tx.RouteType != filterRoute) {
 			return false
 		}

docs/api-spec.md

@@ -1125,7 +1125,8 @@ Paginated packet (transmission) list with filtering.
 |--------------|--------|---------|----------------------------------------------------|
 | `limit`      | number | `50`    | Page size                                          |
 | `offset`     | number | `0`     | Pagination offset                                  |
-| `type`       | string | —       | Filter by payload type (number or name)            |
+| `type`       | string | —       | Filter by numeric payload type                    |
+| `excludeTypes` | string | —     | Comma-separated numeric payload types (0–15), excluded before pagination |
 | `route`      | string | —       | Filter by route type                               |
 | `region`     | string | —       | Filter by region (IATA code substring)             |
 | `observer`   | string | —       | Filter by observer ID                              |
@@ -1138,6 +1139,24 @@ Paginated packet (transmission) list with filtering.
 | `groupByHash`| string | —       | Set to `"true"` for grouped response               |
 | `expand`     | string | —       | Set to `"observations"` to include observation arrays |
 
+`excludeTypes=11` excludes CONTROL transmissions before `limit`, `offset`, and
+`total` are calculated, for both raw and `groupByHash=true` responses. It uses
+the same filtering in memory and in the SQLite fallback. It does not delete
+packets or affect WebSocket delivery or packet-detail endpoints.
+
+The list accepts at most 16 entries and 64 characters. Whitespace around entries
+is trimmed; duplicates collapse. Empty/omitted means no exclusion. Codes 0–15
+include reserved wire types; unknown/NULL stored types remain in the result.
+An overlapping `type` inclusion and exclusion returns no packets for that type.
+Malformed values, repeated `excludeTypes` parameters, and a nonempty exclusion
+combined with `nodes` return HTTP 400. Use `node` for a supported single-node
+combination. All other existing filters continue to compose normally.
+
+The Packets page refetches when its type selection or Hide CONTROL checkbox
+changes, so excluded traffic cannot fill the fetched page. Live updates are
+still filtered locally. Pinned-hash views omit these exclusions so a direct
+packet link remains visible.
+
 ### Response `200` (default)
 
 ```jsonc