diff --git a/.changeset/native-netflow-monitoring.md b/.changeset/native-netflow-monitoring.md new file mode 100644 index 0000000000..79a86f89e0 --- /dev/null +++ b/.changeset/native-netflow-monitoring.md @@ -0,0 +1,58 @@ +--- +'@hyperdx/app': minor +'@hyperdx/api': minor +'@hyperdx/common-utils': minor +--- + +Add native NetFlow sources and monitoring with configurable column mappings, +Akvorado schema detection, sampling-aware traffic charts, top talkers, exporter +and interface breakdowns, and flow details. NetFlow tables also support Search +and custom charts. + +Support Lucene search with field autocomplete on the NetFlow page, combining +search queries with quick filters across charts and flow records. + +Add clickable include/exclude filters to flow IPs, protocols, exporters, and +interfaces, with removable selections preserved in the URL. + +Show table column suggestions when focusing an empty NetFlow search input. + +Size time-chart Y axes to their formatted labels so rate units do not wrap or +get clipped. + +Avoid sending API proxy headers when querying ClickHouse directly in local mode. + +Add a Sankey visualization with ordered table dimensions, sampling-adjusted path +weights, average bit rates, configurable path limits, and clickable shared +filters. Include interface classifications in the local demo data. + +Recover from invalid time ranges without crashing, clear source-specific click +filters when switching sources, and keep chart geometry aligned with automatic +axis widths. + +Preserve NetFlow mappings through MCP source tools, ignore stale source +inference, share filter keys across visualizations, retain quoted columns, and +make explicit include/exclude actions idempotent. Format flow times using user +preferences. + +Support NetFlow saved-search alerts with default aliases, previews, and flow +samples in notifications. Keep unfinished search fields pending when applying +click filters, and display the configured record limit. + +Search mapped flow dimensions with bare Lucene terms or a custom full-text +expression. Share the summary aggregate query, reuse accessible filter menus, +restore valid Sankey dimensions across source changes, and document source +mappings consistently in REST and MCP. + +Apply the same bare-term search defaults to dashboards, chart editing, and alert +details. Support timestamp expression lists in flow records and normalize +partial traffic buckets over their selected duration. + +Preserve saved source mappings when connections, databases, or metadata change. +Show removable range filters using the shared filter pills and apply configured +full-text mappings consistently in the external charts API. Mark NetFlow +navigation as beta. + +Clear applied and pending searches when switching NetFlow sources, treat invalid +sampling multipliers as one, guard stale Sankey render indices, and create cell +filter popovers only after interaction. diff --git a/docs/images/netflow-column-suggestions.png b/docs/images/netflow-column-suggestions.png new file mode 100644 index 0000000000..e63d5772e7 Binary files /dev/null and b/docs/images/netflow-column-suggestions.png differ diff --git a/docs/images/netflow-flow-details.png b/docs/images/netflow-flow-details.png new file mode 100644 index 0000000000..8596234cbf Binary files /dev/null and b/docs/images/netflow-flow-details.png differ diff --git a/docs/images/netflow-lucene-filters.png b/docs/images/netflow-lucene-filters.png new file mode 100644 index 0000000000..46226ff5dd Binary files /dev/null and b/docs/images/netflow-lucene-filters.png differ diff --git a/docs/images/netflow-overview.png b/docs/images/netflow-overview.png new file mode 100644 index 0000000000..07a6b4baae Binary files /dev/null and b/docs/images/netflow-overview.png differ diff --git a/docs/images/netflow-sankey.png b/docs/images/netflow-sankey.png new file mode 100644 index 0000000000..a1bf1c27a4 Binary files /dev/null and b/docs/images/netflow-sankey.png differ diff --git a/docs/images/netflow-source-mappings.png b/docs/images/netflow-source-mappings.png new file mode 100644 index 0000000000..591bf31e8c Binary files /dev/null and b/docs/images/netflow-source-mappings.png differ diff --git a/docs/netflow.md b/docs/netflow.md new file mode 100644 index 0000000000..43af8f746a --- /dev/null +++ b/docs/netflow.md @@ -0,0 +1,187 @@ +# NetFlow monitoring + +The NetFlow section queries a configured ClickHouse table through HyperDX's +existing connections and source mappings. An Akvorado `flows` table can be used +directly. HyperDX does not collect UDP NetFlow, IPFIX, or sFlow packets: keep +Akvorado or another collector responsible for decoding and ingestion. + +## Local demo + +Use Python 3, the repository's Node/Yarn dependencies, and a local ClickHouse +server with HTTP access and browser CORS enabled. The demo uses +`http://127.0.0.1:8123`, user `default`, and an empty password. Override these +with `CLICKHOUSE_URL`, `CLICKHOUSE_USER`, and `CLICKHOUSE_PASSWORD` as needed. +Local mode sends these credentials to the browser, so use a local development +account. + +From the repository root: + +```sh +yarn setup +yarn build:common-utils +python3 scripts/netflow-seed.py +python3 scripts/netflow-dev.py +``` + +The launcher prints the URL, normally `http://127.0.0.1:3000/netflow`, and uses +port 3001 if 3000 is occupied. It supplies the connection and native NetFlow +source through HyperDX's supported default configuration environment variables. +No MongoDB, API server, account registration, or browser storage edits are +needed for this local demo. A fresh browser session uses these defaults; +previously saved local connections and sources take precedence. + +Choose the last two hours to see the whole fixture, including its TCP traffic +burst. Re-run the seeder to move the data to the current time. The seeder +replaces only its own `netflow_demo.flows` table, checking its ownership comment +first. It refuses remote endpoints and does not modify other databases or +tables. Refreshes do not accumulate duplicate records. A failed insert can leave +this disposable fixture table empty; re-run the seeder to restore it. + +For a fixed, reproducible time range: + +```sh +python3 scripts/netflow-seed.py --end 2026-10-09T18:34:38Z +``` + +This produces the UTC interval `[2026-10-09 16:34:38, 2026-10-09 18:34:38)`. The +last stored timestamp is `18:34:37`; use that absolute range when inspecting +this fixed fixture after its timestamps leave the relative time window. + +## Data and sampling + +The data is deterministic synthetic traffic generated locally, not a packet +capture or customer traffic. The schema follows +[Akvorado's flow schema](https://github.com/akvorado/akvorado/blob/main/common/schema/definition.go), +using documentation IP ranges and private AS numbers. The fixture includes +14,400 records across three exporters, TCP, UDP, ICMP, ICMPv6, inbound and +outbound interfaces, IPv4-mapped IPv6 addresses, and native IPv6 addresses. +Interface classification columns (`InIfConnectivity`, `OutIfConnectivity`, +`InIfProvider`, `OutIfProvider`) contain deterministic transit, IX, and PNI +classifications with synthetic provider names tied to each exporter. As with +Akvorado's default classifier, internal interfaces leave these fields empty. The +seeder adds missing classification columns only after confirming ownership of +the demo table; existing counters and record counts stay unchanged. Interface +speeds are stored in Mbps, as in Akvorado. `ExporterAddress` uses plain `IPv6` +to work without ClickHouse's optional low-cardinality IPv6 setting. + +`Bytes` and `Packets` hold observed counters. `SamplingRate` holds the expansion +factor (1, 100, or 1,000). Traffic estimates are `sum(Bytes * SamplingRate)` and +`sum(Packets * SamplingRate)`, matching +[Akvorado's aggregation](https://github.com/akvorado/akvorado/blob/main/console/widgets.go). +Multiply estimated bytes by eight and divide by the requested duration in +seconds for bits per second. The record count is the number of stored records, +not an estimate of distinct connections or sampling-expanded flows. + +The seeder checks actual ClickHouse query results against generated totals: + +| Measurement | Expected value | +| ------------------- | ----------------: | +| Records | 14,400 | +| Exporters | 3 | +| Protocols | 4 | +| Native IPv6 records | 2,572 | +| Observed bytes | 7,572,836,640 | +| Observed packets | 9,162,160 | +| Estimated bytes | 2,776,744,315,920 | +| Estimated packets | 3,359,797,180 | + +## Source configuration + +The NetFlow search bar uses the same Lucene syntax, field autocomplete, query +history, and syntax reference as log Search. Focus the search bar to see table +columns; type a prefix to narrow them, then click a suggestion or use the arrow +keys and Enter/Tab. After a column and colon, matching values are suggested. +Press Enter or Run to apply the query to all charts and flow records. Queries +combine with the exporter, protocol, and address filters using AND; Clear +filters resets both the search and quick filters. The query and selected +language are saved in the URL. The shared language selector also supports SQL +WHERE expressions. Bare terms search the mapped addresses, ports, protocol, +exporter, and interfaces. Set the optional **Full-text search expression** in +the source editor to search a different expression. + +Click an IP address, protocol, exporter, or interface in a breakdown chart, flow +row, or flow details to **Include** or **Exclude** it. These actions apply +immediately while preserving the search query and time range. Selected values +appear as removable filters and survive URL sharing and reloads. Multiple +included values for the same field use OR; different fields and exclusions use +AND. Clear filters also removes these selections. + +Use your table's actual field names, for example with the Akvorado schema: + +```text +Proto:6 AND DstPort:443 +ExporterName:edge* AND Bytes:[1000 TO 100000] +(DstPort:80 OR DstPort:443) AND NOT Proto:17 +``` + +In the source editor, select NetFlow, a ClickHouse connection, and the desired +database/table. Use these Akvorado mappings: + +| Source setting | Column/expression | +| ---------------------------- | ------------------------ | +| Timestamp | `TimeReceived` | +| Bytes / packets | `Bytes` / `Packets` | +| Sampling rate | `SamplingRate` | +| Source / destination address | `SrcAddr` / `DstAddr` | +| Source / destination port | `SrcPort` / `DstPort` | +| Protocol | `Proto` | +| Exporter | `ExporterName` | +| Input / output interface | `InIfName` / `OutIfName` | + +For already sampling-adjusted counters, omit the sampling-rate mapping to avoid +expanding the counters twice. Full deployments persist the source through the +existing authenticated source API and MongoDB model. + +Missing, null, or non-positive sampling multipliers are treated as one. Positive +multipliers scale bytes and packets once across charts, records, and Sankey +paths. + +Traffic rates use the seconds covered by each bucket within the selected time +range, including partial first and last buckets. Flow records display and sort +by the first timestamp expression when multiple timestamp mappings are +configured. + +Switching sources clears the search and quick filters, including pending edits, +so predicates for one table do not carry into another table. + +## Sankey visualization + +Select **Sankey** in the NetFlow visualization selector to explore traffic +paths. Choose two to five dimensions in left-to-right order from the source +mappings or scalar table columns. Akvorado tables default to `SrcAS` → +`InIfConnectivity` → `InIfProvider` → exporter when those columns are available. + +Link widths represent sampling-adjusted bytes for the top 10, 20, or 50 paths. +The table and tooltips show transferred bytes and average bit rate over the +selected time range. Paths outside the limit are omitted from the diagram. Click +a node or table value to include or exclude it using the shared filters; Lucene +search, quick filters, and the time range also apply to this view. The +visualization, ordered dimensions, and path limit are saved in the URL. + +The path limit bounds returned paths, not the number of groups ClickHouse +aggregates. For high-cardinality dimensions or wide ranges, configure +`max_memory_usage` and `max_bytes_before_external_group_by` through the +connection's query settings. + +A local validation with two million flow records, five dimensions, and a 30-day +range completed in 0.71 seconds while reading 106 MB, with a 512 MiB memory cap +and a 64 MiB external aggregation threshold. This checks that configuration on +the local fixture; it is not a production performance guarantee. + +## Automated verification + +The NetFlow browser and ClickHouse query checks run in the existing Playwright +suite and are discovered by CI. Each test creates and tears down its own +ClickHouse database and source configuration; the demo seeder is not required. + +```sh +make dev-e2e FILE=netflow +``` + +The suite checks sampled overview totals and raw counters, every chart query, +IPv4/IPv6 filters, literal escaping, Lucene and SQL composition, Sankey traffic +conservation, empty classifications, source switching, pending query drafts, URL +reloads, invalid time ranges, partial-bucket rates, timestamp expression lists, +flow details, and chart axes at wide and narrow viewport sizes. Failures use the +normal Playwright screenshots, traces, and reports under +`packages/app/test-results/`. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 6c2fd20b0f..a56cfc41af 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -5739,6 +5739,142 @@ } } }, + "NetflowSource": { + "type": "object", + "description": "Network flow table with configurable SQL expressions. Traffic totals multiply bytes and packets by the optional sampling multiplier (defaults to one).", + "required": [ + "name", + "kind", + "connection", + "from", + "timestampValueExpression", + "defaultTableSelectExpression", + "bytesExpression", + "packetsExpression", + "srcAddrExpression", + "dstAddrExpression", + "srcPortExpression", + "dstPortExpression", + "protocolExpression" + ], + "properties": { + "id": { + "type": "string", + "readOnly": true, + "description": "Unique source ID generated by the server.", + "example": "507f1f77bcf86cd799439011" + }, + "name": { + "type": "string", + "description": "Display name for the source.", + "example": "Network flows" + }, + "section": { + "type": "string", + "maxLength": 256, + "description": "Optional source grouping label.", + "example": "Network" + }, + "disabled": { + "type": "boolean", + "description": "Hide this source from source selectors." + }, + "kind": { + "type": "string", + "enum": [ + "netflow" + ], + "description": "Source kind discriminator for network flow tables.", + "example": "netflow" + }, + "connection": { + "type": "string", + "description": "ClickHouse connection ID.", + "example": "507f1f77bcf86cd799439012" + }, + "from": { + "$ref": "#/components/schemas/SourceFrom", + "description": "Database and table containing the flow records." + }, + "querySettings": { + "type": "array", + "description": "ClickHouse query settings applied to this source.", + "items": { + "$ref": "#/components/schemas/QuerySetting" + } + }, + "timestampValueExpression": { + "type": "string", + "description": "Timestamp column or SQL expression used to filter flow records by time and group them into time buckets. Must produce a ClickHouse date or timestamp value.", + "example": "TimeReceived" + }, + "defaultTableSelectExpression": { + "type": "string", + "description": "Comma-separated SQL select list used when displaying flow records in search results and alert samples. May include SQL expressions and aliases.", + "example": "TimeReceived, SrcAddr, DstAddr, Bytes" + }, + "implicitColumnExpression": { + "type": "string", + "description": "Optional SQL expression searched by bare Lucene terms. When omitted or blank, searches mapped IP addresses, ports, protocol, exporter, and interfaces.", + "example": "SearchText" + }, + "bytesExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "Bytes" + }, + "packetsExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "Packets" + }, + "srcAddrExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "SrcAddr" + }, + "dstAddrExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "DstAddr" + }, + "srcPortExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "SrcPort" + }, + "dstPortExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "DstPort" + }, + "protocolExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "Proto" + }, + "samplingRateExpression": { + "type": "string", + "description": "Sampling multiplier applied to observed byte and packet counters. Omit or leave empty for unsampled or already scaled counters. Omitted mappings and null or non-positive row values use a multiplier of one; positive row values are preserved.", + "example": "SamplingRate" + }, + "exporterExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "ExporterName" + }, + "inIfExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "InIfName" + }, + "outIfExpression": { + "type": "string", + "description": "SQL expression for the mapped flow field.", + "example": "OutIfName" + } + } + }, "Source": { "oneOf": [ { @@ -5755,6 +5891,9 @@ }, { "$ref": "#/components/schemas/PromqlSource" + }, + { + "$ref": "#/components/schemas/NetflowSource" } ], "discriminator": { @@ -5764,7 +5903,8 @@ "trace": "#/components/schemas/TraceSource", "metric": "#/components/schemas/MetricSource", "session": "#/components/schemas/SessionSource", - "promql": "#/components/schemas/PromqlSource" + "promql": "#/components/schemas/PromqlSource", + "netflow": "#/components/schemas/NetflowSource" } } }, diff --git a/packages/api/src/controllers/__tests__/sources.int.test.ts b/packages/api/src/controllers/__tests__/sources.int.test.ts index 71a7d73466..296e850c55 100644 --- a/packages/api/src/controllers/__tests__/sources.int.test.ts +++ b/packages/api/src/controllers/__tests__/sources.int.test.ts @@ -1,6 +1,7 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; import mongoose from 'mongoose'; -import { getSource } from '@/controllers/sources'; +import { createSource, getSource, updateSource } from '@/controllers/sources'; import { clearDBCollections, closeDB, connectDB } from '@/fixtures'; describe('sources controller', () => { @@ -16,6 +17,45 @@ describe('sources controller', () => { await closeDB(); }); + it('creates and updates native NetFlow mappings without losing fields', async () => { + const team = new mongoose.Types.ObjectId().toString(); + const input = { + team, + kind: SourceKind.Netflow as const, + name: 'Flows', + connection: new mongoose.Types.ObjectId().toString(), + from: { databaseName: 'default', tableName: 'flows' }, + timestampValueExpression: 'TimeReceived', + defaultTableSelectExpression: 'SrcAddr, DstAddr', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + samplingRateExpression: 'SamplingRate', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + exporterExpression: 'ExporterName', + inIfExpression: 'InIfName', + outIfExpression: 'OutIfName', + }; + const created = await createSource(team, input); + expect(created).toBeTruthy(); + const updated = await updateSource(team, String(created!._id), { + ...input, + bytesExpression: 'octets', + }); + expect(updated?.get('bytesExpression')).toBe('octets'); + expect(updated?.get('samplingRateExpression')).toBe('SamplingRate'); + expect(updated?.get('outIfExpression')).toBe('OutIfName'); + expect( + await getSource( + new mongoose.Types.ObjectId().toString(), + String(created!._id), + ), + ).toBeNull(); + }); + describe('getSource', () => { it('returns null when sourceId is not a valid ObjectId', async () => { // Non-ObjectId strings used to bubble a Mongoose CastError up diff --git a/packages/api/src/controllers/sources.ts b/packages/api/src/controllers/sources.ts index ad3670cd27..dd9ee0918f 100644 --- a/packages/api/src/controllers/sources.ts +++ b/packages/api/src/controllers/sources.ts @@ -5,6 +5,7 @@ import { ISourceInput, LogSource, MetricSource, + NetflowSource, PromqlSource, SessionSource, Source, @@ -16,6 +17,8 @@ import { // recognises kind-specific fields (e.g. metricTables on MetricSource). function getModelForKind(kind: SourceKind) { switch (kind) { + case SourceKind.Netflow: + return NetflowSource; case SourceKind.Log: return LogSource; case SourceKind.Trace: diff --git a/packages/api/src/mcp/__tests__/netflow-search.int.test.ts b/packages/api/src/mcp/__tests__/netflow-search.int.test.ts new file mode 100644 index 0000000000..f6c57ac4bd --- /dev/null +++ b/packages/api/src/mcp/__tests__/netflow-search.int.test.ts @@ -0,0 +1,197 @@ +import { createNativeClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +import * as config from '@/config'; +import { createTeam } from '@/controllers/team'; +import { getLoggedInAgent, getServer } from '@/fixtures'; +import { buildTile, runConfigTile } from '@/mcp/tools/query/helpers'; +import Connection from '@/models/connection'; +import { NetflowSource, Source } from '@/models/source'; +import { runSearchConfig } from '@/routers/external-api/v2/utils/search'; + +describe('NetFlow bare-term search through MCP and REST query paths', () => { + const server = getServer(); + const ch = createNativeClient({ + url: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const startDate = new Date('2026-01-01T00:00:00Z'); + const endDate = new Date('2026-01-01T01:00:00Z'); + + beforeAll(async () => { + await server.start(); + await ch.command({ + query: `CREATE TABLE default.netflow_search_test ( + timestamp DateTime('UTC'), client IPv6, server IPv6, router Nullable(String), + src_port UInt16, dst_port UInt16, protocol UInt8, bytes UInt64, packets UInt64, + search_text String + ) ENGINE = Memory`, + }); + await ch.insert({ + table: 'default.netflow_search_test', + format: 'JSONEachRow', + values: [ + { router: 'edge-west', bytes: 2000, search_text: 'custom-west' }, + { router: 'edge-east', bytes: 100, search_text: 'custom-east' }, + { router: null, bytes: 3000, search_text: '' }, + ].map(row => ({ + timestamp: '2026-01-01 00:30:00', + client: '2001:db8::1', + server: '2001:db8::2', + src_port: 443, + dst_port: 50000, + protocol: 6, + packets: 2, + ...row, + })), + }); + }); + afterEach(async () => { + await server.clearDBs(); + }); + afterAll(async () => { + await ch.command({ + query: 'DROP TABLE IF EXISTS default.netflow_search_test', + }); + await ch.close(); + await server.stop(); + }); + + it.each([SourceKind.Log, SourceKind.Trace, SourceKind.Netflow])( + 'REST chart series resolves bare Lucene terms for %s', + async kind => { + const { agent, team, user } = await getLoggedInAgent(server); + const connection = await Connection.create({ + team: team._id, + name: 'Chart ClickHouse', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const source = await Source.create({ + kind, + team: team._id, + connection: connection.id, + name: 'Chart source', + from: { databaseName: 'default', tableName: 'netflow_search_test' }, + timestampValueExpression: 'timestamp', + defaultTableSelectExpression: 'router, bytes', + implicitColumnExpression: + kind === SourceKind.Netflow ? undefined : 'router', + srcAddrExpression: 'client', + dstAddrExpression: 'server', + srcPortExpression: 'src_port', + dstPortExpression: 'dst_port', + protocolExpression: 'protocol', + exporterExpression: 'router', + bytesExpression: 'bytes', + packetsExpression: 'packets', + }); + const response = await agent + .post('/api/v2/charts/series') + .set('Authorization', `Bearer ${user.accessKey}`) + .send({ + startTime: startDate.getTime(), + endTime: endDate.getTime(), + series: [ + { + sourceId: source.id, + dataSource: 'events', + aggFn: 'count', + where: 'west AND bytes:[1000 TO *]', + whereLanguage: 'lucene', + groupBy: [], + }, + ], + }) + .expect(200); + expect(response.body.data).toHaveLength(1); + expect(Number(response.body.data[0]['series_0.data'])).toBe(1); + }, + ); + + it.each([ + { + name: 'fallback exporter mapping', + implicit: undefined, + where: 'west AND bytes:[1000 TO *]', + expected: ['edge-west'], + }, + { + name: 'fallback with nullable exporter', + implicit: undefined, + where: 'db8 AND bytes:[1000 TO *]', + expected: ['edge-west', null], + }, + { + name: 'custom expression', + implicit: 'search_text', + where: 'custom-west AND bytes:[1000 TO *]', + expected: ['edge-west'], + }, + ])('supports $name', async ({ implicit, where, expected }) => { + const team = await createTeam({ name: 'NetFlow search' }); + const connection = await Connection.create({ + team: team._id, + name: 'Test ClickHouse', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const source = await NetflowSource.create({ + kind: SourceKind.Netflow, + team: team._id, + connection: connection.id, + name: 'Custom flows', + from: { databaseName: 'default', tableName: 'netflow_search_test' }, + timestampValueExpression: 'timestamp', + defaultTableSelectExpression: 'router, bytes', + srcAddrExpression: 'client', + dstAddrExpression: 'server', + srcPortExpression: 'src_port', + dstPortExpression: 'dst_port', + protocolExpression: 'protocol', + bytesExpression: 'bytes', + packetsExpression: 'packets', + exporterExpression: 'router', + implicitColumnExpression: implicit, + }); + const query = { + displayType: 'search' as const, + select: '', + sourceId: source.id, + where, + whereLanguage: 'lucene' as const, + }; + const rest = await runSearchConfig({ + teamId: team.id, + config: query, + startDate, + endDate, + maxResults: 10, + offset: 0, + }); + expect(rest.isError).toBe(false); + if (rest.isError) throw new Error(rest.message); + expect(rest.data.map(row => row.router)).toEqual( + expect.arrayContaining(expected), + ); + expect(rest.data).toHaveLength(expected.length); + + const mcp = await runConfigTile( + team.id, + buildTile('Flows', 24, 6, query), + startDate, + endDate, + ); + expect('isError' in mcp && mcp.isError).toBeFalsy(); + const text = mcp.content.find(item => item.type === 'text'); + if (!text || text.type !== 'text') throw new Error('Missing MCP result'); + const rows = JSON.parse(text.text).result.data; + expect(rows).toHaveLength(expected.length); + expect(rows.map((row: { router: string | null }) => row.router)).toEqual( + expect.arrayContaining(expected), + ); + }); +}); diff --git a/packages/api/src/mcp/__tests__/sources.int.test.ts b/packages/api/src/mcp/__tests__/sources.int.test.ts index 59367443b3..ad32c7360b 100644 --- a/packages/api/src/mcp/__tests__/sources.int.test.ts +++ b/packages/api/src/mcp/__tests__/sources.int.test.ts @@ -1375,6 +1375,96 @@ describe('MCP Source Tools', () => { // ── clickstack_save_source ───────────────────────────────────────────────── describe('clickstack_save_source (create)', () => { + it('round-trips every NetFlow mapping through create, describe, update, and clone', async () => { + const mappings = { + implicitColumnExpression: 'SearchText', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + samplingRateExpression: 'SamplingRate', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + exporterExpression: 'ExporterName', + inIfExpression: 'InIfName', + outIfExpression: 'OutIfName', + }; + const created = await callTool(client, 'clickstack_save_source', { + kind: 'netflow', + name: 'Network traffic', + connection: connection._id.toString(), + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_LOGS_TABLE, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp, Body', + ...mappings, + }); + expect(created.isError).toBeFalsy(); + const saved = JSON.parse(getFirstText(created)); + expect(saved).toMatchObject({ kind: SourceKind.Netflow, ...mappings }); + const stored = await Source.findById(saved.id); + expect(stored?.team.toString()).toBe(team._id.toString()); + expect(stored?.toObject()).toMatchObject(mappings); + + const described = await callTool(client, 'clickstack_describe_source', { + sourceId: saved.id, + }); + expect(described.isError).toBeFalsy(); + const describedSource = JSON.parse(getFirstText(described)).source; + expect(describedSource.keyColumns).toMatchObject({ + bytes: 'Bytes', + packets: 'Packets', + samplingRate: 'SamplingRate', + srcAddr: 'SrcAddr', + dstAddr: 'DstAddr', + srcPort: 'SrcPort', + dstPort: 'DstPort', + protocol: 'Proto', + exporter: 'ExporterName', + inIf: 'InIfName', + outIf: 'OutIfName', + }); + const sourceConfig = describedSource.config; + expect(sourceConfig).toMatchObject({ id: saved.id, ...mappings }); + const updatedMappings = Object.fromEntries( + Object.entries(mappings).map(([key, value]) => [key, `(${value})`]), + ); + const updated = await callTool(client, 'clickstack_save_source', { + ...sourceConfig, + name: 'Updated network traffic', + ...updatedMappings, + }); + expect(updated.isError).toBeFalsy(); + expect(JSON.parse(getFirstText(updated))).toMatchObject(updatedMappings); + expect((await Source.findById(saved.id))?.toObject()).toMatchObject( + updatedMappings, + ); + + const describedUpdate = await callTool( + client, + 'clickstack_describe_source', + { + sourceId: saved.id, + }, + ); + expect(describedUpdate.isError).toBeFalsy(); + const updatedConfig = JSON.parse(getFirstText(describedUpdate)).source + .config; + expect(updatedConfig).toMatchObject(updatedMappings); + const { id: _id, ...cloneConfig } = updatedConfig; + const cloned = await callTool(client, 'clickstack_save_source', { + ...cloneConfig, + name: 'Cloned network traffic', + }); + expect(cloned.isError).toBeFalsy(); + const clone = JSON.parse(getFirstText(cloned)); + expect(clone.id).not.toBe(saved.id); + expect((await Source.findById(clone.id))?.toObject()).toMatchObject( + updatedMappings, + ); + }); + it('creates a log source scoped to the team', async () => { const result = await callTool(client, 'clickstack_save_source', { kind: 'log', diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index ec94c8f950..04d97c5227 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -4,6 +4,7 @@ import { } from '@clickhouse/client-common'; import { getHeatmapMode } from '@hyperdx/common-utils/dist/core/heatmap'; import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { getSourceImplicitColumnExpression } from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { convertToCategoricalChartConfig, getFirstTimestampValueExpression, @@ -578,10 +579,7 @@ export async function runConfigTile( 'defaultTableSelectExpression' in source ? source.defaultTableSelectExpression : undefined; - const implicitColumn = - 'implicitColumnExpression' in source - ? source.implicitColumnExpression - : undefined; + const implicitColumn = getSourceImplicitColumnExpression(source); const useTextIndexForImplicitColumn = 'useTextIndexForImplicitColumn' in source ? source.useTextIndexForImplicitColumn @@ -723,10 +721,7 @@ export async function runConfigTile( if (source) { sourceFields = { from: source.from, - implicitColumnExpression: - 'implicitColumnExpression' in source - ? source.implicitColumnExpression - : undefined, + implicitColumnExpression: getSourceImplicitColumnExpression(source), useTextIndexForImplicitColumn: 'useTextIndexForImplicitColumn' in source ? source.useTextIndexForImplicitColumn diff --git a/packages/api/src/mcp/tools/sources/__tests__/schemas.test.ts b/packages/api/src/mcp/tools/sources/__tests__/schemas.test.ts new file mode 100644 index 0000000000..f863ca7d3c --- /dev/null +++ b/packages/api/src/mcp/tools/sources/__tests__/schemas.test.ts @@ -0,0 +1,89 @@ +import { SourceKind, SourceSchemaNoId } from '@hyperdx/common-utils/dist/types'; + +import { + buildMcpSaveSourceSchema, + buildSourceInput, + extractSourceConfig, +} from '@/mcp/tools/sources/schemas'; + +const mcpSaveSourceSchema = buildMcpSaveSourceSchema(true); + +const mappings = { + implicitColumnExpression: 'SearchText', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + samplingRateExpression: 'SamplingRate', + exporterExpression: 'ExporterName', + inIfExpression: 'InIfName', + outIfExpression: 'OutIfName', +}; +const input = { + kind: 'netflow', + name: 'Network traffic', + connection: 'connection-id', + databaseName: 'akvorado', + tableName: 'flows', + timestampValueExpression: 'TimeReceived', + defaultTableSelectExpression: 'TimeReceived, SrcAddr, DstAddr, Bytes', + ...mappings, +}; + +describe('MCP source schema', () => { + it('exposes every canonical source kind to the MCP SDK', () => { + expect(new Set(mcpSaveSourceSchema.shape.kind.options)).toEqual( + new Set(Object.values(SourceKind)), + ); + }); + + it('keeps netflow when PromQL is disabled', () => { + expect(buildMcpSaveSourceSchema(false).shape.kind.options).toContain( + 'netflow', + ); + }); + + it('preserves all NetFlow mappings when assembling the canonical input', () => { + const assembled = buildSourceInput(mcpSaveSourceSchema.parse(input)); + expect(SourceSchemaNoId.parse(assembled)).toMatchObject({ + kind: SourceKind.Netflow, + ...mappings, + from: { databaseName: 'akvorado', tableName: 'flows' }, + }); + }); + + it('round-trips all NetFlow mappings through describe and update input', () => { + const stored = { + ...buildSourceInput(mcpSaveSourceSchema.parse(input)), + _id: 'source-id', + }; + const described = extractSourceConfig(stored); + expect(described).toEqual({ ...input, id: 'source-id' }); + const updated = mcpSaveSourceSchema.parse({ + ...described, + name: 'Renamed', + }); + expect(SourceSchemaNoId.parse(buildSourceInput(updated))).toMatchObject({ + name: 'Renamed', + ...mappings, + }); + }); + + it.each([ + 'bytesExpression', + 'packetsExpression', + 'srcAddrExpression', + 'dstAddrExpression', + 'srcPortExpression', + 'dstPortExpression', + 'protocolExpression', + ])('keeps canonical NetFlow validation for missing %s', field => { + const parsed = mcpSaveSourceSchema.parse({ ...input, [field]: undefined }); + expect(SourceSchemaNoId.safeParse(buildSourceInput(parsed)).success).toBe( + false, + ); + }); +}); diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index aeda0f3bd8..703e2d2a5f 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -157,6 +157,20 @@ async function describeSourceSchema( eventName: source.eventNameExpression, traceId: source.traceIdExpression, }; + } else if (source.kind === SourceKind.Netflow) { + meta.keyColumns = { + bytes: source.bytesExpression, + packets: source.packetsExpression, + samplingRate: source.samplingRateExpression, + srcAddr: source.srcAddrExpression, + dstAddr: source.dstAddrExpression, + srcPort: source.srcPortExpression, + dstPort: source.dstPortExpression, + protocol: source.protocolExpression, + exporter: source.exporterExpression, + inIf: source.inIfExpression, + outIf: source.outIfExpression, + }; } else if (source.kind === SourceKind.Metric) { // Filter out implementation-detail keys (e.g. a stray Mongoose `_id` // on the metricTables subdoc) so the agent only sees valid metric diff --git a/packages/api/src/mcp/tools/sources/saveSource.ts b/packages/api/src/mcp/tools/sources/saveSource.ts index ed90ed4ef0..73e1a9ad90 100644 --- a/packages/api/src/mcp/tools/sources/saveSource.ts +++ b/packages/api/src/mcp/tools/sources/saveSource.ts @@ -39,7 +39,10 @@ export function registerSaveSource({ 'defaultTableSelectExpression; trace also needs durationExpression, ' + 'traceIdExpression, spanIdExpression, parentSpanIdExpression, ' + 'spanNameExpression, spanKindExpression; session needs traceSourceId; ' + - 'metric needs metricTables and resourceAttributesExpression. Get ' + + 'metric needs metricTables and resourceAttributesExpression; ' + + 'netflow needs defaultTableSelectExpression, bytesExpression, ' + + 'packetsExpression, srcAddrExpression, dstAddrExpression, ' + + 'srcPortExpression, dstPortExpression, protocolExpression. Get ' + 'connection and source IDs from clickstack_list_sources.', inputSchema: buildMcpSaveSourceSchema(isPromqlEnabled(context)), }, diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index f2cc515173..f7b6b95e68 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -9,17 +9,25 @@ import { sanitizeMetricTables } from './metricKinds'; // // String literals (not SourceKind members) so z.enum narrows at the MCP SDK // boundary; the assertion keeps them in sync with the enum. -const SOURCE_KINDS = ['log', 'trace', 'session', 'metric', 'promql'] as const; -const _assertSourceKindsMatchEnum: readonly (typeof SOURCE_KINDS)[number][] = [ - SourceKind.Log, - SourceKind.Trace, - SourceKind.Session, - SourceKind.Metric, - SourceKind.Promql, -]; +const SOURCE_KINDS = [ + 'log', + 'trace', + 'session', + 'metric', + 'promql', + 'netflow', +] as const satisfies readonly `${SourceKind}`[]; +const _assertSourceKindsMatchEnum: readonly (typeof SOURCE_KINDS)[number][] = + Object.values(SourceKind); void _assertSourceKindsMatchEnum; -const NON_PROMQL_SOURCE_KINDS = ['log', 'trace', 'session', 'metric'] as const; +const NON_PROMQL_SOURCE_KINDS = [ + 'log', + 'trace', + 'session', + 'metric', + 'netflow', +] as const; const KIND_DESCRIPTION = 'Type of telemetry the table holds. Determines which other fields are ' + @@ -234,8 +242,9 @@ const mcpSaveSourceSchema = z.object({ .string() .optional() .describe( - 'Log and trace sources. Column searched by full-text Lucene terms that ' + - 'name no field, typically the log body (Body) or SpanName.', + 'Log, trace and netflow sources. Column searched by full-text Lucene ' + + 'terms that name no field, typically the log body (Body) or SpanName. ' + + 'NetFlow defaults to mapped flow dimensions when omitted.', ), knownColumnsListExpression: z .string() @@ -263,13 +272,14 @@ const mcpSaveSourceSchema = z.object({ 'Merge engine tables; "disabled" falls back to hasToken().', ), - // ── Log / Trace ── + // ── Log / Trace / NetFlow ── defaultTableSelectExpression: z .string() .optional() .describe( 'Comma-separated columns shown in search results by default, e.g. ' + - '"Timestamp, ServiceName, Body". Required for log and trace sources.', + '"Timestamp, ServiceName, Body". Required for log, trace, and netflow ' + + 'sources.', ), bodyExpression: z .string() @@ -296,6 +306,53 @@ const mcpSaveSourceSchema = z.object({ 'EventName. Omit if the table has no such column.', ), + bytesExpression: z + .string() + .optional() + .describe('Required for netflow: observed byte counter expression.'), + packetsExpression: z + .string() + .optional() + .describe('Required for netflow: observed packet counter expression.'), + srcAddrExpression: z + .string() + .optional() + .describe('Required for netflow: source IP address expression.'), + dstAddrExpression: z + .string() + .optional() + .describe('Required for netflow: destination IP address expression.'), + srcPortExpression: z + .string() + .optional() + .describe('Required for netflow: source port expression.'), + dstPortExpression: z + .string() + .optional() + .describe('Required for netflow: destination port expression.'), + protocolExpression: z + .string() + .optional() + .describe('Required for netflow: IP protocol number expression.'), + samplingRateExpression: z + .string() + .optional() + .describe( + 'NetFlow sampling multiplier. Omit for unsampled or already scaled counters. Omitted mappings and null or non-positive row values use a multiplier of one.', + ), + exporterExpression: z + .string() + .optional() + .describe('NetFlow sources. Exporter (router) name or address expression.'), + inIfExpression: z + .string() + .optional() + .describe('NetFlow sources. Input interface expression.'), + outIfExpression: z + .string() + .optional() + .describe('NetFlow sources. Output interface expression.'), + // ── Trace (required for trace kind) ── durationExpression: z .string() diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 19902a41a0..dbc3bf9ec4 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -3,6 +3,7 @@ import { LogSourceSchema, MetricsDataType, MetricSourceSchema, + NetflowSourceSchema, PromqlSourceSchema, QuerySettings, SessionSourceSchema, @@ -22,6 +23,10 @@ import { objectIdSchema } from '@/utils/zod'; // runtime value is intentionally never parsed, so no-unused-vars is a false positive here. // eslint-disable-next-line @typescript-eslint/no-unused-vars const ISourceSchema = z.discriminatedUnion('kind', [ + NetflowSourceSchema.omit({ connection: true }).extend({ + team: objectIdSchema, + connection: objectIdSchema.or(z.string()), + }), LogSourceSchema.omit({ connection: true }).extend({ team: objectIdSchema, connection: objectIdSchema.or(z.string()), @@ -309,3 +314,23 @@ export const PromqlSource = Source.discriminator( minAutoGranularity: minAutoGranularityField, }), ); + +type INetflowSource = Extract; +export const NetflowSource = Source.discriminator( + SourceKind.Netflow, + new Schema({ + defaultTableSelectExpression: String, + implicitColumnExpression: String, + bytesExpression: String, + packetsExpression: String, + samplingRateExpression: String, + srcAddrExpression: String, + dstAddrExpression: String, + srcPortExpression: String, + dstPortExpression: String, + protocolExpression: String, + exporterExpression: String, + inIfExpression: String, + outIfExpression: String, + }), +); diff --git a/packages/api/src/routers/external-api/__tests__/netflow-sources.int.test.ts b/packages/api/src/routers/external-api/__tests__/netflow-sources.int.test.ts new file mode 100644 index 0000000000..f34049febb --- /dev/null +++ b/packages/api/src/routers/external-api/__tests__/netflow-sources.int.test.ts @@ -0,0 +1,113 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { SuperAgentTest } from 'supertest'; + +import * as config from '@/config'; +import { getLoggedInAgent, getServer } from '@/fixtures'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; + +describe('External API NetFlow sources', () => { + const server = getServer(); + let agent: SuperAgentTest; + let accessKey: string; + let connectionId: string; + const path = '/api/v2/sources'; + const mappings = { + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + samplingRateExpression: 'SamplingRate', + exporterExpression: 'ExporterName', + inIfExpression: 'InIfName', + outIfExpression: 'OutIfName', + implicitColumnExpression: 'SearchText', + }; + const body = () => ({ + kind: SourceKind.Netflow, + name: 'Network flows', + connection: connectionId, + from: { databaseName: 'default', tableName: 'flows' }, + timestampValueExpression: 'TimeReceived', + defaultTableSelectExpression: 'TimeReceived, SrcAddr, DstAddr, Bytes', + ...mappings, + }); + const authRequest = (method: 'get' | 'post' | 'put', url: string) => + (method === 'get' + ? agent.get(url) + : method === 'post' + ? agent.post(url) + : agent.put(url) + ).set('Authorization', `Bearer ${accessKey}`); + + beforeAll(async () => { + await server.start(); + }); + beforeEach(async () => { + const session = await getLoggedInAgent(server); + agent = session.agent; + accessKey = session.user.accessKey; + const connection = await Connection.create({ + team: session.team._id, + name: 'ClickHouse', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + connectionId = connection.id; + }); + afterEach(async () => { + await server.clearDBs(); + }); + afterAll(async () => { + await server.stop(); + }); + + it('creates, reads, updates, and persists every mapping', async () => { + const created = await authRequest('post', path).send(body()).expect(200); + const id = created.body.data.id; + expect(created.body.data).toMatchObject(body()); + const read = await authRequest('get', `${path}/${id}`).expect(200); + expect(read.body.data).toMatchObject(body()); + const updated = { + ...body(), + name: 'Updated flows', + ...Object.fromEntries( + Object.entries(mappings).map(([key, value]) => [key, `(${value})`]), + ), + }; + await authRequest('put', `${path}/${id}`).send(updated).expect(200); + const readUpdate = await authRequest('get', `${path}/${id}`).expect(200); + expect(readUpdate.body.data).toMatchObject(updated); + expect(await Source.findById(id).lean()).toMatchObject({ + ...updated, + connection: expect.anything(), + }); + }); + + it.each([ + 'defaultTableSelectExpression', + 'bytesExpression', + 'packetsExpression', + 'srcAddrExpression', + 'dstAddrExpression', + 'srcPortExpression', + 'dstPortExpression', + 'protocolExpression', + ])( + 'rejects create and update missing %s without changing saved mappings', + async field => { + const invalid = { ...body(), [field]: undefined }; + await authRequest('post', path).send(invalid).expect(400); + expect(await Source.countDocuments({ kind: SourceKind.Netflow })).toBe(0); + const created = await authRequest('post', path).send(body()).expect(200); + const id = created.body.data.id; + await authRequest('put', `${path}/${id}`).send(invalid).expect(400); + const read = await authRequest('get', `${path}/${id}`).expect(200); + expect(read.body.data).toMatchObject(body()); + }, + ); +}); diff --git a/packages/api/src/routers/external-api/v2/charts.ts b/packages/api/src/routers/external-api/v2/charts.ts index 3dd5051cf3..17eb665278 100644 --- a/packages/api/src/routers/external-api/v2/charts.ts +++ b/packages/api/src/routers/external-api/v2/charts.ts @@ -1,4 +1,5 @@ import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { getSourceImplicitColumnExpression } from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { Granularity } from '@hyperdx/common-utils/dist/core/utils'; import { ChartConfigWithOptDateRange, @@ -298,6 +299,7 @@ const buildChartConfigFromRequest = async ( ], where: '', timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: getSourceImplicitColumnExpression(source), ...pickSampleWeightExpressionProps(source), dateRange: [new Date(params.startTime), new Date(params.endTime)] as [ Date, diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index df25307038..61dc0b03fa 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -93,6 +93,8 @@ function formatExternalSource(source: SourceDocument) { const json = JSON.stringify( (() => { switch (source.kind) { + case SourceKind.Netflow: + return source.toJSON({ getters: true }); case SourceKind.Log: return source.toJSON({ getters: true }); case SourceKind.Trace: @@ -830,6 +832,114 @@ function formatExternalSource(source: SourceDocument) { * type: string * description: Required by the API for all source kinds; not used by PromQL queries. * example: timestamp + * NetflowSource: + * type: object + * description: Network flow table with configurable SQL expressions. Traffic totals multiply bytes and packets by the optional sampling multiplier (defaults to one). + * required: + * - name + * - kind + * - connection + * - from + * - timestampValueExpression + * - defaultTableSelectExpression + * - bytesExpression + * - packetsExpression + * - srcAddrExpression + * - dstAddrExpression + * - srcPortExpression + * - dstPortExpression + * - protocolExpression + * properties: + * id: + * type: string + * readOnly: true + * description: Unique source ID generated by the server. + * example: 507f1f77bcf86cd799439011 + * name: + * type: string + * description: Display name for the source. + * example: Network flows + * section: + * type: string + * maxLength: 256 + * description: Optional source grouping label. + * example: Network + * disabled: + * type: boolean + * description: Hide this source from source selectors. + * kind: + * type: string + * enum: [netflow] + * description: Source kind discriminator for network flow tables. + * example: netflow + * connection: + * type: string + * description: ClickHouse connection ID. + * example: 507f1f77bcf86cd799439012 + * from: + * $ref: '#/components/schemas/SourceFrom' + * description: Database and table containing the flow records. + * querySettings: + * type: array + * description: ClickHouse query settings applied to this source. + * items: + * $ref: '#/components/schemas/QuerySetting' + * timestampValueExpression: + * type: string + * description: Timestamp column or SQL expression used to filter flow records by time and group them into time buckets. Must produce a ClickHouse date or timestamp value. + * example: TimeReceived + * defaultTableSelectExpression: + * type: string + * description: Comma-separated SQL select list used when displaying flow records in search results and alert samples. May include SQL expressions and aliases. + * example: TimeReceived, SrcAddr, DstAddr, Bytes + * implicitColumnExpression: + * type: string + * description: Optional SQL expression searched by bare Lucene terms. When omitted or blank, searches mapped IP addresses, ports, protocol, exporter, and interfaces. + * example: SearchText + * bytesExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: Bytes + * packetsExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: Packets + * srcAddrExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: SrcAddr + * dstAddrExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: DstAddr + * srcPortExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: SrcPort + * dstPortExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: DstPort + * protocolExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: Proto + * samplingRateExpression: + * type: string + * description: Sampling multiplier applied to observed byte and packet counters. Omit or leave empty for unsampled or already scaled counters. Omitted mappings and null or non-positive row values use a multiplier of one; positive row values are preserved. + * example: SamplingRate + * exporterExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: ExporterName + * inIfExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: InIfName + * outIfExpression: + * type: string + * description: SQL expression for the mapped flow field. + * example: OutIfName * Source: * oneOf: * - $ref: '#/components/schemas/LogSource' @@ -837,6 +947,7 @@ function formatExternalSource(source: SourceDocument) { * - $ref: '#/components/schemas/MetricSource' * - $ref: '#/components/schemas/SessionSource' * - $ref: '#/components/schemas/PromqlSource' + * - $ref: '#/components/schemas/NetflowSource' * discriminator: * propertyName: kind * mapping: @@ -845,6 +956,7 @@ function formatExternalSource(source: SourceDocument) { * metric: '#/components/schemas/MetricSource' * session: '#/components/schemas/SessionSource' * promql: '#/components/schemas/PromqlSource' + * netflow: '#/components/schemas/NetflowSource' * SourcesListResponse: * type: object * properties: diff --git a/packages/api/src/tasks/checkAlerts/__tests__/netflowAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/netflowAlerts.int.test.ts new file mode 100644 index 0000000000..62465f6520 --- /dev/null +++ b/packages/api/src/tasks/checkAlerts/__tests__/netflowAlerts.int.test.ts @@ -0,0 +1,292 @@ +import { + ClickhouseClient, + createNativeClient, +} from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { + AlertState, + AlertThresholdType, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; +import mongoose from 'mongoose'; + +import * as config from '@/config'; +import { createAlert } from '@/controllers/alerts'; +import { createTeam } from '@/controllers/team'; +import { getServer } from '@/fixtures'; +import Alert, { AlertSource } from '@/models/alert'; +import AlertHistory from '@/models/alertHistory'; +import Connection from '@/models/connection'; +import { SavedSearch } from '@/models/savedSearch'; +import { NetflowSource } from '@/models/source'; +import Webhook from '@/models/webhook'; +import { processAlert } from '@/tasks/checkAlerts'; +import { AlertTaskType, loadProvider } from '@/tasks/checkAlerts/providers'; +import { + fetchSampleLines, + renderAlertTemplate, +} from '@/tasks/checkAlerts/template'; +import * as slack from '@/utils/slack'; + +describe('NetFlow saved-search alerts', () => { + const server = getServer(); + const nativeClient = createNativeClient({ + url: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const clickhouseClient = new ClickhouseClient({ + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const table = 'default.netflow_alert_test'; + const startTime = new Date('2023-11-16T22:05:00Z'); + const endTime = new Date('2023-11-16T22:10:00Z'); + const select = 'SrcAddr, Bytes * SamplingRate AS traffic'; + + beforeAll(async () => { + await server.start(); + await nativeClient.command({ + query: `CREATE TABLE ${table} ( + TimeReceived DateTime('UTC'), SrcAddr IPv6, DstAddr IPv6, + SrcPort UInt16, DstPort UInt16, Proto UInt8, + Bytes UInt64, Packets UInt64, SamplingRate UInt32 + ) ENGINE = Memory`, + }); + }); + + afterEach(async () => { + await nativeClient.command({ query: `TRUNCATE TABLE ${table}` }); + await server.clearDBs(); + jest.clearAllMocks(); + jest.restoreAllMocks(); + }); + + afterAll(async () => { + await nativeClient.command({ query: `DROP TABLE IF EXISTS ${table}` }); + await nativeClient.close(); + await server.stop(); + }); + + it.each([ + { + name: 'default SELECT and Lucene', + savedSelect: undefined, + where: 'traffic:[1000 TO *]', + whereLanguage: 'lucene' as const, + }, + { + name: 'empty SELECT and SQL', + savedSelect: '', + where: 'traffic >= 1000', + whereLanguage: 'sql' as const, + }, + { + name: 'explicit SELECT and Lucene', + savedSelect: select, + where: 'traffic:[1000 TO *]', + whereLanguage: 'lucene' as const, + }, + { + name: 'bare Lucene terms with default mappings', + savedSelect: undefined, + where: 'db8 AND traffic:[1000 TO *]', + whereLanguage: 'lucene' as const, + }, + { + name: 'bare Lucene terms with custom implicit expression', + savedSelect: undefined, + where: 'customflow AND traffic:[1000 TO *]', + whereLanguage: 'lucene' as const, + implicitColumnExpression: "concat('customflow ', toString(SrcAddr))", + }, + { + name: 'no matching flows', + savedSelect: undefined, + where: 'traffic:[999999 TO *]', + whereLanguage: 'lucene' as const, + noMatch: true, + }, + { + name: 'grouped flow alerts', + savedSelect: undefined, + where: 'traffic:[1000 TO *]', + whereLanguage: 'lucene' as const, + groupBy: 'SrcAddr', + }, + ])( + 'evaluates and delivers $name with aliased samples', + async ({ + savedSelect, + where, + whereLanguage, + implicitColumnExpression, + noMatch, + groupBy, + }) => { + const postMessage = jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + const team = await createTeam({ name: 'NetFlow alerts' }); + const connection = await Connection.create({ + team: team._id, + name: 'Test ClickHouse', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + const source = await NetflowSource.create({ + kind: SourceKind.Netflow, + team: team._id, + connection: connection.id, + name: 'Flows', + from: { databaseName: 'default', tableName: 'netflow_alert_test' }, + timestampValueExpression: 'TimeReceived', + defaultTableSelectExpression: select, + implicitColumnExpression, + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + samplingRateExpression: 'SamplingRate', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + }); + const savedSearch = await SavedSearch.create({ + team: team._id, + name: 'Large TCP flows', + source: source.id, + select: savedSelect, + where, + whereLanguage, + filters: [{ type: 'sql', condition: 'Proto = 6' }], + orderBy: 'SrcAddr', + tags: [], + }); + const webhook = await Webhook.create({ + team: team._id, + name: 'Test webhook', + service: 'slack', + url: 'https://hooks.slack.com/services/netflow-test', + }); + const alert = await createAlert( + team._id, + { + name: 'Large TCP flow alert', + source: AlertSource.SAVED_SEARCH, + savedSearchId: savedSearch.id, + channel: { type: 'webhook', webhookId: webhook.id }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: groupBy ? 1 : 2, + groupBy, + }, + new mongoose.Types.ObjectId(), + ); + await nativeClient.insert({ + table, + format: 'JSONEachRow', + values: [ + { SrcAddr: '2001:db8::1', Bytes: 20, SamplingRate: 100 }, + { SrcAddr: '2001:db8::2', Bytes: 30, SamplingRate: 100 }, + { SrcAddr: '2001:db8::3', Bytes: 30, SamplingRate: 100, Proto: 17 }, + { SrcAddr: '2001:db8::4', Bytes: 1, SamplingRate: 100 }, + { + SrcAddr: '2001:db8::5', + Bytes: 30, + SamplingRate: 100, + TimeReceived: '2023-11-16 22:10:00', + }, + ].map(row => ({ + TimeReceived: '2023-11-16 22:05:00', + DstAddr: '2001:db8::ffff', + SrcPort: 443, + DstPort: 50000, + Proto: 6, + Packets: 1, + ...row, + })), + }); + const alertProvider = await loadProvider(); + const metadata = getMetadata(clickhouseClient); + const teamWebhooksById = new Map([[webhook.id, webhook]]); + const expectedSamples = noMatch + ? '' + : '"2001:db8::1",2000\n"2001:db8::2",3000\n'; + expect( + await fetchSampleLines({ + clickhouseClient, + metadata, + savedSearch, + source, + startTime, + endTime, + }), + ).toBe(expectedSamples); + + await processAlert( + new Date('2023-11-16T22:12:00Z'), + { + alert, + source, + savedSearch, + taskType: AlertTaskType.SAVED_SEARCH, + previousMap: new Map(), + }, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + + expect(await Alert.findById(alert.id).lean()).toMatchObject({ + state: noMatch ? AlertState.OK : AlertState.ALERT, + executionErrors: [], + }); + const histories = await AlertHistory.find({ alert: alert.id }).lean(); + expect(histories).toHaveLength(groupBy ? 2 : 1); + for (const history of histories) + expect(history).toMatchObject({ + state: noMatch ? AlertState.OK : AlertState.ALERT, + lastValues: [{ count: noMatch ? 0 : groupBy ? 1 : 2 }], + }); + expect(postMessage).toHaveBeenCalledTimes(noMatch ? 0 : groupBy ? 2 : 1); + if (noMatch) return; + expect(JSON.stringify(postMessage.mock.calls[0][1])).toContain( + `${groupBy ? 1 : 2} lines found`, + ); + expect(JSON.stringify(postMessage.mock.calls[0][1])).toContain( + '2001:db8::1', + ); + expect(JSON.stringify(postMessage.mock.calls[0][1])).not.toContain( + '2001:db8::3', + ); + + // Standalone rendering must also fetch samples without the evaluator's cache. + const rendered = await renderAlertTemplate({ + alertProvider, + clickhouseClient, + metadata, + state: AlertState.ALERT, + title: 'NetFlow alert', + teamId: team.id, + teamWebhooksById: new Map(), + view: { + alert: { ...alert.toObject(), channel: { type: null }, channels: [] }, + source, + savedSearch, + startTime, + endTime, + value: 2, + attributes: {}, + granularity: '5m', + isGroupedAlert: false, + }, + }); + expect(rendered.failures).toEqual([]); + expect(rendered.body).toContain(expectedSamples); + }, + ); +}); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 6381d90507..d4ac02a14f 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -21,6 +21,7 @@ import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartCo import { ALERT_COUNT_DEFAULT_SELECT, buildSearchChartConfig, + getSourceImplicitColumnExpression, } from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { aliasMapToWithClauses, @@ -167,7 +168,9 @@ export async function computeAliasWithClauses( ): Promise { const resolvedSelect = savedSearch.select || - ((source.kind === SourceKind.Log || source.kind === SourceKind.Trace) && + ((source.kind === SourceKind.Log || + source.kind === SourceKind.Trace || + source.kind === SourceKind.Netflow) && source.defaultTableSelectExpression) || ''; const config: BuilderChartConfigWithOptDateRange = { @@ -177,10 +180,7 @@ export async function computeAliasWithClauses( select: resolvedSelect, where: savedSearch.where, whereLanguage: savedSearch.whereLanguage, - implicitColumnExpression: - source.kind === SourceKind.Log || source.kind === SourceKind.Trace - ? source.implicitColumnExpression - : undefined, + implicitColumnExpression: getSourceImplicitColumnExpression(source), useTextIndexForImplicitColumn: source.kind === SourceKind.Log || source.kind === SourceKind.Trace ? source.useTextIndexForImplicitColumn @@ -734,13 +734,9 @@ const buildAlertChartConfigFromSavedConfig = ({ } if (displayTypeSupportsBuilderAlerts(savedConfig.displayType)) { - // Alerts can use Log, Trace, or Metric sources. - // implicitColumnExpression+useTextIndexForImplicitColumn exist on Log and Trace sources; - // metricTables exists on Metric sources. - const implicitColumnExpression = - source.kind === SourceKind.Log || source.kind === SourceKind.Trace - ? source.implicitColumnExpression - : undefined; + // NetFlow derives its implicit expression from mapped flow dimensions. + // Text-index options exist on Log and Trace; metricTables exists on Metric. + const implicitColumnExpression = getSourceImplicitColumnExpression(source); const useTextIndexForImplicitColumn = source.kind === SourceKind.Log || source.kind === SourceKind.Trace ? source.useTextIndexForImplicitColumn diff --git a/packages/api/src/tasks/checkAlerts/template.ts b/packages/api/src/tasks/checkAlerts/template.ts index 94e96d1dd3..8ad057e06c 100644 --- a/packages/api/src/tasks/checkAlerts/template.ts +++ b/packages/api/src/tasks/checkAlerts/template.ts @@ -19,8 +19,8 @@ import { ChartConfigWithOptDateRange, Filter, isRangeThresholdType, + isSearchableSource, SavedChartConfig, - SourceKind, zAlertChannelType, } from '@hyperdx/common-utils/dist/types'; import Handlebars, { HelperOptions } from 'handlebars'; @@ -908,10 +908,8 @@ ${targetTemplate}`; if (source == null) { throw new Error(`Source ID is ${alert.source} but source is null`); } - if (source.kind !== SourceKind.Log && source.kind !== SourceKind.Trace) { - throw new Error( - `Expecting SourceKind 'trace' or 'log', got ${source.kind}`, - ); + if (!isSearchableSource(source)) { + throw new Error(`Expecting a searchable source, got ${source.kind}`); } // TODO: show group + total count for group-by alerts // Pass query results through the view so Handlebars syntax in log lines diff --git a/packages/app/pages/netflow.tsx b/packages/app/pages/netflow.tsx new file mode 100644 index 0000000000..d1604178db --- /dev/null +++ b/packages/app/pages/netflow.tsx @@ -0,0 +1,2 @@ +import NetflowPage from '@/NetflowPage'; +export default NetflowPage; diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 629b1209b7..49039862c0 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -26,6 +26,7 @@ import { TableConnection, tcFromSource, } from '@hyperdx/common-utils/dist/core/metadata'; +import { getSourceImplicitColumnExpression } from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { convertToDashboardTemplate, displayTypeSupportsBuilderAlerts, @@ -731,7 +732,6 @@ const Tile = ({ ...chart.config, // Populate these columns from the source to support Lucene-based filters and metric table macros ...pick(source, [ - 'implicitColumnExpression', 'useTextIndexForImplicitColumn', 'from', 'metricTables', @@ -739,6 +739,7 @@ const Tile = ({ ...(isLogSource(source) ? { bodyExpression: source.bodyExpression } : {}), + implicitColumnExpression: getSourceImplicitColumnExpression(source), sampleWeightExpression: getSampleWeightExpression(source), dateRange, granularity, @@ -771,10 +772,7 @@ const Tile = ({ databaseName: source.from?.databaseName || 'default', tableName: tableName || '', }, - implicitColumnExpression: - isLogSource(source) || isTraceSource(source) - ? source.implicitColumnExpression - : undefined, + implicitColumnExpression: getSourceImplicitColumnExpression(source), useTextIndexForImplicitColumn: isLogSource(source) || isTraceSource(source) ? source.useTextIndexForImplicitColumn diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 4466eadd8c..0242f29a6f 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -194,7 +194,11 @@ const LIVE_TAIL_REFRESH_FREQUENCY_OPTIONS = [ ]; const DEFAULT_REFRESH_FREQUENCY = 10000; -const ALLOWED_SOURCE_KINDS = [SourceKind.Log, SourceKind.Trace]; +const ALLOWED_SOURCE_KINDS = [ + SourceKind.Log, + SourceKind.Trace, + SourceKind.Netflow, +] satisfies SourceKind[]; const SearchConfigSchema = z.object({ select: z.string(), source: z.string(), @@ -236,7 +240,7 @@ export function getDefaultSourceId( // Restrict to sources that this page can actually display. const searchableSources = sources.filter( - s => ALLOWED_SOURCE_KINDS.includes(s.kind) && !s.disabled, + s => ALLOWED_SOURCE_KINDS.some(kind => kind === s.kind) && !s.disabled, ); if (searchableSources.length === 0) return ''; @@ -519,7 +523,7 @@ function SaveSearchModalComponent({ const { data: sourceObj } = useSource({ id: searchedConfig.source, - kinds: [SourceKind.Log, SourceKind.Trace], + kinds: ALLOWED_SOURCE_KINDS, }); const effectiveSelect = searchedConfig.select || sourceObj?.defaultTableSelectExpression || ''; @@ -800,7 +804,7 @@ function useSearchedConfigToChartConfig( ) { const { data: sourceObj, isLoading } = useSource({ id: source, - kinds: [SourceKind.Log, SourceKind.Trace], + kinds: ALLOWED_SOURCE_KINDS, }); const defaultOrderBy = useDefaultOrderBy(source); @@ -876,7 +880,7 @@ function optimizeDefaultOrderBy( export function useDefaultOrderBy(sourceID: string | undefined | null) { const { data: source } = useSource({ id: sourceID, - kinds: [SourceKind.Log, SourceKind.Trace], + kinds: ALLOWED_SOURCE_KINDS, }); const { data: tableMetadata } = useTableMetadata(tcFromSource(source)); @@ -884,11 +888,16 @@ export function useDefaultOrderBy(sourceID: string | undefined | null) { return useMemo(() => { // If no source, return undefined so that the orderBy is not set incorrectly if (!source) return undefined; - const trimmedOrderBy = source.orderByExpression?.trim(); + const trimmedOrderBy = + 'orderByExpression' in source + ? source.orderByExpression?.trim() + : undefined; if (trimmedOrderBy) return trimmedOrderBy; return optimizeDefaultOrderBy( source?.timestampValueExpression ?? '', - source.displayedTimestampValueExpression, + 'displayedTimestampValueExpression' in source + ? source.displayedTimestampValueExpression + : undefined, tableMetadata?.sorting_key, ); }, [source, tableMetadata]); @@ -1064,7 +1073,7 @@ function DBSearchPageContent() { const { source: searchedSource } = useResolvedSourceParam( rawSearchedConfig.source, { - kinds: [SourceKind.Log, SourceKind.Trace], + kinds: ALLOWED_SOURCE_KINDS, }, ); @@ -1227,7 +1236,8 @@ function DBSearchPageContent() { select: _savedSearch?.select ?? (searchedSource?.kind === SourceKind.Log || - searchedSource?.kind === SourceKind.Trace + searchedSource?.kind === SourceKind.Trace || + searchedSource?.kind === SourceKind.Netflow ? searchedSource.defaultTableSelectExpression : undefined), where: _savedSearch?.where ?? '', diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx index 3de997cf70..61a2113ae7 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -23,6 +23,7 @@ import { ReferenceLine, ResponsiveContainer, Tooltip, + usePlotArea, XAxis, YAxis, } from 'recharts'; @@ -95,8 +96,6 @@ const NEAREST_SERIES_MAX_DISTANCE_PX = 30; // tooltip's Popover `offset` so both land in the same spot. const TOOLTIP_POINT_OFFSET_PX = 12; -const Y_AXIS_WIDTH = 40; -const SINGLE_POINT_BAR_RIGHT_PADDING = 10; const SINGLE_POINT_BAR_WIDTH_RATIO = 0.8; // Top margin (px) reserved above the plot for annotation labels ("Alert"/"OK"), // added only when a chart is showing annotations so other charts keep their @@ -785,9 +784,8 @@ const MAX_AXIS_MANTISSA = 2; /** * Base width ceiling for a bare signed number - see MAX_AXIS_MANTISSA's - * comment. A character budget rather than a pixel one, which is why - * Y_AXIS_WIDTH scales with the content font size preference: the gutter has to - * grow with the tick font for this to keep holding at every setting. + * comment. A character budget rather than a pixel one, so it holds at every + * content font size: the auto-sized y-axis gutter grows with the tick font. */ const AXIS_CHAR_BUDGET = 5; @@ -1303,6 +1301,12 @@ export function computeYAxisBounds( }; } +function ReportPlotWidth({ onChange }: { onChange: (width: number) => void }) { + const width = usePlotArea()?.width ?? 0; + useEffect(() => onChange(width), [width, onChange]); + return null; +} + export const MemoChart = memo(function MemoChart({ graphResults, setIsClickActive, @@ -1380,11 +1384,7 @@ export const MemoChart = memo(function MemoChart({ const _id = useId(); const id = _id.replace(/:/g, ''); - // The y-axis width budget is expressed in characters (see MAX_AXIS_MANTISSA), - // so it has to grow with the tick font or the widest labels clip. - const { compact: axisTickFontSize, scale: contentFontScale } = - useContentFontSize(); - const yAxisWidth = Math.round(Y_AXIS_WIDTH * contentFontScale); + const { compact: axisTickFontSize } = useContentFontSize(); // recharts sync group, scoped via context (see chartSync). const syncId = useChartSyncId(); @@ -1532,7 +1532,7 @@ export const MemoChart = memo(function MemoChart({ ); const yAxisDomain = yAxisBounds.domain; - const [containerWidth, setContainerWidth] = useState(0); + const [plotWidth, setPlotWidth] = useState(0); // The chart's outer positioned container. Used to convert a pointer's // viewport clientX into a stable container-relative X for measuring @@ -1622,16 +1622,9 @@ export const MemoChart = memo(function MemoChart({ const singlePointBarSize = useMemo(() => { if (displayType !== DisplayType.StackedBar) return undefined; if (graphResults.length !== 1) return undefined; - const drawableWidth = Math.max( - 0, - containerWidth - yAxisWidth - SINGLE_POINT_BAR_RIGHT_PADDING, - ); - if (drawableWidth <= 0) return undefined; - return Math.max( - 1, - Math.floor(drawableWidth * SINGLE_POINT_BAR_WIDTH_RATIO), - ); - }, [displayType, graphResults.length, containerWidth, yAxisWidth]); + if (plotWidth <= 0) return undefined; + return Math.max(1, Math.floor(plotWidth * SINGLE_POINT_BAR_WIDTH_RATIO)); + }, [displayType, graphResults.length, plotWidth]); const formatTime = useFormatTime(); const xTickFormatter = useCallback( @@ -1971,9 +1964,9 @@ export const MemoChart = memo(function MemoChart({ } return layoutAnnotations(coloredAnnotations, { domain: xAxisDomain, - plotWidth: Math.max(0, containerWidth - yAxisWidth), + plotWidth, }); - }, [coloredAnnotations, xAxisDomain, containerWidth, yAxisWidth]); + }, [coloredAnnotations, xAxisDomain, plotWidth]); const [hoveredAnnotation, setHoveredAnnotation] = useState(null); @@ -1985,11 +1978,11 @@ export const MemoChart = memo(function MemoChart({ return getAnnotationElements(coloredAnnotations, { domain: xAxisDomain, // Drawable width, so markers too close together share one label. Zero on - // the first paint (before ResponsiveContainer measures), which the + // the first paint (before Recharts measures the plot), which the // renderer treats as "label everything". - plotWidth: Math.max(0, containerWidth - yAxisWidth), + plotWidth, }); - }, [coloredAnnotations, xAxisDomain, containerWidth, yAxisWidth]); + }, [coloredAnnotations, xAxisDomain, plotWidth]); return (
{ - const w = width ?? 1; - setContainerWidth(prev => (prev === w ? prev : w)); - }} className={isLoading ? 'effect-pulse' : ''} > + {/* Gradient defs cover every hex that any fill may reference. `COLORS` (the unified categorical palette) is included up-front @@ -2108,7 +2098,7 @@ export const MemoChart = memo(function MemoChart({ }} /> (null); + const netflowSources = sources?.filter( + (s): s is TNetflowSource => s.kind === SourceKind.Netflow && !s.disabled, + ); + const source = params.source + ? netflowSources?.find(s => [s.id, s.name].includes(params.source)) + : netflowSources?.[0]; + const { filters: appliedFilters, ...formParams } = params; + const { control, handleSubmit, reset } = useForm({ + values: { ...formParams, source: source?.id ?? '' }, + }); + const clearFilters = (id = source?.id ?? '') => { + const cleared = { + source: id, + where: '', + whereLanguage: params.whereLanguage, + exporter: '', + protocol: '', + srcAddr: '', + dstAddr: '', + }; + reset(cleared); + void setParams({ ...cleared, filters: [] }); + }; + const selectedSource = useWatch({ control, name: 'source' }); + const syncSource = useEffectEvent((id: string) => { + if (id && id !== params.source) { + if (id !== source?.id) clearFilters(id); + else void setParams({ source: id }); + } + }); + useEffect(() => { + syncSource(selectedSource); + }, [selectedSource]); + const clickFilters = useNetflowFilterState({ + source, + filters: appliedFilters, + onChange: filters => void setParams({ filters }), + }); + const defaultTimeRange = useDefaultTimeRange(DEFAULT_INTERVAL); + const [displayedTimeInputValue, setDisplayedTimeInputValue] = + useState(DEFAULT_INTERVAL); + const { searchedTimeRange, onSearch, onTimeRangeSelect } = useNewTimeQuery({ + initialDisplayValue: DEFAULT_INTERVAL, + initialTimeRange: defaultTimeRange, + setDisplayedTimeInputValue, + }); + const { refresh, manualRefreshCooloff } = useDashboardRefresh({ + searchedTimeRange, + onTimeRangeSelect, + isLive: false, + }); + const hasValidTimeRange = searchedTimeRange[1] > searchedTimeRange[0]; + const configs = useMemo( + () => + source && hasValidTimeRange + ? buildNetflowQueryConfigs({ + source, + dateRange: searchedTimeRange, + filters: { + exporter: params.exporter, + protocol: params.protocol, + srcAddr: params.srcAddr, + dstAddr: params.dstAddr, + }, + where: params.where, + whereLanguage: params.whereLanguage, + extraFilters: params.filters, + }) + : undefined, + [source, searchedTimeRange, params, hasValidTimeRange], + ); + const run = handleSubmit(values => { + void setParams(values); + onSearch(displayedTimeInputValue); + }); + + return ( + <> + + {title} + + setSourceModal(null)} + onCreate={created => { + // The form can save another kind; keep the current flows then. + if (created.kind === SourceKind.Netflow) clearFilters(created.id); + setSourceModal(null); + }} + /> +
+ setSourceModal('new')} + onEdit={() => setSourceModal('edit')} + placeholder="NetFlow source" + aria-label="NetFlow source" + /> + } + actions={ + + + + + + + + } + content={ + + {source && ( + + )} + + {( + [ + ['exporter', 'Exporter', 'All exporters'], + ['protocol', 'Protocol', '6 (TCP), 17 (UDP)'], + ['srcAddr', 'Source IP', 'IPv4 or IPv6 address'], + ['dstAddr', 'Destination IP', 'IPv4 or IPv6 address'], + ] as const + ).map(([name, label, placeholder]) => ( + + ))} + + + {configs && ( + + )} + + Explore traffic, top talkers, and recent flows. Rates average + over the selected time range; counters account for sampling. + + {error ? ( + + {error.message} + + ) : isLoading ? ( + + ) : !hasValidTimeRange ? ( + + + + ) : configs ? ( + + ) : ( + setSourceModal('new')} + /> + )} + + } + /> + + + ); +} + +const NetflowPageDynamic = dynamic(async () => NetflowPage, { ssr: false }); +// @ts-expect-error Next.js layout typing +NetflowPageDynamic.getLayout = withAppNavForSurface('dashboard', 'netflow'); +export default NetflowPageDynamic; diff --git a/packages/app/src/Spotlights.tsx b/packages/app/src/Spotlights.tsx index d73b5683e2..b29cd66090 100644 --- a/packages/app/src/Spotlights.tsx +++ b/packages/app/src/Spotlights.tsx @@ -90,6 +90,14 @@ export const useSpotlightActions = () => { }, ] : []), + { + id: 'preset-netflow', + label: 'NetFlow', + description: + 'Network traffic, top talkers, exporters, and flow records', + href: '/netflow', + keywords: ['netflow', 'network', 'akvorado', 'traffic', 'flows'], + }, ]; presetDashboards.forEach(preset => { diff --git a/packages/app/src/__tests__/HDXMultiSeriesTimeChart.geometry.test.tsx b/packages/app/src/__tests__/HDXMultiSeriesTimeChart.geometry.test.tsx new file mode 100644 index 0000000000..a9550a2e3f --- /dev/null +++ b/packages/app/src/__tests__/HDXMultiSeriesTimeChart.geometry.test.tsx @@ -0,0 +1,100 @@ +import { ComponentProps, ReactElement } from 'react'; +import { DisplayType } from '@hyperdx/common-utils/dist/types'; +import { waitFor } from '@testing-library/react'; + +import { + getAnnotationElements, + layoutAnnotations, +} from '@/components/charts/chartAnnotations'; +import { MemoChart } from '@/HDXMultiSeriesTimeChart'; + +let mockContainerWidth = 500; +jest.mock('recharts', () => { + const React = jest.requireActual('react'); + return { + ...jest.requireActual('recharts'), + // jsdom has no layout engine; only supply the container's measured size. + ResponsiveContainer: ({ + children, + }: { + children: ReactElement<{ width: number; height: number }>; + }) => + React.createElement(children.type, { + ...children.props, + width: mockContainerWidth, + height: 300, + }), + }; +}); +jest.mock('@/components/charts/chartAnnotations', () => ({ + ...jest.requireActual('@/components/charts/chartAnnotations'), + layoutAnnotations: jest.fn( + jest.requireActual('@/components/charts/chartAnnotations') + .layoutAnnotations, + ), + getAnnotationElements: jest.fn( + jest.requireActual('@/components/charts/chartAnnotations') + .getAnnotationElements, + ), +})); +jest.mock('@/useFormatTime', () => ({ useFormatTime: () => String })); + +const props = { + graphResults: [{ ts_bucket: 100, value: 1 }], + lineData: [ + { + dataKey: 'value', + currentPeriodKey: 'value', + previousPeriodKey: 'previous', + displayName: 'Traffic', + valueColumnName: 'value', + color: 'var(--color-chart-blue)', + groupValues: [], + }, + ], + dateRange: [new Date(0), new Date(200000)], + granularity: '1 minute', + displayType: DisplayType.StackedBar, + isClickActive: undefined, + setIsClickActive: jest.fn(), + tooltipNumberFormatsByKey: new Map(), + showLegend: false, + annotations: [{ time: 100000, label: 'Alert' }], +} satisfies ComponentProps; + +describe('time chart plot geometry', () => { + beforeEach(() => { + mockContainerWidth = 500; + jest.clearAllMocks(); + }); + + it('uses the real Recharts plot area for single bars and annotations after resize', async () => { + const { container, rerender } = renderWithMantine(); + const assertGeometry = async () => { + await waitFor(() => { + const plot = container.querySelector('clipPath rect'); + expect(plot).not.toBeNull(); + const width = Number(plot?.getAttribute('width')); + expect(width).toBeGreaterThan(0); + expect(width).toBeLessThan(mockContainerWidth); + const bar = container.querySelector('.recharts-bar-rectangle rect'); + expect(bar).not.toBeNull(); + expect(Number(bar?.getAttribute('width'))).toBe( + Math.floor(width * 0.8), + ); + expect(layoutAnnotations).toHaveBeenLastCalledWith( + expect.any(Array), + expect.objectContaining({ plotWidth: width }), + ); + expect(getAnnotationElements).toHaveBeenLastCalledWith( + expect.any(Array), + expect.objectContaining({ plotWidth: width }), + ); + }); + }; + await assertGeometry(); + mockContainerWidth = 350; + rerender(); + await assertGeometry(); + }); +}); diff --git a/packages/app/src/__tests__/NetflowPage.drafts.test.tsx b/packages/app/src/__tests__/NetflowPage.drafts.test.tsx new file mode 100644 index 0000000000..d0ea54933c --- /dev/null +++ b/packages/app/src/__tests__/NetflowPage.drafts.test.tsx @@ -0,0 +1,263 @@ +import { ComponentType, useEffect, useState } from 'react'; +import { + Filter, + SourceKind, + TNetflowSource, +} from '@hyperdx/common-utils/dist/types'; +import { act, fireEvent, screen, waitFor } from '@testing-library/react'; + +import { buildNetflowQueryConfigs } from '@/netflow'; +import NetflowPage from '@/NetflowPage'; + +const mockSource: TNetflowSource = { + id: 'flows', + name: 'Flows', + kind: SourceKind.Netflow, + connection: 'local', + from: { databaseName: 'default', tableName: 'flows' }, + timestampValueExpression: 'TimeReceived', + defaultTableSelectExpression: '*', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + exporterExpression: 'ExporterName', +}; +const mockOtherSource = { + ...mockSource, + id: 'other', + name: 'Other flows', + protocolExpression: 'IPProtocol', +}; +const mockRange = [ + new Date('2026-10-09T12:00:00Z'), + new Date('2026-10-09T13:00:00Z'), +]; +let mockInitialParams = { + source: 'flows', + where: 'Proto:6', + whereLanguage: 'lucene', + filters: [], + exporter: '', + protocol: '', + srcAddr: '', + dstAddr: '', +}; +let mockFilterChange: (filters: Filter[]) => void; + +jest.mock( + 'next/dynamic', + () => (loader: () => Promise) => () => { + const [Component, setComponent] = useState(null); + useEffect(() => { + void loader().then(component => setComponent(() => component)); + }, []); + return Component ? : null; + }, +); +jest.mock('nuqs', () => ({ + ...jest.requireActual('nuqs'), + useQueryStates: () => { + const [params, setParams] = useState(mockInitialParams); + return [ + params, + (next: Partial) => + setParams(previous => ({ ...previous, ...next })), + ]; + }, +})); +jest.mock('@/source', () => ({ + useSources: () => ({ data: [mockSource, mockOtherSource] }), +})); +jest.mock('@/layout', () => ({ + withAppNavForSurface: () => (page: unknown) => page, +})); +jest.mock('@/brand', () => ({ + ...jest.requireActual('@/brand'), + usePageTitle: () => 'NetFlow', +})); +jest.mock('@/netflow', () => ({ + buildNetflowQueryConfigs: jest.fn(() => ({})), +})); +jest.mock('@/components/NetflowCharts', () => () => null); +let mockCreateSource: (source: { id: string; kind: SourceKind }) => void; +jest.mock('@/components/NetflowSourceModal', () => ({ + __esModule: true, + default: ({ onCreate }: { onCreate: typeof mockCreateSource }) => { + mockCreateSource = onCreate; + return null; + }, +})); +jest.mock('@/components/SourceSelect', () => { + const Input = jest.requireActual( + '@/components/InputControlled', + ).TextInputControlled; + return { + SourceSelectControlled: ({ control }: { control: unknown }) => ( + + ), + }; +}); +jest.mock('@/components/TimePicker', () => ({ TimePicker: () => null })); +jest.mock('@/hooks/useDashboardRefresh', () => ({ + useDashboardRefresh: () => ({ refresh: jest.fn() }), +})); +jest.mock('@/timeQuery', () => ({ + useDefaultTimeRange: () => mockRange, + useNewTimeQuery: () => ({ + searchedTimeRange: mockRange, + onSearch: jest.fn(), + }), +})); +jest.mock('@/components/NetflowFilterPills', () => ({ + __esModule: true, + default: () => null, + useNetflowFilterState: ({ + onChange, + }: { + onChange: typeof mockFilterChange; + }) => { + mockFilterChange = onChange; + return {}; + }, +})); +jest.mock('@/components/SearchInput/SearchWhereInput', () => { + const Input = jest.requireActual( + '@/components/InputControlled', + ).TextInputControlled; + return ({ control }: { control: unknown }) => ( + + ); +}); + +describe('NetFlow draft search fields', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockInitialParams = { + source: 'flows', + where: 'Proto:6', + whereLanguage: 'lucene', + filters: [], + exporter: '', + protocol: '', + srcAddr: '', + dstAddr: '', + }; + }); + + it('clears applied search, quick filters and unsubmitted drafts when switching sources', async () => { + mockInitialParams = { + ...mockInitialParams, + exporter: 'edge', + protocol: '6', + srcAddr: '192.0.2.1', + dstAddr: '192.0.2.2', + }; + renderWithMantine(); + const query = await screen.findByLabelText('Query'); + act(() => + mockFilterChange([{ type: 'sql', condition: "ExporterName = 'edge'" }]), + ); + fireEvent.change(query, { target: { value: 'MissingOldColumn:443' } }); + fireEvent.change(screen.getByLabelText('Protocol'), { + target: { value: '17' }, + }); + fireEvent.change(screen.getByLabelText('Source'), { + target: { value: 'other' }, + }); + await waitFor(() => + expect( + jest.mocked(buildNetflowQueryConfigs).mock.lastCall?.[0], + ).toMatchObject({ + source: mockOtherSource, + where: '', + extraFilters: [], + filters: { exporter: '', protocol: '', srcAddr: '', dstAddr: '' }, + }), + ); + for (const label of [ + 'Query', + 'Exporter', + 'Protocol', + 'Source IP', + 'Destination IP', + ]) { + expect(screen.getByLabelText(label)).toHaveValue(''); + } + expect( + Object.keys( + jest.mocked(buildNetflowQueryConfigs).mock.lastCall?.[0].filters ?? {}, + ).sort(), + ).toEqual(['dstAddr', 'exporter', 'protocol', 'srcAddr']); + }); + + it('preserves applied filters while canonicalizing a source name to its ID', async () => { + mockInitialParams = { + ...mockInitialParams, + source: 'Flows', + protocol: '6', + }; + renderWithMantine(); + await screen.findByLabelText('Query'); + await waitFor(() => + expect(screen.getByLabelText('Source')).toHaveValue('flows'), + ); + expect( + jest.mocked(buildNetflowQueryConfigs).mock.lastCall?.[0], + ).toMatchObject({ + source: mockSource, + where: 'Proto:6', + filters: { protocol: '6' }, + }); + }); + + it('switches only to NetFlow sources created from the source modal', async () => { + renderWithMantine(); + await screen.findByLabelText('Query'); + act(() => mockCreateSource({ id: 'logs', kind: SourceKind.Log })); + expect(screen.getByLabelText('Source')).toHaveValue('flows'); + expect(screen.getByLabelText('Query')).toHaveValue('Proto:6'); + act(() => mockCreateSource({ id: 'other', kind: SourceKind.Netflow })); + await waitFor(() => + expect( + jest.mocked(buildNetflowQueryConfigs).mock.lastCall?.[0], + ).toMatchObject({ source: mockOtherSource, where: '' }), + ); + }); + + it('preserves drafts until Run when clicks or migrations update filters', async () => { + renderWithMantine(); + const query = await screen.findByLabelText('Query'); + const protocol = screen.getByLabelText('Protocol'); + fireEvent.change(query, { target: { value: 'DstPort:443' } }); + fireEvent.change(protocol, { target: { value: '17' } }); + const filters: Filter[] = [ + { type: 'sql', condition: "ExporterName = 'edge-a'" }, + ]; + act(() => mockFilterChange(filters)); + await waitFor(() => + expect( + jest.mocked(buildNetflowQueryConfigs).mock.lastCall?.[0], + ).toMatchObject({ + where: 'Proto:6', + filters: { protocol: '' }, + extraFilters: filters, + }), + ); + expect(query).toHaveValue('DstPort:443'); + expect(protocol).toHaveValue('17'); + fireEvent.click(screen.getByRole('button', { name: 'Run' })); + await waitFor(() => + expect( + jest.mocked(buildNetflowQueryConfigs).mock.lastCall?.[0], + ).toMatchObject({ + where: 'DstPort:443', + filters: { protocol: '17' }, + extraFilters: filters, + }), + ); + }); +}); diff --git a/packages/app/src/__tests__/netflow.test.ts b/packages/app/src/__tests__/netflow.test.ts new file mode 100644 index 0000000000..86f7db2415 --- /dev/null +++ b/packages/app/src/__tests__/netflow.test.ts @@ -0,0 +1,310 @@ +import { Granularity } from '@hyperdx/common-utils/dist/core/utils'; +import { + DisplayType, + SourceKind, + TNetflowSource, +} from '@hyperdx/common-utils/dist/types'; + +import { buildAlertChartConfig } from '@/components/alerts/AlertDetailChart'; +import { convertFormStateToChartConfig } from '@/components/ChartEditor/utils'; +import { buildReleaseChartConfig } from '@/hooks/useReleaseAnnotations'; +import { buildNetflowQueryConfigs, buildNetflowWhere } from '@/netflow'; + +const source: TNetflowSource = { + id: 'flows', + name: 'Network', + kind: SourceKind.Netflow, + connection: 'local', + from: { databaseName: 'netflow_demo', tableName: 'flows' }, + timestampValueExpression: 'TimeReceived', + defaultTableSelectExpression: '*', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + samplingRateExpression: 'SamplingRate', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + exporterExpression: 'ExporterName', + inIfExpression: 'InIfName', + outIfExpression: 'OutIfName', +}; +const dateRange: [Date, Date] = [ + new Date('2026-10-09T12:00:00Z'), + new Date('2026-10-09T13:00:00Z'), +]; + +describe('NetFlow queries', () => { + it.each([ + ['TimeReceived, FlowDate', 'TimeReceived'], + [ + "toDateTime(TimeReceived, 'UTC'), toDate(TimeReceived)", + "toDateTime(TimeReceived, 'UTC')", + ], + ])( + 'uses the first timestamp expression for flow rows (%s)', + (mapping, timestamp) => { + const configs = buildNetflowQueryConfigs({ + source: { ...source, timestampValueExpression: mapping }, + dateRange, + filters: {}, + }); + expect(configs.flows.select[0]).toEqual({ + valueExpression: timestamp, + alias: '__netflow_timestamp', + }); + expect(configs.flows.orderBy).toBe(`${timestamp} DESC`); + expect(configs.flows.timestampValueExpression).toBe(mapping); + expect(configs.traffic.timestampValueExpression).toBe(mapping); + }, + ); + + it('normalizes partial traffic buckets without expanding the selected range', () => { + const selectedRange: [Date, Date] = [ + new Date('2026-10-09T12:00:30Z'), + new Date('2026-10-09T13:00:20Z'), + ]; + const configs = buildNetflowQueryConfigs({ + source, + dateRange: selectedRange, + filters: {}, + }); + expect(configs.traffic).toMatchObject({ + dateRange: selectedRange, + alignDateRangeToGranularity: false, + }); + expect(configs.traffic.select).toEqual([ + { + alias: 'Bits per second', + valueExpression: expect.stringMatching(/least\(.*greatest\(/), + }, + ]); + expect(configs.summary.dateRange).toEqual(selectedRange); + }); + + it('estimates sampled traffic but counts stored flow records', () => { + const configs = buildNetflowQueryConfigs({ + source, + dateRange, + filters: {}, + }); + expect(configs.flows.select).toContainEqual({ + valueExpression: 'if((SamplingRate) > 0, (SamplingRate), 1)', + alias: '__netflow_samplingRate', + }); + expect(configs.summary.select).toEqual([ + { + valueExpression: + 'sum(toFloat64(Bytes) * (if((SamplingRate) > 0, (SamplingRate), 1))) * 8 / 3600', + alias: '__netflow_bitsPerSecond', + }, + { + valueExpression: + 'sum(toFloat64(Packets) * (if((SamplingRate) > 0, (SamplingRate), 1))) / 3600', + alias: '__netflow_packetsPerSecond', + }, + { + valueExpression: + 'sum(toFloat64(Bytes) * (if((SamplingRate) > 0, (SamplingRate), 1)))', + alias: '__netflow_bytes', + }, + { valueExpression: 'count()', alias: '__netflow_flowRecords' }, + ]); + expect(configs.traffic.select).toEqual([ + { + valueExpression: expect.stringContaining( + 'sum(toFloat64(Bytes) * (if((SamplingRate) > 0, (SamplingRate), 1))) * 8 /', + ), + alias: 'Bits per second', + }, + ]); + }); + + it.each([undefined, ' '])( + 'supports empty optional mappings (%s)', + blank => { + const configs = buildNetflowQueryConfigs({ + source: { + ...source, + samplingRateExpression: blank, + exporterExpression: blank, + inIfExpression: blank, + outIfExpression: blank, + }, + dateRange, + filters: {}, + }); + expect(configs.totalBytes.select).toEqual([ + { + valueExpression: 'sum(toFloat64(Bytes) * (1))', + alias: '__netflow_value', + }, + ]); + expect(configs.exporters).toBeUndefined(); + expect(configs.inInterfaces).toBeUndefined(); + expect(configs.outInterfaces).toBeUndefined(); + }, + ); + + it('normalizes IPv4 and IPv6 filters and quotes user values', () => { + const where = buildNetflowWhere(source, { + srcAddr: '192.0.2.1', + dstAddr: '2001:db8::1', + exporter: "edge\\' OR 1=1 --", + protocol: '6', + }); + expect(where).toContain( + "toIPv6OrNull(toString(SrcAddr)) = toIPv6OrNull('192.0.2.1')", + ); + expect(where).toContain( + "toIPv6OrNull(toString(DstAddr)) = toIPv6OrNull('2001:db8::1')", + ); + expect(where).toContain("'edge\\\\'' OR 1=1 --'"); + expect(where).toContain("toString(Proto) = '6'"); + }); + + it.each([ + ['tcp', '6'], + [' ICMPv6 ', '58'], + ['99', '99'], + ['SCTP', 'SCTP'], + ])('filters protocol %p as %p', (protocol, expected) => { + expect(buildNetflowWhere(source, { protocol })).toBe( + `(toString(Proto) = '${expected}')`, + ); + }); + + it('keeps the source, connection, time range and filters on every query', () => { + const configs = buildNetflowQueryConfigs({ + source, + dateRange, + filters: { protocol: '17' }, + }); + for (const config of Object.values(configs)) { + expect(config).toMatchObject({ + source: 'flows', + connection: 'local', + dateRange, + from: source.from, + whereLanguage: 'lucene', + where: '', + filters: [{ type: 'sql', condition: "(toString(Proto) = '17')" }], + }); + } + expect(configs.flows.limit).toEqual({ limit: 500 }); + expect(configs.topSourceAddresses.limit).toEqual({ limit: 10 }); + }); + + it.each(['lucene', 'sql'] as const)( + 'combines %s search with quick filters on every chart', + whereLanguage => { + const where = + whereLanguage === 'lucene' + ? 'DstPort:443 OR DstPort:80' + : 'DstPort = 443 OR DstPort = 80'; + const configs = buildNetflowQueryConfigs({ + source, + dateRange, + filters: { protocol: '6' }, + where, + whereLanguage, + extraFilters: [{ type: 'sql', condition: "ExporterName = 'edge-a'" }], + }); + for (const config of Object.values(configs)) { + expect(config).toMatchObject({ + where, + whereLanguage, + filters: [ + { type: 'sql', condition: "ExporterName = 'edge-a'" }, + { type: 'sql', condition: "(toString(Proto) = '6')" }, + ], + }); + } + }, + ); + + it('rejects empty or reversed time ranges', () => { + expect(() => + buildNetflowQueryConfigs({ + source, + dateRange: [dateRange[1], dateRange[0]], + filters: {}, + }), + ).toThrow('Time range'); + }); +}); + +describe.each([undefined, 'SearchText'])( + 'NetFlow full-text hydration with override %s', + implicitColumnExpression => { + const selectedSource = { ...source, implicitColumnExpression }; + const select = [{ valueExpression: 'sum(Bytes)' }]; + const expected = + implicitColumnExpression ?? expect.stringContaining('toString(SrcAddr)'); + + it.each(['sql', 'builder'] as const)( + 'carries the search mapping into %s chart editor previews', + configType => { + const config = convertFormStateToChartConfig( + { + configType, + displayType: DisplayType.Line, + source: source.id, + connection: source.connection, + where: 'edge', + whereLanguage: 'lucene', + series: select, + sqlTemplate: 'SELECT sum(Bytes) FROM flows WHERE $where', + }, + dateRange, + selectedSource, + ); + expect(config).toMatchObject({ implicitColumnExpression: expected }); + }, + ); + + it.each(['sql', 'builder'] as const)( + 'carries the search mapping into %s saved alert charts', + configType => { + const savedConfig = + configType === 'sql' + ? { + configType: 'sql' as const, + displayType: DisplayType.Line, + source: source.id, + connection: source.connection, + sqlTemplate: 'SELECT sum(Bytes) FROM flows WHERE $where', + } + : { + displayType: DisplayType.Line, + source: source.id, + select, + where: 'edge', + whereLanguage: 'lucene' as const, + }; + const config = buildAlertChartConfig({ + savedConfig, + source: selectedSource, + dateRange, + granularity: Granularity.OneMinute, + variables: [], + }); + expect(config).toMatchObject({ implicitColumnExpression: expected }); + }, + ); + + it('carries the search mapping into release queries scoped by bare Lucene terms', () => { + const config = buildReleaseChartConfig( + selectedSource, + 'Version', + dateRange, + { where: 'edge', whereLanguage: 'lucene' }, + ); + expect(config).toMatchObject({ + implicitColumnExpression: expected, + filters: [{ type: 'lucene', condition: 'edge' }], + }); + }); + }, +); diff --git a/packages/app/src/__tests__/netflowSankey.test.ts b/packages/app/src/__tests__/netflowSankey.test.ts new file mode 100644 index 0000000000..bc4d93aac1 --- /dev/null +++ b/packages/app/src/__tests__/netflowSankey.test.ts @@ -0,0 +1,213 @@ +import { + BuilderChartConfigWithDateRange, + DisplayType, +} from '@hyperdx/common-utils/dist/types'; + +import { + buildNetflowSankeyConfig, + buildNetflowSankeyData, + SankeyDimension, + sankeyDimensionExpression, + unwrapSankeyDimensionExpression, +} from '@/netflowSankey'; + +const dimensions: SankeyDimension[] = [ + { key: 'source', label: 'Source IP', expression: 'SrcAddr' }, + { + key: 'exporter', + label: 'Exporter', + expression: "coalesce(router, 'unknown')", + }, + { key: 'destination', label: 'Destination IP', expression: 'DstAddr' }, +]; +const baseConfig: BuilderChartConfigWithDateRange = { + source: 'flows', + connection: 'connection', + from: { databaseName: 'default', tableName: 'flows' }, + timestampValueExpression: 'TimeReceived', + dateRange: [ + new Date('2026-10-09T10:00:00Z'), + new Date('2026-10-09T11:00:00Z'), + ], + where: 'Proto:6 OR Proto:17', + whereLanguage: 'lucene', + filters: [{ type: 'sql', condition: "ExporterName NOT IN ('offline')" }], + dateRangeEndInclusive: false, + displayType: DisplayType.Number, + select: [ + { + valueExpression: 'sum(toFloat64(Bytes) * (SamplingRate))', + alias: '__netflow_value', + }, + ], +}; + +describe('NetFlow Sankey query', () => { + it('preserves source, filters, time range, and sampled byte aggregation', () => { + const config = buildNetflowSankeyConfig({ + baseConfig, + dimensions, + limit: 50, + }); + expect(config).toMatchObject({ + source: baseConfig.source, + connection: baseConfig.connection, + from: baseConfig.from, + dateRange: baseConfig.dateRange, + timestampValueExpression: 'TimeReceived', + where: baseConfig.where, + whereLanguage: 'lucene', + filters: baseConfig.filters, + dateRangeEndInclusive: false, + displayType: DisplayType.Table, + groupBy: + '__netflow_dimension_0, __netflow_dimension_1, __netflow_dimension_2', + orderBy: '__netflow_value DESC', + limit: { limit: 50 }, + }); + expect(config.select).toEqual([ + { + valueExpression: "ifNull(toString(SrcAddr), '')", + alias: '__netflow_dimension_0', + }, + { + valueExpression: "ifNull(toString(coalesce(router, 'unknown')), '')", + alias: '__netflow_dimension_1', + }, + { + valueExpression: "ifNull(toString(DstAddr), '')", + alias: '__netflow_dimension_2', + }, + { + valueExpression: 'sum(toFloat64(Bytes) * (SamplingRate))', + alias: '__netflow_value', + }, + ]); + expect(baseConfig.select).toHaveLength(1); + expect(baseConfig).not.toHaveProperty('groupBy'); + }); + + it.each([ + [undefined, 20], + [NaN, 20], + [Infinity, 20], + [-5, 20], + [0, 20], + [101, 20], + [2.9, 20], + [10, 10], + [50, 50], + ])('bounds the requested path limit %s to %s', (limit, expected) => { + expect( + buildNetflowSankeyConfig({ baseConfig, dimensions, limit }).limit, + ).toEqual({ limit: expected }); + }); + + it('rejects queries without a path or a byte aggregate', () => { + expect(() => + buildNetflowSankeyConfig({ baseConfig, dimensions: [] }), + ).toThrow(); + expect(() => + buildNetflowSankeyConfig({ baseConfig, dimensions: [dimensions[0]] }), + ).toThrow(); + expect(() => + buildNetflowSankeyConfig({ + baseConfig, + dimensions: [dimensions[0], { ...dimensions[1], expression: ' ' }], + }), + ).toThrow(); + expect(() => + buildNetflowSankeyConfig({ + baseConfig: { ...baseConfig, select: '*' }, + dimensions, + }), + ).toThrow(); + }); +}); + +describe('NetFlow Sankey data', () => { + const row = (values: unknown[], value: unknown) => ({ + ...Object.fromEntries( + values.map((v, i) => [`__netflow_dimension_${i}`, v]), + ), + __netflow_value: value, + }); + + it('aggregates converging links and conserves sampled bytes at every stage', () => { + const data = buildNetflowSankeyData( + [ + row(['a', 'router', 'x'], '1000'), + row(['b', 'router', 'x'], 2000), + row(['a', 'router', 'y'], 4000), + ], + dimensions, + ); + expect(data.paths.map(path => path.value)).toEqual([1000, 2000, 4000]); + expect(data.links).toHaveLength(4); + for (const stage of [0, 1]) { + expect( + data.links + .filter(link => data.nodes[link.source].stage === stage) + .reduce((total, link) => total + link.value, 0), + ).toBe(7000); + } + expect( + data.links.find(link => data.nodes[link.target].rawValue === 'x')?.value, + ).toBe(3000); + expect( + data.links.find(link => data.nodes[link.source].rawValue === 'a')?.value, + ).toBe(5000); + }); + + it('keeps repeated labels distinct across stages without cycles', () => { + const data = buildNetflowSankeyData( + [row(['same', 'same', 'same'], 5)], + dimensions, + ); + expect(new Set(data.nodes.map(node => node.id)).size).toBe(3); + expect(data.links).toEqual([ + { source: 0, target: 1, value: 5 }, + { source: 1, target: 2, value: 5 }, + ]); + expect(data.nodes.map(node => node.dimension)).toEqual(dimensions); + }); + + it('keeps empty raw values separate from the literal empty display label', () => { + const data = buildNetflowSankeyData( + [row(['', null, '(empty)'], 5)], + dimensions, + ); + expect(data.paths[0].values).toEqual(['', '', '(empty)']); + expect(data.nodes.map(node => node.name)).toEqual([ + '(empty)', + '(empty)', + '(empty)', + ]); + expect(data.nodes.map(node => node.rawValue)).toEqual(['', '', '(empty)']); + }); + + it('ignores invalid counters, nonpositive paths, and rows missing a dimension', () => { + const rows = [NaN, Infinity, -1, 0, '', null, true, 'bad'].map(value => + row(['a', 'b', 'c'], value), + ); + rows.push(row(['missing'], 5)); + expect(buildNetflowSankeyData(rows, dimensions)).toEqual({ + nodes: [], + links: [], + paths: [], + }); + expect(buildNetflowSankeyData([], dimensions).paths).toEqual([]); + expect(buildNetflowSankeyData([row(['a'], 5)], []).nodes).toEqual([]); + }); +}); + +it('unwraps its own dimension expression while preserving custom SQL', () => { + for (const dimension of dimensions) { + expect( + unwrapSankeyDimensionExpression(sankeyDimensionExpression(dimension)), + ).toBe(dimension.expression); + expect(unwrapSankeyDimensionExpression(dimension.expression)).toBe( + dimension.expression, + ); + } +}); diff --git a/packages/app/src/__tests__/netflowSankeyDimensions.test.ts b/packages/app/src/__tests__/netflowSankeyDimensions.test.ts new file mode 100644 index 0000000000..b891aa6d6e --- /dev/null +++ b/packages/app/src/__tests__/netflowSankeyDimensions.test.ts @@ -0,0 +1,129 @@ +import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; +import { SourceKind, TNetflowSource } from '@hyperdx/common-utils/dist/types'; + +import { + getDefaultSankeyDimensions, + getSankeyDimensionOptions, +} from '@/netflowSankeyDimensions'; + +const source: TNetflowSource = { + id: 'flows', + name: 'Flows', + kind: SourceKind.Netflow, + connection: 'local', + from: { databaseName: 'default', tableName: 'flows' }, + timestampValueExpression: 'TimeReceived', + defaultTableSelectExpression: '*', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + exporterExpression: 'ExporterName', +}; + +const column = (name: string, type = 'String'): ColumnMeta => ({ + name, + type, + codec_expression: '', + comment: '', + default_expression: '', + default_type: '', + ttl_expression: '', +}); + +describe('NetFlow Sankey dimension options', () => { + it.each([ + 'String', + 'LowCardinality(Nullable(String))', + 'UInt32', + 'Float64', + 'Bool', + 'DateTime64(3)', + 'IPv6', + ])('offers scalar %s columns and quotes unusual identifiers', type => { + expect( + getSankeyDimensionOptions(source, [column('custom field', type)]), + ).toContainEqual({ + key: 'column:custom field', + label: 'custom field', + expression: '`custom field`', + }); + }); + + it.each([ + 'Array(String)', + 'Map(String, String)', + 'Tuple(String, UInt32)', + 'JSON', + 'Dynamic', + 'Unknown', + ])('excludes %s columns from scalar dimensions', type => { + expect( + getSankeyDimensionOptions(source, [column('nested', type)]), + ).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'column:nested' }), + ]), + ); + }); + + it('omits blank optional mappings and retains trimmed mapped expressions', () => { + const options = getSankeyDimensionOptions( + { + ...source, + exporterExpression: ' ', + inIfExpression: '', + outIfExpression: ' OutIfName ', + }, + [], + ); + expect(options.map(option => option.key)).toEqual([ + 'srcAddr', + 'dstAddr', + 'protocol', + 'outputInterface', + ]); + expect(options).toContainEqual({ + key: 'outputInterface', + label: 'Output interface', + expression: 'OutIfName', + }); + }); +}); + +describe('NetFlow Sankey default dimensions', () => { + const enrichment = [ + column('SrcAS', 'UInt32'), + column('InIfConnectivity'), + column('InIfProvider'), + ]; + + it('uses the Akvorado path when every enriched dimension is available', () => { + expect( + getDefaultSankeyDimensions(getSankeyDimensionOptions(source, enrichment)), + ).toEqual([ + 'column:SrcAS', + 'column:InIfConnectivity', + 'column:InIfProvider', + 'exporter', + ]); + }); + + it.each([ + [source, []], + [source, enrichment.slice(0, 2)], + [{ ...source, exporterExpression: '' }, enrichment], + ] as const)( + 'falls back when an Akvorado dimension is unavailable', + (selectedSource, columns) => { + expect( + getDefaultSankeyDimensions( + getSankeyDimensionOptions(selectedSource, [...columns]), + ), + ).toEqual(['srcAddr', 'protocol', 'dstAddr']); + }, + ); +}); diff --git a/packages/app/src/__tests__/source.test.ts b/packages/app/src/__tests__/source.test.ts index fbc43e78ee..0bcebc79cc 100644 --- a/packages/app/src/__tests__/source.test.ts +++ b/packages/app/src/__tests__/source.test.ts @@ -1,5 +1,9 @@ import { ColumnMetaType } from '@hyperdx/common-utils/dist/clickhouse'; -import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; +import { + getMetadata, + Metadata, +} from '@hyperdx/common-utils/dist/core/metadata'; import { ChartConfigWithOptTimestamp, NumberFormat, @@ -813,3 +817,76 @@ describe('getBuilderValueColumnCount', () => { expect(getBuilderValueColumnCount(makeRawSqlConfig({}))).toBe(0); }); }); + +describe('NetFlow source inference', () => { + const infer = async (names: string[]) => { + const metadata = getMetadata( + new ClickhouseClient({ host: 'http://localhost:8123' }), + ); + jest.spyOn(metadata, 'getColumns').mockResolvedValue( + names.map(name => ({ + name, + type: name === 'TimeReceived' ? 'DateTime' : 'UInt64', + codec_expression: '', + comment: '', + default_expression: '', + default_type: '', + ttl_expression: '', + })), + ); + jest.spyOn(metadata, 'getTableMetadata').mockResolvedValue(undefined); + return inferTableSourceConfig({ + databaseName: 'default', + tableName: 'flows', + connectionId: 'connection', + kind: SourceKind.Netflow, + metadata, + }); + }; + it('maps Akvorado columns and preserves sampling semantics', async () => { + expect( + await infer([ + 'TimeReceived', + 'Bytes', + 'Packets', + 'SamplingRate', + 'SrcAddr', + 'DstAddr', + 'SrcPort', + 'DstPort', + 'Proto', + 'ExporterName', + 'InIfName', + 'OutIfName', + ]), + ).toMatchObject({ + kind: SourceKind.Netflow, + timestampValueExpression: 'TimeReceived', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + samplingRateExpression: 'SamplingRate', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + protocolExpression: 'Proto', + }); + }); + it('does not invent optional columns for unenriched or unsampled tables', async () => { + const config = await infer([ + 'TimeReceived', + 'Bytes', + 'Packets', + 'SrcAddr', + 'DstAddr', + 'SrcPort', + 'DstPort', + 'Proto', + ]); + // Empty, not omitted, so that a mapping from a previously selected table is cleared. + expect(config).toMatchObject({ + samplingRateExpression: '', + exporterExpression: '', + inIfExpression: '', + outIfExpression: '', + }); + }); +}); diff --git a/packages/app/src/components/ActiveFilterPills.tsx b/packages/app/src/components/ActiveFilterPills.tsx index 86d8234951..ad88da4c7f 100644 --- a/packages/app/src/components/ActiveFilterPills.tsx +++ b/packages/app/src/components/ActiveFilterPills.tsx @@ -74,6 +74,8 @@ function flattenFilters( function SearchFilterPill({ pill, + fieldLabel, + enableValueEditing, isInvalid, invalidReason, chartConfig, @@ -82,6 +84,8 @@ function SearchFilterPill({ onReplaceValue, }: { pill: PillItem; + fieldLabel?: string; + enableValueEditing: boolean; isInvalid?: boolean; invalidReason?: string; chartConfig: BuilderChartConfigWithDateRange; @@ -92,7 +96,7 @@ function SearchFilterPill({ const isExcluded = pill.type === 'excluded'; // A range pill has no single value to copy or flip, and an unapplied filter // (column missing on the active source) can only be removed. - const isEditable = pill.type !== 'range' && !isInvalid; + const isEditable = enableValueEditing && pill.type !== 'range' && !isInvalid; const [opened, setOpened] = useState(false); // The picker lists values to switch this pill to, so it must not be scoped @@ -118,7 +122,7 @@ function SearchFilterPill({ return ( 0 ? select - : isLogSource(source) || isTraceSource(source) + : isLogSource(source) || isTraceSource(source) || isNetflowSource(source) ? source.defaultTableSelectExpression : undefined) ?? ''; const { data: aliasMap } = useAliasMapFromChartConfig({ select: resolvedSelect, + implicitColumnExpression: getSourceImplicitColumnExpression(source), where: where || '', connection: source.connection, from: source.from, diff --git a/packages/app/src/components/AppNav/AppNav.tsx b/packages/app/src/components/AppNav/AppNav.tsx index ff77863ba2..6fcc91c054 100644 --- a/packages/app/src/components/AppNav/AppNav.tsx +++ b/packages/app/src/components/AppNav/AppNav.tsx @@ -14,6 +14,7 @@ import { IconDeviceFloppy, IconDeviceLaptop, IconLayoutGrid, + IconNetwork, IconSettings, IconSitemap, IconTable, @@ -102,6 +103,13 @@ const NAV_LINKS: NavLinkConfig[] = [ icon: , isBeta: true, }, + { + id: 'netflow', + label: 'NetFlow', + href: '/netflow', + icon: , + isBeta: true, + }, ]; export default function AppNav({ fixed = false }: { fixed?: boolean }) { diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 5607dcdaac..f898022335 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -7,6 +7,7 @@ import { displayTypeSupportsReducer, getPromqlSeries, } from '@hyperdx/common-utils/dist/core/promql'; +import { getSourceImplicitColumnExpression } from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { isFormulaDisplayType, isFormulaSourceKind, @@ -418,10 +419,9 @@ export function convertFormStateToChartConfig( connection: form.connection ?? '', source: form.source || undefined, from: source?.from, - implicitColumnExpression: - source && (isLogSource(source) || isTraceSource(source)) - ? source.implicitColumnExpression - : undefined, + implicitColumnExpression: source + ? getSourceImplicitColumnExpression(source) + : undefined, // Body expression is only populated for log sources; trace sources use // `spanNameExpression` for display, which has a different semantic for // bare-text search and should not auto-fall-back. @@ -459,10 +459,7 @@ export function convertFormStateToChartConfig( timestampValueExpression: source.timestampValueExpression, dateRange, connection: source.connection, - implicitColumnExpression: - isLogSource(source) || isTraceSource(source) - ? source.implicitColumnExpression - : undefined, + implicitColumnExpression: getSourceImplicitColumnExpression(source), // Logs-only body fallback (see comment above for raw-sql config). bodyExpression: isLogSource(source) ? source.bodyExpression : undefined, useTextIndexForImplicitColumn: diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx index 4524f411ab..f3330e75d6 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx @@ -550,13 +550,15 @@ export function ChartEditorControls({ name="select" placeholder={ ((tableSource?.kind === SourceKind.Log || - tableSource?.kind === SourceKind.Trace) && + tableSource?.kind === SourceKind.Trace || + tableSource?.kind === SourceKind.Netflow) && tableSource.defaultTableSelectExpression) || 'SELECT Columns' } defaultValue={ tableSource?.kind === SourceKind.Log || - tableSource?.kind === SourceKind.Trace + tableSource?.kind === SourceKind.Trace || + tableSource?.kind === SourceKind.Netflow ? tableSource.defaultTableSelectExpression : undefined } diff --git a/packages/app/src/components/DBListBarChart.tsx b/packages/app/src/components/DBListBarChart.tsx index ef463ff1df..f77d39ec85 100644 --- a/packages/app/src/components/DBListBarChart.tsx +++ b/packages/app/src/components/DBListBarChart.tsx @@ -27,7 +27,7 @@ function ListItem({ hoverCardContent, hoverCardPosition = 'right', }: { - title: string; + title: React.ReactNode; value: string; color: string; percent: number; @@ -81,6 +81,7 @@ function ListBar({ getRowSearchLink, columns, hoverCardPosition, + renderGroupLabel, }: { data: any[]; valueColumn: string; @@ -93,6 +94,7 @@ function ListBar({ visible?: boolean; }[]; hoverCardPosition?: FloatingPosition; + renderGroupLabel?: (value: string) => React.ReactNode; }) { const values = (data ?? []).map(row => row[valueColumn]); const maxValue = Math.max(...values); @@ -148,7 +150,9 @@ function ListBar({ c="inherit" > void; @@ -201,6 +208,7 @@ export default function DBListBarChart({ toolbarItems?: React.ReactNode[]; showMVOptimizationIndicator?: boolean; errorVariant?: ChartErrorStateVariant; + renderGroupLabel?: (value: string) => React.ReactNode; }) { const queriedConfig = omit(config, ['granularity']); const { data, isLoading, isPlaceholderData, isError, error } = @@ -301,6 +309,7 @@ export default function DBListBarChart({ hoverCardPosition={hoverCardPosition} groupColumn={groupColumn} valueColumn={valueColumn} + renderGroupLabel={renderGroupLabel} />
)} diff --git a/packages/app/src/components/DBNumberChart.tsx b/packages/app/src/components/DBNumberChart.tsx index 6bbf580e3c..c58e3f3e11 100644 --- a/packages/app/src/components/DBNumberChart.tsx +++ b/packages/app/src/components/DBNumberChart.tsx @@ -12,6 +12,7 @@ import { } from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithDateRange, + NumberFormat, resolveChartPaletteToken, } from '@hyperdx/common-utils/dist/types'; import { Flex, Text } from '@mantine/core'; @@ -216,6 +217,8 @@ export default function DBNumberChart({ toolbarSuffix, showMVOptimizationIndicator = true, errorVariant, + valueColumn: selectedValueColumn, + numberFormat, }: { config: ChartConfigWithDateRange; queryKeyPrefix?: string; @@ -225,6 +228,8 @@ export default function DBNumberChart({ toolbarSuffix?: React.ReactNode[]; showMVOptimizationIndicator?: boolean; errorVariant?: ChartErrorStateVariant; + valueColumn?: string; + numberFormat?: NumberFormat; }) { const { data: source } = useSource({ id: config.source, @@ -255,10 +260,12 @@ export default function DBNumberChart({ enabled, }); - // The value is the first numeric value in the first row of the result - const valueColumn = data?.meta - ? filterColumnMetaByType(data?.meta, [JSDataType.Number])?.[0] - : undefined; + // An explicit column lets several tiles share one aggregate query. + const valueColumn = selectedValueColumn + ? data?.meta?.find(column => column.name === selectedValueColumn) + : data?.meta + ? filterColumnMetaByType(data?.meta, [JSDataType.Number])?.[0] + : undefined; const resultError = data && !valueColumn && isRawSqlChartConfig(queriedConfig) ? new Error( @@ -279,10 +286,15 @@ export default function DBNumberChart({ const resolvedNumberFormat = useSingleSeriesNumberFormat(queriedConfig); - const value = valueColumn - ? data?.data?.[0]?.[valueColumn.name] - : (Object.values(data?.data?.[0] ?? {})?.[0] ?? Number.NaN); - const formattedValue = formatNumber(value as number, resolvedNumberFormat); + const value = selectedValueColumn + ? (data?.data?.[0]?.[selectedValueColumn] ?? Number.NaN) + : valueColumn + ? data?.data?.[0]?.[valueColumn.name] + : (Object.values(data?.data?.[0] ?? {})?.[0] ?? Number.NaN); + const formattedValue = formatNumber( + value as number, + numberFormat ?? resolvedNumberFormat, + ); // Resolve the display color in three layers: // 1. Conditional color rules evaluated against the raw value diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index b7fccb9640..7c028383d0 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -839,7 +839,11 @@ export const DBRowSidePanelInner = ({ const hasStack = crumbSourceStack.length > 0 || crumbNavStack.length > 0; const rootLabel = initialMainContent || - (rootSource.kind === SourceKind.Trace ? 'Trace' : 'Log'); + (rootSource.kind === SourceKind.Trace + ? 'Trace' + : rootSource.kind === SourceKind.Netflow + ? 'Flow' + : 'Log'); if (hasStack) { items.push({ diff --git a/packages/app/src/components/Dashboards/DashboardsListPage.tsx b/packages/app/src/components/Dashboards/DashboardsListPage.tsx index 49777c9f27..02e0ee1ba2 100644 --- a/packages/app/src/components/Dashboards/DashboardsListPage.tsx +++ b/packages/app/src/components/Dashboards/DashboardsListPage.tsx @@ -94,6 +94,11 @@ const PRESET_DASHBOARDS = [ href: '/llm', description: 'LLM calls, token usage, cost, and latency by model', }, + { + name: 'NetFlow', + href: '/netflow', + description: 'Network traffic, top talkers, exporters, and flow records', + }, ]; export default function DashboardsListPage() { diff --git a/packages/app/src/components/NetflowCharts.tsx b/packages/app/src/components/NetflowCharts.tsx new file mode 100644 index 0000000000..8c6b4a909f --- /dev/null +++ b/packages/app/src/components/NetflowCharts.tsx @@ -0,0 +1,94 @@ +import { SimpleGrid, Stack } from '@mantine/core'; + +import { + buildNetflowQueryConfigs, + NETFLOW_ALIASES, + NETFLOW_SUMMARY_TILES, +} from '@/netflow'; +import { SankeyFilterHandler } from '@/netflowSankey'; + +import { ChartCard } from './charts/ChartCard'; +import DBListBarChart from './DBListBarChart'; +import DBNumberChart from './DBNumberChart'; +import NetflowFilterMenu, { NetflowFilterHandler } from './NetflowFilterMenu'; +import NetflowRecords from './NetflowRecords'; +import NetflowVisualization from './NetflowVisualization'; + +export default function NetflowCharts({ + configs, + onTimeRangeSelect, + onFilter, + onDimensionFilter, +}: { + configs: ReturnType; + onTimeRangeSelect: (start: Date, end: Date) => void; + onFilter?: NetflowFilterHandler; + onDimensionFilter?: SankeyFilterHandler; +}) { + return ( + + + {NETFLOW_SUMMARY_TILES.map(({ title, column, numberFormat }) => ( + + + + ))} + + + + {( + [ + ['Top source addresses', configs.topSourceAddresses, 'srcAddr'], + [ + 'Top destination addresses', + configs.topDestinationAddresses, + 'dstAddr', + ], + ['Protocols', configs.protocols, 'protocol'], + ['Exporters', configs.exporters, 'exporter'], + ['Input interfaces', configs.inInterfaces, 'inputInterface'], + ['Output interfaces', configs.outInterfaces, 'outputInterface'], + ] as const + ).map( + ([title, config, field]) => + config && ( + + ( + + )} + /> + + ), + )} + + + + ); +} diff --git a/packages/app/src/components/NetflowFilterMenu.tsx b/packages/app/src/components/NetflowFilterMenu.tsx new file mode 100644 index 0000000000..b9a2d4d636 --- /dev/null +++ b/packages/app/src/components/NetflowFilterMenu.tsx @@ -0,0 +1,88 @@ +import { KeyboardEventHandler, memo, ReactElement, useState } from 'react'; +import { Button, Menu } from '@mantine/core'; + +import { NETFLOW_DIMENSION_LABELS, NetflowFilterField } from '@/netflow'; + +export type NetflowFilterHandler = ( + field: NetflowFilterField, + value: string, + excluded: boolean, +) => void; + +type TargetProps = { + opened: boolean; + buttonProps: { + 'aria-label': string; + 'aria-haspopup'?: 'menu'; + 'aria-expanded'?: boolean; + onClick: () => void; + onKeyDown: KeyboardEventHandler; + role?: 'button'; + tabIndex?: number; + }; +}; +type Props = { + value: string; + allowEmpty?: boolean; + target?: (props: TargetProps) => ReactElement; +} & ( + | { field: NetflowFilterField; onFilter?: NetflowFilterHandler } + | { label: string; onSelect?: (excluded: boolean) => void } +); + +export default memo(function NetflowFilterMenu(props: Props) { + const [opened, setOpened] = useState(false); + const [activated, setActivated] = useState(false); + const { value, allowEmpty = false, target } = props; + const label = + 'field' in props ? NETFLOW_DIMENSION_LABELS[props.field] : props.label; + const onSelect = + 'field' in props + ? props.onFilter && + ((excluded: boolean) => props.onFilter?.(props.field, value, excluded)) + : props.onSelect; + const interactive = !!onSelect && (allowEmpty || !!value.trim()); + const displayValue = value || (allowEmpty ? '(empty)' : '—'); + const buttonProps: TargetProps['buttonProps'] = { + 'aria-label': `Filter ${label}: ${displayValue}`, + 'aria-haspopup': interactive ? 'menu' : undefined, + 'aria-expanded': interactive ? opened : undefined, + onClick: () => { + if (interactive && !activated) { + setActivated(true); + setOpened(true); + } + }, + role: interactive ? 'button' : undefined, + tabIndex: interactive ? 0 : undefined, + onKeyDown: event => { + if (interactive && (event.key === 'Enter' || event.key === ' ')) { + event.preventDefault(); + setActivated(true); + setOpened(true); + } + }, + }; + const renderedTarget = target?.({ opened, buttonProps }) ?? ( + + ); + if (!interactive) return target ? renderedTarget : <>{displayValue}; + // Flow tables can have thousands of filter cells; create a popover only when used. + if (!activated) return renderedTarget; + return ( + + {renderedTarget} + + onSelect?.(false)}>Include + onSelect?.(true)}>Exclude + + + ); +}); diff --git a/packages/app/src/components/NetflowFilterPills.tsx b/packages/app/src/components/NetflowFilterPills.tsx new file mode 100644 index 0000000000..fc03377630 --- /dev/null +++ b/packages/app/src/components/NetflowFilterPills.tsx @@ -0,0 +1,215 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { + FilterState, + filtersToQuery, + isRenderablePinnedFilter, + parseQuery, +} from '@hyperdx/common-utils/dist/filters'; +import { + BuilderChartConfigWithDateRange, + Filter, + TNetflowSource, +} from '@hyperdx/common-utils/dist/types'; + +import { getNetflowDimensions, NETFLOW_DIMENSION_LABELS } from '@/netflow'; +import { + sankeyDimensionExpression, + SankeyFilterHandler, + unwrapSankeyDimensionExpression, +} from '@/netflowSankey'; +import { useSearchPageFilterState } from '@/searchFilters'; + +import { cleanClickHouseExpression } from './DBSearchPageFilters/utils'; +import { ActiveFilterPills } from './ActiveFilterPills'; +import { NetflowFilterHandler } from './NetflowFilterMenu'; + +// Table and Sankey filters can wrap the same mapped field in different SQL expressions. +function canonicalizeMappedFilters( + filters: Filter[], + expressions: Map, +): Filter[] { + const mapped: FilterState = {}; + const remaining: Filter[] = []; + let migrated = false; + for (const filter of filters) { + if (!isRenderablePinnedFilter(filter)) { + remaining.push(filter); + continue; + } + const [entry] = Object.entries(parseQuery([filter]).filters); + const canonical = + entry && expressions.get(cleanClickHouseExpression(entry[0])); + if (!canonical) { + remaining.push(filter); + continue; + } + const [key, values] = entry; + migrated ||= key !== canonical; + const target = (mapped[canonical] ??= { + included: new Set(), + excluded: new Set(), + }); + values.included.forEach(value => target.included.add(value)); + values.excluded.forEach(value => target.excluded.add(value)); + if (values.range) target.range = values.range; + } + return migrated ? [...remaining, ...filtersToQuery(mapped)] : filters; +} + +export function useNetflowFilterState({ + source, + filters, + onChange, +}: { + source?: TNetflowSource; + filters: Filter[]; + onChange: (filters: Filter[]) => void; +}) { + // Keys are SQL expressions, not physical columns; restore their quoting at emission. + const knownColumns = useMemo(() => new Set(), []); + const dimensions = useMemo( + () => (source ? getNetflowDimensions(source) : undefined), + [source], + ); + const expressions = useMemo(() => { + const result = new Map( + Object.keys(parseQuery(filters).filters).map(expression => [ + cleanClickHouseExpression(expression), + expression, + ]), + ); + for (const [field, expression] of Object.entries(dimensions ?? {})) { + if (!expression) continue; + const canonical = sankeyDimensionExpression({ + key: field, + label: field, + expression, + }); + result.set(cleanClickHouseExpression(expression), canonical); + result.set(cleanClickHouseExpression(canonical), canonical); + } + return result; + }, [dimensions, filters]); + const normalizedFilters = useMemo( + () => canonicalizeMappedFilters(filters, expressions), + [filters, expressions], + ); + const onChangeRef = useRef(onChange); + useEffect(() => { + onChangeRef.current = onChange; + }, [onChange]); + useEffect(() => { + if (normalizedFilters !== filters) onChangeRef.current(normalizedFilters); + }, [normalizedFilters, filters]); + const clickedExpressionsRef = useRef(new Map()); + // Conditions that pills cannot represent (e.g. `Bytes > 100`) stay in the search untouched. + const emitFilters = (next: FilterState) => + onChange([ + ...normalizedFilters.filter( + filter => Object.keys(parseQuery([filter]).filters).length === 0, + ), + ...filtersToQuery( + Object.fromEntries( + Object.entries(next).map(([key, selection]) => [ + expressions.get(key) ?? + clickedExpressionsRef.current.get(key) ?? + key, + selection, + ]), + ), + ), + ]); + const state = useSearchPageFilterState({ + searchQuery: normalizedFilters, + onFilterChange: next => emitFilters(parseQuery(next).filters), + knownColumns, + }); + const applyFilter = ( + expression: string, + value: string, + excluded: boolean, + ) => { + const field = cleanClickHouseExpression(expression); + clickedExpressionsRef.current.set(field, expression); + state.setFilters(previous => { + const old = previous[field]; + const polarity = excluded ? 'excluded' : 'included'; + const opposite = excluded ? 'included' : 'excluded'; + if (old?.[polarity].has(value) && !old[opposite].has(value)) + return previous; + const values = { + ...old, + included: new Set(old?.included), + excluded: new Set(old?.excluded), + }; + values[polarity].add(value); + values[opposite].delete(value); + const next = { ...previous, [field]: values }; + emitFilters(next); + return next; + }); + }; + const onFilter: NetflowFilterHandler = (field, value, excluded) => { + const expression = dimensions?.[field]; + const canonical = + expression && expressions.get(cleanClickHouseExpression(expression)); + if (canonical) applyFilter(canonical, value, excluded); + }; + const onDimensionFilter: SankeyFilterHandler = ( + dimension, + value, + excluded, + ) => { + applyFilter( + expressions.get(cleanClickHouseExpression(dimension.expression)) ?? + sankeyDimensionExpression(dimension), + value, + excluded, + ); + }; + const fieldLabels = Object.fromEntries( + Object.entries(dimensions ?? {}).flatMap(([field, expression]) => + expression + ? [ + [ + cleanClickHouseExpression(expression), + NETFLOW_DIMENSION_LABELS[field], + ], + [ + cleanClickHouseExpression( + sankeyDimensionExpression({ + key: field, + label: field, + expression, + }), + ), + NETFLOW_DIMENSION_LABELS[field], + ], + ] + : [], + ), + ); + return { state, onFilter, onDimensionFilter, fieldLabels }; +} + +export default function NetflowFilterPills({ + state, + fieldLabels, + chartConfig, +}: Pick, 'state' | 'fieldLabels'> & { + chartConfig: BuilderChartConfigWithDateRange; +}) { + return ( + + fieldLabels[field] ?? unwrapSankeyDimensionExpression(field) + } + px={0} + gap="xs" + data-testid="netflow-filter-pills" + /> + ); +} diff --git a/packages/app/src/components/NetflowRecords.tsx b/packages/app/src/components/NetflowRecords.tsx new file mode 100644 index 0000000000..8922231664 --- /dev/null +++ b/packages/app/src/components/NetflowRecords.tsx @@ -0,0 +1,227 @@ +import { useMemo, useState } from 'react'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { + Button, + Drawer, + Loader, + ScrollArea, + Stack, + Table, + Text, +} from '@mantine/core'; + +import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { NETFLOW_DIMENSION_LABELS, netflowColumnAlias } from '@/netflow'; +import { useFormatTime } from '@/useFormatTime'; +import { formatNumber } from '@/utils'; + +import { ChartCard } from './charts/ChartCard'; +import ChartContainer from './charts/ChartContainer'; +import ChartErrorState from './charts/ChartErrorState'; +import EmptyState from './EmptyState'; +import NetflowFilterMenu, { NetflowFilterHandler } from './NetflowFilterMenu'; + +const fields = [ + ['timestamp', 'Time'], + ['srcAddr', NETFLOW_DIMENSION_LABELS.srcAddr], + ['srcPort', 'Source port'], + ['dstAddr', NETFLOW_DIMENSION_LABELS.dstAddr], + ['dstPort', 'Destination port'], + ['protocol', NETFLOW_DIMENSION_LABELS.protocol], + ['exporter', NETFLOW_DIMENSION_LABELS.exporter], + ['bytes', 'Bytes'], + ['packets', 'Packets'], + ['inputInterface', NETFLOW_DIMENSION_LABELS.inputInterface], + ['outputInterface', NETFLOW_DIMENSION_LABELS.outputInterface], + ['samplingRate', 'Sampling rate'], + ['rawBytes', 'Raw bytes'], + ['rawPackets', 'Raw packets'], +] as const; + +export default function NetflowRecords({ + config, + onFilter, +}: { + config: BuilderChartConfigWithDateRange; + onFilter?: NetflowFilterHandler; +}) { + const { data, isLoading, error } = useQueriedChartConfig(config); + const formatTime = useFormatTime(); + const [selected, setSelected] = useState | null>( + null, + ); + const rows = useMemo(() => { + const repeats = new Map(); + return (data?.data ?? []).map(row => { + const values = Object.fromEntries( + fields.map(([key]) => [key, row[netflowColumnAlias(key)]]), + ); + // Flow exports can repeat a record exactly and carry no ID. + const content = JSON.stringify(values); + const repeat = (repeats.get(content) ?? 0) + 1; + repeats.set(content, repeat); + return { values, key: `${content}#${repeat}` }; + }); + }, [data]); + + return ( + <> + + + Latest {rows.length} records + {config.limit?.limit != null && ` · limit ${config.limit.limit}`} + , + ]} + > + {error ? ( + + ) : isLoading ? ( + + ) : rows.length === 0 ? ( + + ) : ( + + + + + {[ + 'Time', + 'Source', + 'Destination', + 'Protocol', + 'Exporter', + 'Bytes', + 'Packets', + 'Details', + ].map(label => ( + {label} + ))} + + + + {rows.map(({ values: row, key }, index) => ( + + {formatTime(String(row.timestamp))} + + {' '} + · {String(row.srcPort)} + + + {' '} + · {String(row.dstPort)} + + + + + + + + + {formatNumber(String(row.bytes), { + output: 'byte', + mantissa: 2, + })} + + + {formatNumber(String(row.packets), { + output: 'number', + thousandSeparated: true, + })} + + + + + + ))} + +
+ + )} + + + setSelected(null)} + position="right" + title="Flow details" + size="lg" + > + {selected && ( + + + Bytes and packets include sampling adjustment. Raw counters show + the values received from the exporter. + + + + {fields.map(([key, label]) => ( + + {label} + + {key === 'srcAddr' || + key === 'dstAddr' || + key === 'protocol' || + key === 'exporter' || + key === 'inputInterface' || + key === 'outputInterface' ? ( + + ) : key === 'timestamp' ? ( + formatTime(String(selected[key])) + ) : ( + String(selected[key] ?? '—') + )} + + + ))} + +
+ + )} + + + ); +} diff --git a/packages/app/src/components/NetflowSankey.tsx b/packages/app/src/components/NetflowSankey.tsx new file mode 100644 index 0000000000..19380188f4 --- /dev/null +++ b/packages/app/src/components/NetflowSankey.tsx @@ -0,0 +1,191 @@ +import { useMemo } from 'react'; +import { + parseAsArrayOf, + parseAsInteger, + parseAsString, + useQueryStates, +} from 'nuqs'; +import { + BuilderChartConfigWithDateRange, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; +import { Group, Loader, MultiSelect, Select, Stack, Text } from '@mantine/core'; + +import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { useColumns } from '@/hooks/useMetadata'; +import { + buildNetflowSankeyConfig, + buildNetflowSankeyData, + normalizeNetflowSankeyLimit, + SankeyDimension, + SankeyFilterHandler, +} from '@/netflowSankey'; +import { + getDefaultSankeyDimensions, + getSankeyDimensionOptions, +} from '@/netflowSankeyDimensions'; +import { useSource } from '@/source'; + +import { ChartCard } from './charts/ChartCard'; +import ChartContainer from './charts/ChartContainer'; +import ChartErrorState from './charts/ChartErrorState'; +import EmptyState from './EmptyState'; +import NetflowSankeyChart from './NetflowSankeyChart'; +import NetflowSankeyTable from './NetflowSankeyTable'; + +function SankeyResults({ + baseConfig, + dimensions, + limit, + onFilter, +}: { + baseConfig: BuilderChartConfigWithDateRange; + dimensions: SankeyDimension[]; + limit: number; + onFilter?: SankeyFilterHandler; +}) { + const config = useMemo( + () => buildNetflowSankeyConfig({ baseConfig, dimensions, limit }), + [baseConfig, dimensions, limit], + ); + const { data, isLoading, error } = useQueriedChartConfig(config); + const graph = useMemo( + () => buildNetflowSankeyData(data?.data ?? [], dimensions), + [data, dimensions], + ); + const rangeSeconds = + (baseConfig.dateRange[1].getTime() - baseConfig.dateRange[0].getTime()) / + 1000; + if (error) return ; + if (isLoading) return ; + if (!graph.paths.length) + return ( + + ); + return ( + <> + + + Showing {graph.paths.length} of the top {limit} paths by sampled bytes. + Link widths represent these displayed paths; rates are averaged over the + selected time range. + + + + ); +} + +export default function NetflowSankey({ + baseConfig, + onFilter, +}: { + baseConfig: BuilderChartConfigWithDateRange; + onFilter?: SankeyFilterHandler; +}) { + const [params, setParams] = useQueryStates({ + sankeyDimensions: parseAsArrayOf(parseAsString), + sankeyLimit: parseAsInteger.withDefault(20), + }); + const { data: source } = useSource({ + id: baseConfig.source, + kinds: [SourceKind.Netflow], + }); + const { + data: columns, + isLoading, + error, + } = useColumns({ + databaseName: baseConfig.from.databaseName, + tableName: baseConfig.from.tableName, + connectionId: baseConfig.connection ?? '', + }); + const options = useMemo( + () => (source ? getSankeyDimensionOptions(source, columns ?? []) : []), + [source, columns], + ); + const keys = params.sankeyDimensions?.every(key => + options.some(option => option.key === key), + ) + ? params.sankeyDimensions + : getDefaultSankeyDimensions(options); + const dimensions = keys.flatMap(key => + options.filter(option => option.key === key), + ); + const valid = + dimensions.length >= 2 && + dimensions.length <= 5 && + dimensions.length === keys.length && + new Set(keys).size === keys.length; + const limit = normalizeNetflowSankeyLimit(params.sankeyLimit); + return ( + + + + + ({ + value: option.key, + label: option.label, + }))} + value={keys} + onChange={sankeyDimensions => + void setParams({ sankeyDimensions }) + } + /> + + ); + }, +})); + +it('persists an optional full-text expression without changing mapped columns', async () => { + const onSubmit = jest.fn(); + function Form() { + const { control, setValue, handleSubmit } = useForm({ + defaultValues: { + kind: SourceKind.Netflow, + bytesExpression: 'Octets', + srcAddrExpression: 'ClientIP', + }, + }); + return ( +
+ + + + ); + } + renderWithMantine(
); + expect( + screen.getByText('Full-text search expression (optional)'), + ).toBeInTheDocument(); + const input = screen.getByLabelText('implicitColumnExpression'); + expect(input).toHaveValue(''); + fireEvent.change(input, { + target: { value: 'concat(ClientIP, RouterName)' }, + }); + fireEvent.click(screen.getByRole('button', { name: 'Save' })); + await waitFor(() => + expect(onSubmit).toHaveBeenCalledWith( + expect.objectContaining({ + implicitColumnExpression: 'concat(ClientIP, RouterName)', + bytesExpression: 'Octets', + srcAddrExpression: 'ClientIP', + }), + expect.anything(), + ), + ); +}); diff --git a/packages/app/src/components/__tests__/SourceForm.inference.test.tsx b/packages/app/src/components/__tests__/SourceForm.inference.test.tsx new file mode 100644 index 0000000000..a2b114fb9f --- /dev/null +++ b/packages/app/src/components/__tests__/SourceForm.inference.test.tsx @@ -0,0 +1,327 @@ +import { Control, Controller, FieldPath } from 'react-hook-form'; +import { + SourceKind, + SourceSchema, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { MantineProvider } from '@mantine/core'; +import { notifications } from '@mantine/notifications'; +import { + act, + fireEvent, + render, + screen, + waitFor, +} from '@testing-library/react'; + +import { TableSourceForm } from '@/components/Sources/SourceForm/SourceForm'; +import { inferTableSourceConfig } from '@/source'; + +const mockConnections = [{ id: 'local', name: 'Local ClickHouse' }]; +let mockMetadata = {}; +let mockSource: TSource | undefined; +jest.mock('@/connection', () => ({ + useConnections: () => ({ data: mockConnections }), +})); +jest.mock('@/hooks/useMetadata', () => ({ + useMetadataWithSettings: () => mockMetadata, +})); +jest.mock('@/source', () => ({ + inferTableSourceConfig: jest.fn(), + useSource: () => ({ data: mockSource }), + useSources: () => ({ data: [] }), + useCreateSource: () => ({ mutate: jest.fn() }), + useUpdateSource: () => ({ mutate: jest.fn(), mutateAsync: jest.fn() }), + useDeleteSource: () => ({ mutate: jest.fn() }), +})); +jest.mock('@mantine/notifications', () => ({ + notifications: { show: jest.fn() }, +})); + +function MockInput({ + control, + name, +}: { + control: Control; + name: FieldPath; +}) { + return ( + ( + + )} + /> + ); +} +jest.mock('@/components/ConnectionSelect', () => ({ + ConnectionSelectControlled: MockInput, +})); +jest.mock('@/components/DatabaseSelect', () => ({ + DatabaseSelectControlled: MockInput, +})); +jest.mock('@/components/DBTableSelect', () => ({ + DBTableSelectControlled: MockInput, +})); +jest.mock('@/components/InputControlled', () => ({ + InputControlled: MockInput, + AutocompleteControlled: MockInput, +})); +jest.mock('@/components/ConfirmDeleteMenu', () => () => null); +jest.mock('@/components/Sources/SourceForm/TableModelForm', () => ({ + TableModelForm: ({ control }: { control: Control }) => ( + <> + + + + ), +})); + +const mockInfer = jest.mocked(inferTableSourceConfig); +const deferred = () => + Promise.withResolvers>>(); + +describe('Source form asynchronous inference', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockSource = undefined; + mockMetadata = {}; + }); + + it.each([SourceKind.Log, SourceKind.Trace, SourceKind.Netflow])( + 'preserves saved %s mappings on replica changes and infers a newly selected table or kind', + async kind => { + mockSource = SourceSchema.parse({ + id: 'saved', + name: 'Saved source', + kind, + connection: 'local', + from: { databaseName: 'default', tableName: 'events' }, + timestampValueExpression: 'CustomTime', + defaultTableSelectExpression: '*', + bytesExpression: 'CustomBytes', + packetsExpression: 'Packets', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + durationExpression: 'Duration', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + }); + mockInfer.mockResolvedValue({ + kind, + timestampValueExpression: 'Detected', + }); + const view = render(, { + wrapper: MantineProvider, + }); + for (const field of ['connection', 'from.databaseName', 'metadata']) { + await act(async () => { + if (field === 'metadata') { + mockMetadata = {}; + view.rerender(); + } else { + fireEvent.change(screen.getByLabelText(field), { + target: { value: 'replica' }, + }); + } + }); + expect(mockInfer).not.toHaveBeenCalled(); + expect(screen.getByLabelText('timestampValueExpression')).toHaveValue( + 'CustomTime', + ); + } + fireEvent.change(screen.getByLabelText('from.tableName'), { + target: { value: 'new_events' }, + }); + await waitFor(() => expect(mockInfer).toHaveBeenCalledTimes(1)); + await waitFor(() => + expect(screen.getByLabelText('timestampValueExpression')).toHaveValue( + 'Detected', + ), + ); + fireEvent.click( + screen.getByRole('radio', { + name: kind === SourceKind.Netflow ? 'Logs' : 'NetFlow', + }), + ); + await waitFor(() => expect(mockInfer).toHaveBeenCalledTimes(2)); + }, + ); + + it('ignores a NetFlow response that arrives after the user selects Logs', async () => { + const oldRequest = deferred(); + const latestRequest = deferred(); + mockInfer + .mockReturnValueOnce(oldRequest.promise) + .mockReturnValueOnce(latestRequest.promise); + renderWithMantine( + , + ); + fireEvent.change(screen.getByLabelText('from.tableName'), { + target: { value: 'flows' }, + }); + await waitFor(() => expect(mockInfer).toHaveBeenCalledTimes(1)); + fireEvent.click(screen.getByRole('radio', { name: 'Logs' })); + await waitFor(() => expect(mockInfer).toHaveBeenCalledTimes(2)); + await act(async () => + latestRequest.resolve({ + kind: SourceKind.Log, + timestampValueExpression: 'LogTime', + }), + ); + await act(async () => + oldRequest.resolve({ + kind: SourceKind.Netflow, + timestampValueExpression: 'FlowTime', + }), + ); + expect(screen.getByRole('radio', { name: 'Logs' })).toBeChecked(); + expect(screen.getByLabelText('timestampValueExpression')).toHaveValue( + 'LogTime', + ); + expect(notifications.show).toHaveBeenCalledTimes(1); + }); + + it('keeps the latest table inference and a mapping edited while it loads', async () => { + const oldRequest = deferred(); + const latestRequest = deferred(); + mockInfer + .mockReturnValueOnce(oldRequest.promise) + .mockReturnValueOnce(latestRequest.promise); + renderWithMantine( + , + ); + fireEvent.change(screen.getByLabelText('from.tableName'), { + target: { value: 'old_flows' }, + }); + await waitFor(() => expect(mockInfer).toHaveBeenCalledTimes(1)); + fireEvent.change(screen.getByLabelText('from.tableName'), { + target: { value: 'new_flows' }, + }); + await waitFor(() => expect(mockInfer).toHaveBeenCalledTimes(2)); + fireEvent.change(screen.getByLabelText('bytesExpression'), { + target: { value: 'CustomBytes * 2' }, + }); + await act(async () => + latestRequest.resolve({ + kind: SourceKind.Netflow, + timestampValueExpression: 'NewTime', + bytesExpression: 'NewBytes', + }), + ); + await act(async () => + oldRequest.resolve({ + kind: SourceKind.Netflow, + timestampValueExpression: 'OldTime', + bytesExpression: 'OldBytes', + }), + ); + expect(screen.getByLabelText('timestampValueExpression')).toHaveValue( + 'NewTime', + ); + expect(screen.getByLabelText('bytesExpression')).toHaveValue( + 'CustomBytes * 2', + ); + expect(notifications.show).toHaveBeenCalledTimes(1); + }); + + it('clears an inferred mapping when the new table lacks the column', async () => { + mockInfer + .mockResolvedValueOnce({ + kind: SourceKind.Netflow, + bytesExpression: 'Bytes', + }) + .mockResolvedValueOnce({ + kind: SourceKind.Netflow, + bytesExpression: '', + }); + renderWithMantine( + , + ); + fireEvent.change(screen.getByLabelText('from.tableName'), { + target: { value: 'flows_with_bytes' }, + }); + await waitFor(() => + expect(screen.getByLabelText('bytesExpression')).toHaveValue('Bytes'), + ); + fireEvent.change(screen.getByLabelText('from.tableName'), { + target: { value: 'flows_without_bytes' }, + }); + await waitFor(() => + expect(screen.getByLabelText('bytesExpression')).toHaveValue(''), + ); + }); + + it.each([ + [ + SourceKind.Netflow, + 'from.databaseName', + 'other_database', + { databaseName: 'other_database' }, + ], + [ + SourceKind.Netflow, + 'connection', + 'other_connection', + { connectionId: 'other_connection' }, + ], + [ + SourceKind.Log, + 'connection', + 'other_connection', + { connectionId: 'other_connection' }, + ], + [ + SourceKind.Trace, + 'from.databaseName', + 'other_database', + { databaseName: 'other_database' }, + ], + ] as const)( + 'restarts new %s discovery for the same table after %s changes', + async (kind, field, value, expected) => { + const oldRequest = deferred(); + const latestRequest = deferred(); + mockInfer + .mockReturnValueOnce(oldRequest.promise) + .mockReturnValueOnce(latestRequest.promise); + renderWithMantine(); + fireEvent.change(screen.getByLabelText('from.tableName'), { + target: { value: 'flows' }, + }); + await waitFor(() => expect(mockInfer).toHaveBeenCalledTimes(1)); + fireEvent.change(screen.getByLabelText(field), { target: { value } }); + await waitFor(() => expect(mockInfer).toHaveBeenCalledTimes(2)); + expect(mockInfer).toHaveBeenLastCalledWith( + expect.objectContaining({ tableName: 'flows', ...expected }), + ); + await act(async () => + latestRequest.resolve({ + kind: SourceKind.Netflow, + timestampValueExpression: 'NewTime', + }), + ); + await act(async () => + oldRequest.resolve({ + kind: SourceKind.Netflow, + timestampValueExpression: 'OldTime', + }), + ); + expect(screen.getByLabelText('timestampValueExpression')).toHaveValue( + 'NewTime', + ); + expect(notifications.show).toHaveBeenCalledTimes(1); + }, + ); +}); diff --git a/packages/app/src/components/alerts/AlertDetailChart.tsx b/packages/app/src/components/alerts/AlertDetailChart.tsx index 60608d154e..af215d0963 100644 --- a/packages/app/src/components/alerts/AlertDetailChart.tsx +++ b/packages/app/src/components/alerts/AlertDetailChart.tsx @@ -1,6 +1,7 @@ import * as React from 'react'; import Link from 'next/link'; import { pick } from 'lodash'; +import { getSourceImplicitColumnExpression } from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { Granularity, isTimeSeriesDisplayType, @@ -167,12 +168,12 @@ export function buildAlertChartConfig({ ...savedConfig, variables, ...pick(source, [ - 'implicitColumnExpression', 'useTextIndexForImplicitColumn', 'from', 'metricTables', ]), ...(isLogSource(source) ? { bodyExpression: source.bodyExpression } : {}), + implicitColumnExpression: getSourceImplicitColumnExpression(source), sampleWeightExpression: getSampleWeightExpression(source), dateRange, granularity, @@ -207,10 +208,7 @@ export function buildAlertChartConfig({ databaseName: source.from?.databaseName || 'default', tableName: tableName || '', }, - implicitColumnExpression: - isLogSource(source) || isTraceSource(source) - ? source.implicitColumnExpression - : undefined, + implicitColumnExpression: getSourceImplicitColumnExpression(source), useTextIndexForImplicitColumn: isLogSource(source) || isTraceSource(source) ? source.useTextIndexForImplicitColumn diff --git a/packages/app/src/components/sourceSelectUtils.tsx b/packages/app/src/components/sourceSelectUtils.tsx index b0d34a6934..9c7af65cbb 100644 --- a/packages/app/src/components/sourceSelectUtils.tsx +++ b/packages/app/src/components/sourceSelectUtils.tsx @@ -6,9 +6,11 @@ import { IconConnection, IconDeviceLaptop, IconLogs, + IconNetwork, } from '@tabler/icons-react'; export const SOURCE_KIND_ICONS: Record = { + [SourceKind.Netflow]: , [SourceKind.Log]: , [SourceKind.Trace]: , [SourceKind.Session]: , diff --git a/packages/app/src/hooks/__tests__/useQueriedDashboardFilterValues.test.tsx b/packages/app/src/hooks/__tests__/useQueriedDashboardFilterValues.test.tsx index ba40f07b2e..d983997606 100644 --- a/packages/app/src/hooks/__tests__/useQueriedDashboardFilterValues.test.tsx +++ b/packages/app/src/hooks/__tests__/useQueriedDashboardFilterValues.test.tsx @@ -10,9 +10,14 @@ import { MetricsDataType, QueryExpressionDashboardFilter, SourceKind, + TNetflowSource, TSource, } from '@hyperdx/common-utils/dist/types'; -import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { + QueryClient, + QueryClientProvider, + useQuery, +} from '@tanstack/react-query'; import { renderHook, waitFor } from '@testing-library/react'; import * as useMetadataModule from '@/hooks/useMetadata'; @@ -162,6 +167,64 @@ describe('useQueriedDashboardFilterValues', () => { jest.restoreAllMocks(); }); + it('uses mapped NetFlow fields for bare Lucene terms in filter-value queries', async () => { + const source: TNetflowSource = { + id: 'flows', + name: 'Flows', + kind: SourceKind.Netflow, + connection: 'clickhouse-conn', + from: { databaseName: 'telemetry', tableName: 'flows' }, + timestampValueExpression: 'timestamp', + defaultTableSelectExpression: '*', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + srcAddrExpression: 'ClientIP', + dstAddrExpression: 'ServerIP', + srcPortExpression: 'ClientPort', + dstPortExpression: 'ServerPort', + protocolExpression: 'Protocol', + exporterExpression: 'RouterName', + }; + jest + .spyOn(sourceModule, 'useSources') + .mockImplementation(function useMockSources() { + return useQuery({ + queryKey: ['netflow-test-sources'], + queryFn: async () => [source], + initialData: [source], + }); + }); + const { result } = renderHook( + () => + useQueriedDashboardFilterValues({ + filters: [ + { + id: 'router', + type: 'QUERY_EXPRESSION', + name: 'Router', + expression: 'RouterName', + source: 'flows', + where: 'edge', + whereLanguage: 'lucene', + }, + ], + dateRange: mockDateRange, + }), + { wrapper }, + ); + await waitFor(() => expect(result.current.isFetching).toBe(false)); + expect(mockMetadata.getKeyValues).toHaveBeenCalledWith( + expect.objectContaining({ + chartConfig: expect.objectContaining({ + where: 'edge', + whereLanguage: 'lucene', + implicitColumnExpression: + expect.stringContaining('toString(ClientIP)'), + }), + }), + ); + }); + it('should convert non-string key values to strings', async () => { // Arrange jest.spyOn(sourceModule, 'useSources').mockReturnValue({ diff --git a/packages/app/src/hooks/useQueriedDashboardFilterValues.tsx b/packages/app/src/hooks/useQueriedDashboardFilterValues.tsx index 9023b91ac7..1efe4f93f9 100644 --- a/packages/app/src/hooks/useQueriedDashboardFilterValues.tsx +++ b/packages/app/src/hooks/useQueriedDashboardFilterValues.tsx @@ -5,6 +5,7 @@ import { optimizeFacetedKeyValuesConfig, optimizeGetKeyValuesCalls, } from '@hyperdx/common-utils/dist/core/materializedViews'; +import { getSourceImplicitColumnExpression } from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { FilterSelection } from '@hyperdx/common-utils/dist/dashboardFilterValues'; import { FilterState, @@ -221,10 +222,7 @@ function useOptimizedKeyValuesCalls({ databaseName: source.from.databaseName, tableName, }, - implicitColumnExpression: - isTraceSource(source) || isLogSource(source) - ? source.implicitColumnExpression - : undefined, + implicitColumnExpression: getSourceImplicitColumnExpression(source), // Logs-only body fallback for bare-text Lucene search. bodyExpression: isLogSource(source) ? source.bodyExpression diff --git a/packages/app/src/hooks/useReleaseAnnotations.tsx b/packages/app/src/hooks/useReleaseAnnotations.tsx index 839b3faeb7..d8a7f068ee 100644 --- a/packages/app/src/hooks/useReleaseAnnotations.tsx +++ b/packages/app/src/hooks/useReleaseAnnotations.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useRef } from 'react'; import Link from 'next/link'; import { isMissingColumnError } from '@hyperdx/common-utils/dist/clickhouse'; +import { getSourceImplicitColumnExpression } from '@hyperdx/common-utils/dist/core/searchChartConfig'; import { BuilderChartConfigWithDateRange, Filter, @@ -227,10 +228,7 @@ export function buildReleaseChartConfig( from: source.from, timestampValueExpression: source.timestampValueExpression, // Needed for Lucene scope filters to resolve bare terms. - implicitColumnExpression: - 'implicitColumnExpression' in source - ? source.implicitColumnExpression - : undefined, + implicitColumnExpression: getSourceImplicitColumnExpression(source), useTextIndexForImplicitColumn: 'useTextIndexForImplicitColumn' in source ? source.useTextIndexForImplicitColumn diff --git a/packages/app/src/netflow.ts b/packages/app/src/netflow.ts new file mode 100644 index 0000000000..16150ba6c9 --- /dev/null +++ b/packages/app/src/netflow.ts @@ -0,0 +1,293 @@ +import { FIXED_TIME_BUCKET_EXPR_ALIAS } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { + getSourceImplicitColumnExpression, + NETFLOW_PROTOCOL_NAMES, + netflowProtocolNameExpression, +} from '@hyperdx/common-utils/dist/core/searchChartConfig'; +import { + convertDateRangeToGranularityString, + convertGranularityToSeconds, + escapeSqlString, + getFirstTimestampValueExpression, +} from '@hyperdx/common-utils/dist/core/utils'; +import { + BuilderChartConfigWithDateRange, + DisplayType, + Filter, + NumberFormat, + NumericUnit, + SearchConditionLanguage, + TNetflowSource, +} from '@hyperdx/common-utils/dist/types'; + +export const NETFLOW_DIMENSION_LABELS: Record = { + srcAddr: 'Source IP', + dstAddr: 'Destination IP', + protocol: 'Protocol', + exporter: 'Exporter', + inputInterface: 'Input interface', + outputInterface: 'Output interface', +} satisfies Record; +export type NetflowFilterField = keyof ReturnType; +export const netflowColumnAlias = (field: string) => `__netflow_${field}`; +export const NETFLOW_ALIASES = { + name: netflowColumnAlias('name'), + bytes: netflowColumnAlias('bytes'), + value: netflowColumnAlias('value'), +}; +export const NETFLOW_SUMMARY_TILES = [ + { + title: 'Average bit rate', + column: netflowColumnAlias('bitsPerSecond'), + numberFormat: { + output: 'data_rate', + numericUnit: NumericUnit.BitsSecSI, + mantissa: 2, + }, + }, + { + title: 'Average packet rate', + column: netflowColumnAlias('packetsPerSecond'), + numberFormat: { + output: 'throughput', + numericUnit: NumericUnit.PacketsSec, + mantissa: 2, + }, + }, + { + title: 'Transferred bytes', + column: NETFLOW_ALIASES.bytes, + numberFormat: { output: 'byte', mantissa: 2 }, + }, + { + title: 'Flow records', + column: netflowColumnAlias('flowRecords'), + numberFormat: { output: 'number', thousandSeparated: true }, + }, +] satisfies { title: string; column: string; numberFormat: NumberFormat }[]; + +export type NetflowFilters = { + exporter?: string; + protocol?: string; + srcAddr?: string; + dstAddr?: string; +}; + +export function getNetflowDimensions(source: TNetflowSource) { + const address = (expression: string) => + `replaceRegexpOne(toString(${expression}), '^::ffff:', '')`; + return { + srcAddr: address(source.srcAddrExpression), + dstAddr: address(source.dstAddrExpression), + protocol: netflowProtocolNameExpression(source.protocolExpression), + exporter: source.exporterExpression?.trim(), + inputInterface: source.inIfExpression?.trim(), + outputInterface: source.outIfExpression?.trim(), + }; +} + +// The protocol filter also accepts the names the charts show, e.g. TCP. +const protocolNumber = (value?: string) => + Object.entries(NETFLOW_PROTOCOL_NAMES).find( + ([, name]) => name.toLowerCase() === value?.trim().toLowerCase(), + )?.[0] ?? value; + +export function buildNetflowWhere( + source: TNetflowSource, + filters: NetflowFilters, +) { + const predicates: string[] = []; + for (const [value, expression, address] of [ + [filters.exporter, source.exporterExpression, false], + [protocolNumber(filters.protocol), source.protocolExpression, false], + [filters.srcAddr, source.srcAddrExpression, true], + [filters.dstAddr, source.dstAddrExpression, true], + ] as const) { + if (!value?.trim() || !expression?.trim()) continue; + const literal = `'${escapeSqlString(value.trim())}'`; + predicates.push( + address + ? `toIPv6OrNull(toString(${expression})) = toIPv6OrNull(${literal})` + : `toString(${expression}) = ${literal}`, + ); + } + return predicates.map(predicate => `(${predicate})`).join(' AND '); +} + +export function buildNetflowQueryConfigs({ + source, + dateRange, + filters, + where = '', + whereLanguage = 'lucene', + extraFilters = [], +}: { + source: TNetflowSource; + dateRange: [Date, Date]; + filters: NetflowFilters; + where?: string; + whereLanguage?: SearchConditionLanguage; + extraFilters?: Filter[]; +}) { + const seconds = (dateRange[1].getTime() - dateRange[0].getTime()) / 1000; + if (!Number.isFinite(seconds) || seconds <= 0) + throw new Error('Time range must have a start before its end'); + + const samplingRate = source.samplingRateExpression?.trim(); + const sample = samplingRate + ? `if((${samplingRate}) > 0, (${samplingRate}), 1)` + : '1'; + const exporter = source.exporterExpression?.trim(); + const inputInterface = source.inIfExpression?.trim(); + const outputInterface = source.outIfExpression?.trim(); + // Akvorado stores raw counters: scale once, before summing. Float64 avoids UInt64 multiplication overflow. + const bytes = `toFloat64(${source.bytesExpression}) * (${sample})`; + const packets = `toFloat64(${source.packetsExpression}) * (${sample})`; + const granularity = convertDateRangeToGranularityString(dateRange); + const bucketSeconds = convertGranularityToSeconds(granularity); + const timestamp = getFirstTimestampValueExpression( + source.timestampValueExpression, + ); + const bucketStart = `toUnixTimestamp64Milli(toDateTime64(\`${FIXED_TIME_BUCKET_EXPR_ALIAS}\`, 3)) / 1000`; + // Edge buckets cover only the intersection with the selected range. + const coveredSeconds = `least(${bucketStart} + ${bucketSeconds}, ${dateRange[1].getTime() / 1000}) - greatest(${bucketStart}, ${dateRange[0].getTime() / 1000})`; + const dimensions = getNetflowDimensions(source); + const protocol = dimensions.protocol; + const base = { + from: source.from, + connection: source.connection, + source: source.id, + timestampValueExpression: source.timestampValueExpression, + implicitColumnExpression: getSourceImplicitColumnExpression(source), + dateRange, + dateRangeEndInclusive: false, + alignDateRangeToGranularity: false, + where, + whereLanguage, + filters: [ + ...extraFilters, + { type: 'sql' as const, condition: buildNetflowWhere(source, filters) }, + ], + }; + // Keep result aliases separate from common mapped column names: ClickHouse substitutes aliases in WHERE and sibling expressions. + const byteFormat: NumberFormat = { output: 'byte', mantissa: 2 }; + const bitRateFormat: NumberFormat = { + output: 'data_rate', + numericUnit: NumericUnit.BitsSecSI, + mantissa: 2, + }; + const top = (expression: string): BuilderChartConfigWithDateRange => ({ + ...base, + select: [ + { valueExpression: expression, alias: netflowColumnAlias('name') }, + { valueExpression: `sum(${bytes})`, alias: netflowColumnAlias('bytes') }, + ], + groupBy: NETFLOW_ALIASES.name, + orderBy: `${NETFLOW_ALIASES.bytes} DESC`, + limit: { limit: 10 }, + displayType: DisplayType.Bar, + numberFormat: byteFormat, + }); + const traffic: BuilderChartConfigWithDateRange = { + ...base, + select: [ + { + valueExpression: `sum(${bytes}) * 8 / (${coveredSeconds})`, + alias: 'Bits per second', + }, + ], + granularity, + displayType: DisplayType.Line, + numberFormat: bitRateFormat, + }; + const flows: BuilderChartConfigWithDateRange = { + ...base, + select: [ + { + valueExpression: timestamp, + alias: netflowColumnAlias('timestamp'), + }, + { + valueExpression: dimensions.srcAddr, + alias: netflowColumnAlias('srcAddr'), + }, + { + valueExpression: dimensions.dstAddr, + alias: netflowColumnAlias('dstAddr'), + }, + { + valueExpression: source.srcPortExpression, + alias: netflowColumnAlias('srcPort'), + }, + { + valueExpression: source.dstPortExpression, + alias: netflowColumnAlias('dstPort'), + }, + { valueExpression: protocol, alias: netflowColumnAlias('protocol') }, + { + valueExpression: exporter || "''", + alias: netflowColumnAlias('exporter'), + }, + { valueExpression: bytes, alias: netflowColumnAlias('bytes') }, + { valueExpression: packets, alias: netflowColumnAlias('packets') }, + { + valueExpression: source.bytesExpression, + alias: netflowColumnAlias('rawBytes'), + }, + { + valueExpression: source.packetsExpression, + alias: netflowColumnAlias('rawPackets'), + }, + { valueExpression: sample, alias: netflowColumnAlias('samplingRate') }, + { + valueExpression: inputInterface || "''", + alias: netflowColumnAlias('inputInterface'), + }, + { + valueExpression: outputInterface || "''", + alias: netflowColumnAlias('outputInterface'), + }, + ], + orderBy: `${timestamp} DESC`, + limit: { limit: 500 }, + displayType: DisplayType.Table, + }; + + return { + summary: { + ...base, + select: [ + { + valueExpression: `sum(${bytes}) * 8 / ${seconds}`, + alias: netflowColumnAlias('bitsPerSecond'), + }, + { + valueExpression: `sum(${packets}) / ${seconds}`, + alias: netflowColumnAlias('packetsPerSecond'), + }, + { valueExpression: `sum(${bytes})`, alias: NETFLOW_ALIASES.bytes }, + { + valueExpression: 'count()', + alias: netflowColumnAlias('flowRecords'), + }, + ], + displayType: DisplayType.Number, + } satisfies BuilderChartConfigWithDateRange, + totalBytes: { + ...base, + select: [ + { valueExpression: `sum(${bytes})`, alias: NETFLOW_ALIASES.value }, + ], + displayType: DisplayType.Number, + numberFormat: byteFormat, + } satisfies BuilderChartConfigWithDateRange, + traffic, + topSourceAddresses: top(dimensions.srcAddr), + topDestinationAddresses: top(dimensions.dstAddr), + protocols: top(protocol), + exporters: exporter ? top(exporter) : undefined, + inInterfaces: inputInterface ? top(inputInterface) : undefined, + outInterfaces: outputInterface ? top(outputInterface) : undefined, + flows, + }; +} diff --git a/packages/app/src/netflowSankey.ts b/packages/app/src/netflowSankey.ts new file mode 100644 index 0000000000..6a7ebeeca6 --- /dev/null +++ b/packages/app/src/netflowSankey.ts @@ -0,0 +1,143 @@ +import { + BuilderChartConfigWithDateRange, + DisplayType, +} from '@hyperdx/common-utils/dist/types'; + +import { NETFLOW_ALIASES, netflowColumnAlias } from '@/netflow'; + +export interface SankeyDimension { + key: string; + label: string; + expression: string; +} + +export type SankeyFilterHandler = ( + dimension: SankeyDimension, + value: string, + excluded: boolean, +) => void; + +export interface NetflowSankeyNode { + id: string; + name: string; + rawValue: string; + stage: number; + dimension: SankeyDimension; +} + +interface NetflowSankeyLink { + source: number; + target: number; + value: number; +} + +export interface NetflowSankeyData { + nodes: NetflowSankeyNode[]; + links: NetflowSankeyLink[]; + paths: { values: string[]; value: number }[]; +} + +export function sankeyDimensionExpression(dimension: SankeyDimension): string { + return `ifNull(toString(${dimension.expression}), '')`; +} + +export function unwrapSankeyDimensionExpression(expression: string): string { + return expression.replace(/^ifNull\(toString\((.*)\), ''\)$/s, '$1'); +} + +export function normalizeNetflowSankeyLimit(limit: number): number { + return [10, 20, 50].includes(limit) ? limit : 20; +} + +export function buildNetflowSankeyConfig({ + baseConfig, + dimensions, + limit = 20, +}: { + baseConfig: BuilderChartConfigWithDateRange; + dimensions: SankeyDimension[]; + limit?: number; +}): BuilderChartConfigWithDateRange { + if (dimensions.length < 2 || dimensions.some(d => !d.expression.trim())) { + throw new Error('Choose at least two dimensions with valid expressions'); + } + if (!Array.isArray(baseConfig.select) || baseConfig.select.length !== 1) { + throw new Error('A Sankey query requires one sampled byte aggregate'); + } + + const dimensionSelect = dimensions.map((dimension, stage) => ({ + valueExpression: sankeyDimensionExpression(dimension), + alias: netflowColumnAlias(`dimension_${stage}`), + })); + + return { + ...baseConfig, + select: [ + ...dimensionSelect, + { ...baseConfig.select[0], alias: NETFLOW_ALIASES.value }, + ], + groupBy: dimensionSelect.map(dimension => dimension.alias).join(', '), + orderBy: `${NETFLOW_ALIASES.value} DESC`, + limit: { limit: normalizeNetflowSankeyLimit(limit) }, + displayType: DisplayType.Table, + }; +} + +export function buildNetflowSankeyData( + rows: Record[], + dimensions: SankeyDimension[], +): NetflowSankeyData { + const data: NetflowSankeyData = { + nodes: [], + links: [], + paths: [], + }; + if (dimensions.length < 2) return data; + + const nodeIndices = new Map(); + const links = new Map(); + for (const row of rows) { + const rawBytes = row[NETFLOW_ALIASES.value]; + if (typeof rawBytes !== 'number' && typeof rawBytes !== 'string') continue; + const value = Number(rawBytes); + if (!Number.isFinite(value) || value <= 0) continue; + if ( + dimensions.some( + (_, stage) => + !Object.hasOwn(row, netflowColumnAlias(`dimension_${stage}`)), + ) + ) + continue; + + const values = dimensions.map((_, stage) => + String(row[netflowColumnAlias(`dimension_${stage}`)] ?? ''), + ); + const pathNodes = values.map((rawValue, stage) => { + // Stage-qualified IDs keep repeated values in different dimensions acyclic. + const id = JSON.stringify([stage, rawValue]); + const existing = nodeIndices.get(id); + if (existing !== undefined) return existing; + const index = data.nodes.length; + nodeIndices.set(id, index); + data.nodes.push({ + id, + name: rawValue || '(empty)', + rawValue, + stage, + dimension: dimensions[stage], + }); + return index; + }); + for (let stage = 1; stage < pathNodes.length; stage++) { + const source = pathNodes[stage - 1]; + const target = pathNodes[stage]; + const key = `${source}:${target}`; + const link = links.get(key); + if (link) link.value += value; + else links.set(key, { source, target, value }); + } + data.paths.push({ values, value }); + } + data.links = [...links.values()]; + return data; +} diff --git a/packages/app/src/netflowSankeyDimensions.ts b/packages/app/src/netflowSankeyDimensions.ts new file mode 100644 index 0000000000..8d3afcd940 --- /dev/null +++ b/packages/app/src/netflowSankeyDimensions.ts @@ -0,0 +1,61 @@ +import { + ColumnMeta, + convertCHDataTypeToJSType, + JSDataType, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { quoteIdentifierIfNeeded } from '@hyperdx/common-utils/dist/core/metadata'; +import { TNetflowSource } from '@hyperdx/common-utils/dist/types'; + +import { getNetflowDimensions, NETFLOW_DIMENSION_LABELS } from '@/netflow'; +import { SankeyDimension } from '@/netflowSankey'; + +export function getSankeyDimensionOptions( + source: TNetflowSource, + columns: ColumnMeta[], +): SankeyDimension[] { + const mapped = Object.entries(getNetflowDimensions(source)).flatMap( + ([key, expression]) => + expression + ? [ + { + key, + label: NETFLOW_DIMENSION_LABELS[key], + expression, + }, + ] + : [], + ); + const scalarTypes = [ + JSDataType.String, + JSDataType.Number, + JSDataType.Bool, + JSDataType.Date, + ]; + return [ + ...mapped, + ...columns + .filter(column => { + const type = convertCHDataTypeToJSType(column.type); + return type !== null && scalarTypes.includes(type); + }) + .map(column => ({ + key: `column:${column.name}`, + label: column.name, + expression: quoteIdentifierIfNeeded(column.name), + })), + ]; +} + +export function getDefaultSankeyDimensions( + options: SankeyDimension[], +): string[] { + const akvorado = [ + 'column:SrcAS', + 'column:InIfConnectivity', + 'column:InIfProvider', + 'exporter', + ]; + return akvorado.every(key => options.some(option => option.key === key)) + ? akvorado + : ['srcAddr', 'protocol', 'dstAddr']; +} diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index 2c1b781860..6f0337153c 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -22,11 +22,13 @@ import { ChartConfigWithOptTimestamp, MetricFormula, MetricsDataType, + NETFLOW_COLUMN_EXPRESSIONS, NumberFormat, SourceKind, SourceSchema, TLogSource, TMetricSource, + TNetflowSource, TPromqlSource, TSessionSource, TSource, @@ -313,6 +315,7 @@ type TStrippedSource = Partial< Omit > & { kind: T['kind'] }; type InferredSourceConfig = + | TStrippedSource | TStrippedSource | TStrippedSource | TStrippedSource @@ -361,6 +364,35 @@ export async function inferTableSourceConfig({ kind, }; + if (kind === SourceKind.Netflow) { + const names = new Set(columns.map(column => column.name)); + const selected = [ + 'TimeReceived', + 'SrcAddr', + 'DstAddr', + 'SrcPort', + 'DstPort', + 'Proto', + 'Bytes', + 'Packets', + ].filter(name => names.has(name)); + return { + ...baseConfig, + kind: SourceKind.Netflow, + ...(names.has('TimeReceived') + ? { timestampValueExpression: 'TimeReceived' } + : {}), + defaultTableSelectExpression: selected.join(', ') || '*', + // A mapping for a column the table lacks is returned empty so that switching tables clears it. + ...Object.fromEntries( + Object.entries(NETFLOW_COLUMN_EXPRESSIONS).map(([field, column]) => [ + field, + names.has(column) ? column : '', + ]), + ), + }; + } + if (kind === SourceKind.Promql) { return baseConfig as TStrippedSource; } diff --git a/packages/app/tests/e2e/features/netflow/monitoring.spec.ts b/packages/app/tests/e2e/features/netflow/monitoring.spec.ts new file mode 100644 index 0000000000..fd132c2af3 --- /dev/null +++ b/packages/app/tests/e2e/features/netflow/monitoring.spec.ts @@ -0,0 +1,243 @@ +import { expect, test } from '../../fixtures/netflow'; +import { expectFiltersParam } from '../../utils/filters-param'; + +test.use({ viewport: { width: 1600, height: 1050 } }); + +test('NetFlow URL range filters remain visible and removable after reload', async ({ + page, + netflow, +}) => { + const params = new URLSearchParams({ + source: netflow.source.id, + filters: encodeURIComponent( + JSON.stringify([ + { type: 'sql', condition: 'Bytes BETWEEN 0 AND 2000000' }, + ]), + ), + }); + await page.goto(`/netflow?${params}`); + const rows = page.getByTestId('netflow-records').locator('tbody tr'); + await expect(rows).toHaveCount(300); + await page.reload(); + const range = page.getByTestId('active-filter-pill-Bytes'); + await expect(range).toContainText('0 – 2000000'); + await range.getByRole('button', { name: 'Remove filter' }).click(); + await expect(range).toHaveCount(0); + await expectFiltersParam(page, []); + await expect(rows).toHaveCount(500); +}); + +test('NetFlow searches, drafts, click filters and sampled details survive reload', async ({ + page, + netflow, +}) => { + await page.goto(`/netflow?source=${netflow.source.id}`); + const records = page.getByTestId('netflow-records'); + const pills = page.getByTestId('netflow-filter-pills'); + const search = page.getByTestId('netflow-search'); + const protocol = page.getByLabel('Protocol', { exact: true }); + await expect(records.locator('tbody tr')).toHaveCount(500); + await search.fill('Proto:17'); + await protocol.fill('17'); + await page + .getByTestId('netflow-breakdown-protocol') + .getByRole('button', { name: 'Filter Protocol: TCP', exact: true }) + .click(); + await page.getByRole('menuitem', { name: 'Include', exact: true }).click(); + await expect(pills).toContainText('Protocol = TCP'); + await expect + .poll(() => new URL(page.url()).searchParams.get('where') || '') + .toBe(''); + await expect + .poll(() => new URL(page.url()).searchParams.get('protocol') || '') + .toBe(''); + await expect(search).toHaveValue('Proto:17'); + await expect(protocol).toHaveValue('17'); + await expect( + records.getByRole('button', { name: 'Filter Protocol: UDP', exact: true }), + ).toHaveCount(0); + await page.getByRole('button', { name: 'Run', exact: true }).click(); + await expect(records.getByText('No flow records found')).toBeVisible(); + await expect + .poll(() => new URL(page.url()).searchParams.get('where')) + .toBe('Proto:17'); + await page + .getByRole('button', { name: 'Clear filters', exact: true }) + .click(); + await search.fill('Proto:6 AND DstPort:443'); + await page.getByRole('button', { name: 'Run', exact: true }).click(); + await expect(records.locator('tbody tr')).toHaveCount(500); + await page.reload(); + await expect(search).toHaveValue('Proto:6 AND DstPort:443'); + await expect( + records.getByRole('button', { name: 'Filter Protocol: UDP', exact: true }), + ).toHaveCount(0); + await page.getByLabel('Source IP', { exact: true }).fill('192.0.2.2'); + await page.getByRole('button', { name: 'Run', exact: true }).click(); + await expect(records.locator('tbody tr')).toHaveCount(300); + await page + .getByRole('button', { name: 'Inspect flow 1', exact: true }) + .click(); + const drawer = page.getByRole('dialog', { name: 'Flow details' }); + const field = async (label: string) => + Number( + await drawer + .getByRole('row') + .filter({ + has: page.getByRole('rowheader', { name: label, exact: true }), + }) + .getByRole('cell') + .innerText(), + ); + expect(await field('Bytes')).toBe( + (await field('Raw bytes')) * (await field('Sampling rate')), + ); + expect(await field('Packets')).toBe( + (await field('Raw packets')) * (await field('Sampling rate')), + ); + await drawer + .getByRole('button', { name: 'Filter Exporter: edge-a', exact: true }) + .click(); + await page.getByRole('menuitem', { name: 'Exclude', exact: true }).click(); + await expect(drawer).toBeHidden(); + await expect(records.getByText('No flow records found')).toBeVisible(); + await page + .getByRole('button', { name: 'Clear filters', exact: true }) + .click(); + await expect(records.locator('tbody tr')).toHaveCount(500); +}); + +test('NetFlow source selection applies immediately and clears incompatible filters', async ({ + page, + netflow, +}) => { + await page.goto(`/netflow?source=${netflow.source.id}`); + const records = page.getByTestId('netflow-records'); + await expect(records.locator('tbody tr')).toHaveCount(500); + const search = page.getByTestId('netflow-search'); + const protocol = page.getByLabel('Protocol', { exact: true }); + await search.fill('Proto:6'); + await protocol.fill('6'); + await page.getByRole('button', { name: 'Run', exact: true }).click(); + await expect + .poll(() => new URL(page.url()).searchParams.get('where')) + .toBe('Proto:6'); + await records + .getByRole('button', { name: 'Filter Protocol: TCP', exact: true }) + .first() + .click(); + await page.getByRole('menuitem', { name: 'Include', exact: true }).click(); + const pills = page.getByTestId('netflow-filter-pills'); + await expect(pills).toContainText('Protocol = TCP'); + await search.fill('Proto:17'); + await protocol.fill('17'); + await page + .getByRole('combobox', { name: 'NetFlow source', exact: true }) + .click(); + await page + .getByRole('option', { name: netflow.alternative.name, exact: true }) + .click(); + await expect + .poll(() => new URL(page.url()).searchParams.get('source')) + .toBe(netflow.alternative.id); + await expect( + pills.getByRole('button', { name: 'Remove filter' }), + ).toHaveCount(0); + await expect(search).toHaveValue(''); + await expect(protocol).toHaveValue(''); + for (const name of ['where', 'exporter', 'protocol', 'srcAddr', 'dstAddr']) { + await expect + .poll(() => new URL(page.url()).searchParams.get(name) || '') + .toBe(''); + } + await page + .getByRole('button', { name: 'Source actions', exact: true }) + .click(); + await page + .getByRole('menuitem', { name: 'Edit source', exact: true }) + .click(); + await expect( + page.getByRole('dialog').locator('input[name=name]'), + ).toHaveValue(netflow.alternative.name); + await page.keyboard.press('Escape'); + await page.reload(); + await expect( + page.getByRole('combobox', { name: 'NetFlow source', exact: true }), + ).toHaveValue(netflow.alternative.name); + await expect(records.locator('tbody tr')).toHaveCount(500); + await page.goto('/netflow?source=missing-source'); + await expect(page.getByText('NetFlow source unavailable')).toBeVisible(); +}); + +test('NetFlow invalid time ranges recover and historical ranges remain empty', async ({ + page, + netflow, +}) => { + for (const [from, to] of [ + [1000, 1000], + [2000, 1000], + ]) { + await page.goto( + `/netflow?source=${netflow.source.id}&from=${from}&to=${to}`, + ); + await expect( + page.getByText('Invalid time range', { exact: true }), + ).toBeVisible(); + await expect(page.getByTestId('netflow-records')).toHaveCount(0); + await page + .getByRole('button', { name: 'Use past hour', exact: true }) + .click(); + await expect( + page.getByRole('button', { name: 'Inspect flow 1', exact: true }), + ).toBeVisible(); + } + await page.goto(`/netflow?source=${netflow.source.id}&from=1000&to=3601000`); + await expect( + page.getByTestId('netflow-records').getByText('No flow records found'), + ).toBeVisible(); +}); + +for (const [key, label] of [ + ['srcAddr', 'Source IP'], + ['dstAddr', 'Destination IP'], + ['exporter', 'Exporter'], + ['protocol', 'Protocol'], +]) { + test(`NetFlow ${label} include/exclude filters persist and remove`, async ({ + page, + netflow, + }) => { + await page.goto(`/netflow?source=${netflow.source.id}`); + const records = page.getByTestId('netflow-records'); + await expect(records.locator('tbody tr')).toHaveCount(500); + const button = page + .getByTestId(`netflow-breakdown-${key}`) + .getByRole('button', { name: new RegExp(`^Filter ${label}:`) }) + .first(); + const value = await button.innerText(); + await button.click(); + await page.getByRole('menuitem', { name: 'Include', exact: true }).click(); + const matching = records.getByRole('button', { + name: `Filter ${label}: ${value}`, + exact: true, + }); + await expect + .poll( + async () => + (await matching.count()) > 0 && + (await matching.count()) === + (await records.locator('tbody tr').count()), + ) + .toBe(true); + await page.reload(); + await expect(matching.first()).toBeVisible(); + await page + .getByTestId('netflow-filter-pills') + .getByRole('button', { name: 'Remove filter' }) + .click(); + await expect(records.locator('tbody tr')).toHaveCount(500); + await button.click(); + await page.getByRole('menuitem', { name: 'Exclude', exact: true }).click(); + await expect(matching).toHaveCount(0); + }); +} diff --git a/packages/app/tests/e2e/features/netflow/queries.spec.ts b/packages/app/tests/e2e/features/netflow/queries.spec.ts new file mode 100644 index 0000000000..9059968202 --- /dev/null +++ b/packages/app/tests/e2e/features/netflow/queries.spec.ts @@ -0,0 +1,300 @@ +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { filtersToQuery, parseQuery } from '@hyperdx/common-utils/dist/filters'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; + +import { + buildNetflowQueryConfigs, + getNetflowDimensions, +} from '../../../../src/netflow'; +import { + buildNetflowSankeyConfig, + buildNetflowSankeyData, + sankeyDimensionExpression, +} from '../../../../src/netflowSankey'; +import { expect, test } from '../../fixtures/netflow'; + +const queryRunner = + (client: ClickhouseClient) => + async (config: BuilderChartConfigWithDateRange) => + ( + await client.queryChartConfig({ + config, + metadata: getMetadata(client), + querySettings: undefined, + }) + ).data; + +test('steady traffic keeps its rate in partial first and last buckets', async ({ + netflow, +}) => { + const { client, database, source } = netflow; + const run = queryRunner(client); + const start = Date.parse('2026-10-09T12:00:00Z'); + const view = await client.query({ + query: `CREATE VIEW ${database}.steady AS + SELECT toDateTime(${start / 1000}) + number AS TimeReceived, + 125 AS Bytes, 1 AS Packets, 100 AS SamplingRate FROM numbers(3660)`, + }); + await view.text(); + const configs = buildNetflowQueryConfigs({ + source: { + ...source, + from: { databaseName: database, tableName: 'steady' }, + }, + dateRange: [new Date(start + 30000), new Date(start + 3620000)], + filters: {}, + }); + const result = await run(configs.traffic); + expect(result).toHaveLength(61); + // 125 raw bytes/second * sampling 100 * 8. Both edge buckets have less than a minute of data. + expect(result.map(row => Number(row['Bits per second']))).toEqual( + Array(61).fill(100000), + ); + const summary = await run(configs.summary); + expect(Number(summary[0].__netflow_flowRecords)).toBe(3590); + expect(Number(summary[0].__netflow_bytes)).toBe(3590 * 125 * 100); +}); + +test('NetFlow queries preserve sampled totals, raw records, aliases and Lucene semantics', async ({ + netflow, +}) => { + const { client, source, dateRange, database } = netflow; + const run = queryRunner(client); + const direct = async (where = '1') => + ( + await ( + await client.query({ + query: `SELECT count() AS records, sum(Bytes * SamplingRate) AS bytes, sum(Packets * SamplingRate) AS packets FROM ${database}.flows WHERE ${where}`, + format: 'JSONEachRow', + }) + ).json<{ records: string; bytes: string; packets: string }>() + )[0]; + const expected = await direct(); + const configs = buildNetflowQueryConfigs({ source, dateRange, filters: {} }); + for (const config of Object.values(configs).filter( + config => config != null, + )) { + expect((await run(config)).length).toBeGreaterThan(0); + } + const [summary] = await run(configs.summary); + expect(Number(summary.__netflow_flowRecords)).toBe(1200); + expect(Number(summary.__netflow_bytes)).toBe(Number(expected.bytes)); + expect(Number(summary.__netflow_bitsPerSecond)).toBeCloseTo( + (Number(expected.bytes) * 8) / 3600, + ); + expect(Number(summary.__netflow_packetsPerSecond)).toBeCloseTo( + Number(expected.packets) / 3600, + ); + const records = await run(configs.flows); + expect(records).toHaveLength(500); + for (const row of records) { + expect(Number(row.__netflow_bytes)).toBe( + Number(row.__netflow_rawBytes) * Number(row.__netflow_samplingRate), + ); + expect(Number(row.__netflow_packets)).toBe( + Number(row.__netflow_rawPackets) * Number(row.__netflow_samplingRate), + ); + } + const scaled = buildNetflowQueryConfigs({ + source: { + ...source, + bytesExpression: 'Bytes * SamplingRate', + samplingRateExpression: undefined, + }, + dateRange, + filters: {}, + }); + expect(Number((await run(scaled.totalBytes))[0].__netflow_value)).toBe( + Number(expected.bytes), + ); + for (const [where, sql] of [ + ['Proto:6', 'Proto = 6'], + ['192.0.2.2', "SrcAddr = toIPv6('192.0.2.2')"], + [ + '(Proto:6 OR Proto:17) AND NOT ExporterName:missing*', + "Proto IN (6,17) AND ExporterName NOT ILIKE 'missing%'", + ], + ['Bytes:[500000 TO 2000000]', 'Bytes BETWEEN 500000 AND 2000000'], + [ + 'ExporterName:edge* AND DstPort:443', + "ExporterName ILIKE 'edge%' AND DstPort = 443", + ], + ]) { + const wanted = await direct(sql); + const [actual] = await run( + buildNetflowQueryConfigs({ source, dateRange, filters: {}, where }) + .summary, + ); + expect(Number(actual.__netflow_flowRecords)).toBe(Number(wanted.records)); + expect(Number(actual.__netflow_bytes)).toBe(Number(wanted.bytes)); + } + for (const [filters, count] of [ + [{ protocol: '6' }, 600], + [{ srcAddr: '192.0.2.2' }, 300], + [{ srcAddr: '2001:0db8:0001::2' }, 300], + [{ exporter: "edge' OR 1=1 --" }, 0], + ] as const) { + const [actual] = await run( + buildNetflowQueryConfigs({ source, dateRange, filters }).summary, + ); + expect(Number(actual.__netflow_flowRecords)).toBe(count); + } + await expect( + run( + buildNetflowQueryConfigs({ + source, + dateRange, + filters: {}, + where: 'Proto:(', + }).summary, + ), + ).rejects.toThrow(); +}); + +test('NetFlow handles lowercase counter aliases and custom timestamp and exporter expressions', async ({ + netflow, +}) => { + const { client, source, dateRange, database } = netflow; + const run = queryRunner(client); + const view = await client.query({ + query: `CREATE VIEW ${database}.lowercase AS SELECT + TimeReceived AS timestamp, Bytes AS bytes, Packets AS packets, + [NULL, -1, 0, 1, 100][toUnixTimestamp(TimeReceived) % 5 + 1] AS sampling, Proto AS protocol, ExporterName AS "router-name", + SrcAddr AS srcAddr, DstAddr AS dstAddr, SrcPort AS srcPort, DstPort AS dstPort, + InIfName AS inputInterface, OutIfName AS outputInterface FROM ${database}.flows`, + }); + await view.text(); + const customSource = { + ...source, + from: { databaseName: database, tableName: 'lowercase' }, + timestampValueExpression: 'toDateTime(timestamp), toDate(timestamp)', + bytesExpression: 'bytes', + packetsExpression: 'packets', + samplingRateExpression: 'sampling', + protocolExpression: 'protocol', + exporterExpression: '`router-name`', + srcAddrExpression: 'srcAddr', + dstAddrExpression: 'dstAddr', + srcPortExpression: 'srcPort', + dstPortExpression: 'dstPort', + inIfExpression: 'inputInterface', + outIfExpression: 'outputInterface', + }; + const configs = buildNetflowQueryConfigs({ + source: customSource, + dateRange, + filters: { protocol: '6', exporter: 'edge-a' }, + where: 'dstPort = 443', + whereLanguage: 'sql', + }); + for (const config of Object.values(configs).filter( + config => config != null, + )) { + expect((await run(config)).length).toBeGreaterThan(0); + } + const result = await run(configs.flows); + const summary = await run(configs.summary); + // Each of the five sampling values occurs 120 times after the TCP filter. + expect(Number(summary[0].__netflow_bytes)).toBe(6246240000000); + expect(Number(summary[0].__netflow_packetsPerSecond)).toBeCloseTo( + 12480000 / 3600, + ); + for (const row of result) { + expect([1, 100]).toContain(Number(row.__netflow_samplingRate)); + expect(Number(row.__netflow_bytes)).toBe( + Number(row.__netflow_rawBytes) * Number(row.__netflow_samplingRate), + ); + } + const sankey = await run( + buildNetflowSankeyConfig({ + baseConfig: configs.totalBytes, + dimensions: [ + { key: 'src', label: 'Source', expression: 'srcAddr' }, + { key: 'dst', label: 'Destination', expression: 'dstAddr' }, + ], + }), + ); + expect( + sankey.reduce((total, row) => total + Number(row.__netflow_value), 0), + ).toBe(6246240000000); + + expect(result).toHaveLength(500); + expect(new Date(String(result[0].__netflow_timestamp)).getTime()).toBe( + dateRange[1].getTime() - 904000, + ); + expect( + result.every( + row => + row.__netflow_protocol === 'TCP' && row.__netflow_exporter === 'edge-a', + ), + ).toBe(true); +}); + +test('NetFlow Sankey conserves path traffic and executes quoted, empty and composed filters', async ({ + netflow, +}) => { + const { client, source, dateRange } = netflow; + const run = queryRunner(client); + const dimensions = [ + { + key: 'src', + label: 'Source', + expression: getNetflowDimensions(source).srcAddr, + }, + { key: 'provider', label: 'Provider', expression: '`provider-name`' }, + { key: 'exporter', label: 'Exporter', expression: 'ExporterName' }, + ]; + const configs = buildNetflowQueryConfigs({ source, dateRange, filters: {} }); + const config = buildNetflowSankeyConfig({ + baseConfig: configs.totalBytes, + dimensions, + limit: 100, + }); + const data = buildNetflowSankeyData(await run(config), dimensions); + expect(data.paths).toHaveLength(4); + const total = Number((await run(configs.totalBytes))[0].__netflow_value); + expect(data.paths.reduce((sum, path) => sum + path.value, 0)).toBe(total); + for (const stage of [0, 1]) { + expect( + data.links + .filter(link => data.nodes[link.source].stage === stage) + .reduce((sum, link) => sum + link.value, 0), + ).toBe(total); + } + const expression = sankeyDimensionExpression(dimensions[1]); + for (const [value, count] of [ + ['', 300], + ['provider', 900], + ["' OR 1=1 --", 0], + ] as const) { + const extraFilters = filtersToQuery({ + [expression]: { included: new Set([value]), excluded: new Set() }, + }); + const restored = parseQuery(extraFilters); + expect(Object.keys(restored.filters)).toHaveLength(1); + const filtered = buildNetflowQueryConfigs({ + source, + dateRange, + filters: {}, + extraFilters, + }); + const [summary] = await run(filtered.summary); + expect(Number(summary.__netflow_flowRecords)).toBe(count); + } + const filtered = buildNetflowQueryConfigs({ + source, + dateRange, + filters: { protocol: '6' }, + where: 'DstPort:443', + extraFilters: [{ type: 'sql', condition: "ExporterName IN ('edge-a')" }], + }); + const paths = await run( + buildNetflowSankeyConfig({ + baseConfig: filtered.totalBytes, + dimensions, + limit: 10, + }), + ); + expect(paths).toHaveLength(2); +}); diff --git a/packages/app/tests/e2e/features/netflow/sankey.spec.ts b/packages/app/tests/e2e/features/netflow/sankey.spec.ts new file mode 100644 index 0000000000..f12aa98b32 --- /dev/null +++ b/packages/app/tests/e2e/features/netflow/sankey.spec.ts @@ -0,0 +1,132 @@ +import { expect, test } from '../../fixtures/netflow'; + +test.use({ viewport: { width: 1600, height: 1050 } }); + +test('NetFlow Sankey dimensions, filters, empty classifications and URL state', async ({ + page, + netflow, +}) => { + await page.goto(`/netflow?source=${netflow.source.id}`); + await page.getByText('Sankey', { exact: true }).click(); + const graph = page.getByTestId('netflow-sankey-graph'); + const table = page.getByTestId('netflow-sankey-table'); + const rows = table.locator('tbody tr'); + await expect(graph.locator('svg')).toBeVisible(); + await expect(rows).toHaveCount(4); + await expect(table.getByRole('columnheader')).toHaveText([ + 'SrcAS', + 'InIfConnectivity', + 'InIfProvider', + 'Exporter', + 'Average bit rate', + 'Transferred bytes', + ]); + const provider = graph.getByRole('button', { + name: 'Filter InIfProvider: peer-east', + exact: true, + }); + await provider.hover(); + await expect( + page.getByRole('tooltip').filter({ hasText: 'Average bit rate:' }), + ).toBeVisible(); + await provider.click(); + await page.getByRole('menuitem', { name: 'Include', exact: true }).click(); + await expect(rows).toHaveCount(1); + await expect(rows.first().locator('td').nth(2)).toHaveText('peer-east'); + await page.reload(); + await expect(rows).toHaveCount(1); + await provider.click(); + await page.getByRole('menuitem', { name: 'Exclude', exact: true }).click(); + await expect(rows).toHaveCount(3); + await expect(provider).toHaveCount(0); + await page + .getByRole('button', { name: 'Clear filters', exact: true }) + .click(); + await expect(rows).toHaveCount(4); + await graph + .getByRole('button', { name: 'Filter InIfProvider: (empty)', exact: true }) + .click(); + await page.getByRole('menuitem', { name: 'Include', exact: true }).click(); + await expect(rows).toHaveCount(2); + await page + .getByRole('button', { name: 'Clear filters', exact: true }) + .click(); + await page + .getByRole('button', { name: 'Clear Sankey dimensions', exact: true }) + .click(); + await expect( + page.getByText('Choose two to five dimensions', { exact: true }), + ).toBeVisible(); + const picker = page.locator('input[aria-label="Sankey dimensions"]'); + for (const name of ['Source IP', 'Protocol', 'Destination IP']) { + await picker.fill(name); + await page.getByRole('option', { name, exact: true }).click(); + } + await picker.press('Escape'); + await page + .getByRole('combobox', { name: 'Sankey path limit', exact: true }) + .click(); + await page.getByRole('option', { name: '10', exact: true }).click(); + await expect(rows).toHaveCount(4); + await page.reload(); + await expect(table.getByRole('columnheader').first()).toHaveText('Source IP'); + await expect( + page.getByRole('combobox', { name: 'Sankey path limit', exact: true }), + ).toHaveValue('10'); + await page.setViewportSize({ width: 800, height: 1000 }); + await graph.scrollIntoViewIfNeeded(); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= innerWidth, + ), + ).toBe(true); + await page.getByTestId('netflow-search').fill('ExporterName:missing'); + await page.getByRole('button', { name: 'Run', exact: true }).click(); + await expect( + page.getByText('No traffic paths found', { exact: true }), + ).toBeVisible(); + await page.getByText('Time series', { exact: true }).click(); + await expect(page.getByTestId('netflow-sankey')).toHaveCount(0); +}); + +test('real NetFlow chart axes keep Mbit/s and Gbit/s labels inside the SVG', async ({ + page, + netflow, +}) => { + for (const width of [1280, 800]) { + await page.setViewportSize({ width, height: 1000 }); + for (const [filter, unit] of [ + ['&srcAddr=192.0.2.2', 'Mbit/s'], + ['', 'Gbit/s'], + ]) { + await page.goto(`/netflow?source=${netflow.source.id}${filter}`); + await expect( + page.getByRole('button', { name: 'Inspect flow 1', exact: true }), + ).toBeVisible(); + await page + .getByText('Network traffic', { exact: true }) + .scrollIntoViewIfNeeded(); + await page.evaluate(() => document.fonts.ready); + const labels = page + .locator('text.recharts-cartesian-axis-tick-value') + .filter({ hasText: unit }); + await expect(labels.first()).toBeVisible(); + await expect + .poll(() => + labels.evaluateAll(nodes => + nodes.every(node => { + const bounds = node.getBoundingClientRect(); + const viewport = node.closest('svg')?.getBoundingClientRect(); + return ( + node.querySelectorAll('tspan').length === 1 && + viewport && + bounds.left >= viewport.left && + bounds.right <= viewport.right + ); + }), + ), + ) + .toBe(true); + } + } +}); diff --git a/packages/app/tests/e2e/features/netflow/source.spec.ts b/packages/app/tests/e2e/features/netflow/source.spec.ts new file mode 100644 index 0000000000..d730d289c3 --- /dev/null +++ b/packages/app/tests/e2e/features/netflow/source.spec.ts @@ -0,0 +1,54 @@ +import { expect, test } from '../../fixtures/netflow'; + +test('creates an inferred NetFlow source and opens its records in shared Search', async ({ + page, + netflow, +}) => { + await page.goto(`/netflow?source=${netflow.source.id}`); + await expect( + page.getByRole('button', { name: 'Inspect flow 1', exact: true }), + ).toBeVisible(); + const search = page.getByTestId('netflow-search'); + await search.fill('DstPo'); + await expect( + page + .getByTestId('autocomplete-suggestion') + .filter({ hasText: 'DstPort (number)' }), + ).toBeVisible(); + await search.press('Escape'); + await page + .getByRole('button', { name: 'Source actions', exact: true }) + .click(); + await page + .getByRole('menuitem', { name: 'Create new source', exact: true }) + .click(); + const form = page.getByRole('dialog', { name: 'Add NetFlow source' }); + const name = `${netflow.database} created`; + await form.locator('input[name=name]').fill(name); + await form + .getByPlaceholder('Database', { exact: true }) + .fill(netflow.database); + await page + .getByRole('option', { name: netflow.database, exact: true }) + .click(); + await form.getByPlaceholder('Table', { exact: true }).fill('flows'); + await page.getByRole('option', { name: 'flows', exact: true }).click(); + await expect(form.locator('.cm-content').first()).toHaveText('TimeReceived'); + await form + .getByRole('button', { name: 'Save New Source', exact: true }) + .click(); + await expect(form).toBeHidden(); + await expect( + page.getByRole('combobox', { name: 'NetFlow source', exact: true }), + ).toHaveValue(name); + await expect( + page.getByRole('button', { name: 'Inspect flow 1', exact: true }), + ).toBeVisible(); + await page.getByRole('link', { name: 'Search', exact: true }).click(); + await expect( + page + .getByRole('cell') + .filter({ hasText: /192\.0\.2/ }) + .first(), + ).toBeVisible(); +}); diff --git a/packages/app/tests/e2e/features/netflow/summary.spec.ts b/packages/app/tests/e2e/features/netflow/summary.spec.ts new file mode 100644 index 0000000000..47c6ac0e6c --- /dev/null +++ b/packages/app/tests/e2e/features/netflow/summary.spec.ts @@ -0,0 +1,29 @@ +import { NETFLOW_SUMMARY_TILES } from '../../../../src/netflow'; +import { formatNumber } from '../../../../src/utils'; +import { expect, test } from '../../fixtures/netflow'; + +test('four NetFlow overview tiles share one summary query and show distinct metrics', async ({ + page, + netflow, +}) => { + const requests: string[] = []; + page.on('request', request => { + const sql = request.postData() || ''; + if (request.method() === 'POST' && sql.includes('__netflow_bitsPerSecond')) + requests.push(sql); + }); + const [start, end] = netflow.dateRange; + await page.goto( + `/netflow?source=${netflow.source.id}&from=${start.getTime()}&to=${end.getTime()}`, + ); + const bytes = + 300 * (1000000 * 100 + 1000000000 + 1000000000 * 100 + 1000000000); + const packets = 600 * 1000 * 100 + 600 * 1000; + const expected = [(bytes * 8) / 3600, packets / 3600, bytes, 1200]; + await expect(page.getByTestId('number-chart-value')).toHaveText( + expected.map((value, index) => + formatNumber(value, NETFLOW_SUMMARY_TILES[index].numberFormat), + ), + ); + expect(requests).toHaveLength(1); +}); diff --git a/packages/app/tests/e2e/fixtures/netflow.ts b/packages/app/tests/e2e/fixtures/netflow.ts new file mode 100644 index 0000000000..dc0597d4d4 --- /dev/null +++ b/packages/app/tests/e2e/fixtures/netflow.ts @@ -0,0 +1,135 @@ +import { randomUUID } from 'crypto'; +import { + ClickhouseClient, + createNativeClient, +} from '@hyperdx/common-utils/dist/clickhouse/node'; +import { SourceKind, TNetflowSource } from '@hyperdx/common-utils/dist/types'; + +import { getApiUrl } from '../utils/api-helpers'; +import { expect, test as base } from '../utils/base-test'; + +const host = + process.env.CLICKHOUSE_HOST || + `http://localhost:${process.env.HDX_E2E_CH_PORT || '20500'}`; +const credentials = { + username: process.env.CLICKHOUSE_USER || 'default', + password: process.env.CLICKHOUSE_PASSWORD || '', +}; + +export const test = base.extend<{ + netflow: { + database: string; + source: TNetflowSource; + alternative: TNetflowSource; + dateRange: [Date, Date]; + client: ClickhouseClient; + }; +}>({ + netflow: async ({ request }, provide) => { + const database = `netflow_e2e_${randomUUID().replaceAll('-', '')}`; + const commands = createNativeClient({ url: host, ...credentials }); + const client = new ClickhouseClient({ host, ...credentials }); + const end = new Date(Math.floor(Date.now() / 60000) * 60000); + const start = new Date(end.getTime() - 60 * 60000); + let source: TNetflowSource = { + id: `${database}_flows`, + name: `${database} flows`, + kind: SourceKind.Netflow, + connection: 'netflow-test', + from: { databaseName: database, tableName: 'flows' }, + timestampValueExpression: 'TimeReceived', + defaultTableSelectExpression: '*', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + samplingRateExpression: 'SamplingRate', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + exporterExpression: 'ExporterName', + inIfExpression: 'InIfName', + outIfExpression: 'OutIfName', + }; + let alternative = { + ...source, + id: `${database}_other`, + name: `${database} other`, + }; + const createdSources: string[] = []; + try { + await commands.command({ query: `CREATE DATABASE ${database}` }); + await commands.command({ + query: `CREATE TABLE ${database}.flows ( + TimeReceived DateTime('UTC'), Bytes UInt64, Packets UInt64, SamplingRate UInt64, + SrcAddr IPv6, DstAddr IPv6, SrcPort UInt16, DstPort UInt16, Proto UInt8, + ExporterName String, InIfName String, OutIfName String, + SrcAS UInt32, InIfConnectivity String, InIfProvider String, + "provider-name" Nullable(String) + ) ENGINE = MergeTree ORDER BY TimeReceived`, + }); + // Four paths repeat across an hour, including sampled IPv4, IPv6 and an empty classification. + await commands.command({ + query: `INSERT INTO ${database}.flows SELECT + toDateTime(${end.getTime() / 1000}) - 3300 + number * 2, + if(number % 4 = 0, 1000000, 1000000000), 1000, if(number % 2 = 0, 100, 1), + toIPv6(['192.0.2.2','192.0.2.3','2001:db8:1::2','2001:db8:1::3'][number % 4 + 1]), + toIPv6('198.51.100.1'), 12345, if(number % 2 = 0, 443, 53), + if(number % 2 = 0, 6, 17), if(number % 2 = 0, 'edge-a', 'edge-b'), + 'eth0', 'eth1', 64500 + number % 4, 'transit', + ['peer-east','peer-west','',''][number % 4 + 1], + if(number % 4 = 2, NULL, 'provider') + FROM numbers(1200)`, + }); + if (process.env.E2E_FULLSTACK === 'true') { + const connections = await request.get(`${getApiUrl()}/connections`); + expect(connections.ok()).toBeTruthy(); + const [connection] = await connections.json(); + const create = async ( + input: TNetflowSource, + ): Promise => { + const response = await request.post(`${getApiUrl()}/sources`, { + data: { ...input, connection: connection.id }, + }); + expect(response.ok(), await response.text()).toBeTruthy(); + const saved = await response.json(); + const id = saved.id || saved._id; + createdSources.push(id); + return { ...input, id, connection: connection.id }; + }; + source = await create(source); + alternative = await create(alternative); + } + await provide({ + database, + source, + alternative, + dateRange: [start, end], + client, + }); + } finally { + if (process.env.E2E_FULLSTACK === 'true') { + const response = await request.get(`${getApiUrl()}/sources`); + const sources: TNetflowSource[] = await response.json(); + for (const saved of sources.filter( + item => item.from.databaseName === database, + )) { + if (!createdSources.includes(saved.id)) createdSources.push(saved.id); + } + } + for (const id of createdSources) + await request.delete(`${getApiUrl()}/sources/${id}`); + await commands.command({ query: `DROP DATABASE IF EXISTS ${database}` }); + await client.close(); + await commands.close(); + } + }, + localSources: async ({ netflow }, provide) => + provide([netflow.source, netflow.alternative]), + localConnections: async ({}, provide) => + provide([ + { id: 'netflow-test', name: 'NetFlow test', host, ...credentials }, + ]), +}); + +export { expect }; diff --git a/packages/app/tests/e2e/utils/base-test.ts b/packages/app/tests/e2e/utils/base-test.ts index 3537b976f8..8a199340df 100644 --- a/packages/app/tests/e2e/utils/base-test.ts +++ b/packages/app/tests/e2e/utils/base-test.ts @@ -23,8 +23,13 @@ function loadE2EFixtures(): { connections: unknown[]; sources: unknown[] } { const e2eFixtures = loadE2EFixtures(); // Extend the base test to automatically handle Tanstack devtools -export const test = base.extend({ - page: async ({ page }, fn) => { +export const test = base.extend<{ + localSources: unknown[]; + localConnections: unknown[]; +}>({ + localSources: [e2eFixtures.sources, { option: true }], + localConnections: [e2eFixtures.connections, { option: true }], + page: async ({ page, localSources, localConnections }, fn) => { // Note: page.addInitScript runs in the browser context, which cannot access Node.js // environment variables directly. We pass USE_FULLSTACK and connection/sources from // e2e-fixtures.json so local mode uses the same data as full-stack. @@ -48,11 +53,7 @@ export const test = base.extend({ JSON.stringify(true), ); }, - [ - e2eFixtures.connections, - e2eFixtures.sources, - VIEW_TRACE_CALLOUT_DISMISSED_KEY, - ], + [localConnections, localSources, VIEW_TRACE_CALLOUT_DISMISSED_KEY], ); await fn(page); }, diff --git a/packages/common-utils/src/__tests__/iac.test.ts b/packages/common-utils/src/__tests__/iac.test.ts index 896e311b9e..c4b151bdeb 100644 --- a/packages/common-utils/src/__tests__/iac.test.ts +++ b/packages/common-utils/src/__tests__/iac.test.ts @@ -369,6 +369,10 @@ describe('isImportableSource', () => { } }); + it('rejects NetFlow mappings unsupported by the Terraform provider', () => { + expect(isImportableSource({ kind: SourceKind.Netflow })).toBe(false); + }); + it('rejects a PromQL source', () => { expect(isImportableSource({ kind: SourceKind.Promql })).toBe(false); }); diff --git a/packages/common-utils/src/__tests__/netflowSource.test.ts b/packages/common-utils/src/__tests__/netflowSource.test.ts new file mode 100644 index 0000000000..da88b16523 --- /dev/null +++ b/packages/common-utils/src/__tests__/netflowSource.test.ts @@ -0,0 +1,129 @@ +import { + buildSearchChartConfig, + getSourceImplicitColumnExpression, + netflowProtocolNameExpression, +} from '@/core/searchChartConfig'; +import { + isSearchableSource, + SourceKind, + SourceSchema, + SourceSchemaNoId, +} from '@/types'; + +const source = { + id: 'netflow', + kind: SourceKind.Netflow, + name: 'Network flows', + connection: 'clickhouse', + from: { databaseName: 'default', tableName: 'flows' }, + timestampValueExpression: 'TimeReceived', + defaultTableSelectExpression: 'TimeReceived, SrcAddr, DstAddr, Bytes', + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + samplingRateExpression: 'SamplingRate', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + exporterExpression: 'ExporterName', + inIfExpression: 'InIfName', + outIfExpression: 'OutIfName', +}; + +it('preserves NetFlow mappings through create and update validation', () => { + expect(SourceSchema.parse(source)).toEqual(source); + const { id: _id, ...input } = source; + expect(SourceSchemaNoId.parse(input)).toEqual(input); +}); + +it('supports unsampled data without exporter or interface metadata', () => { + expect( + SourceSchema.safeParse({ + ...source, + samplingRateExpression: undefined, + exporterExpression: undefined, + inIfExpression: undefined, + outIfExpression: undefined, + }).success, + ).toBe(true); +}); + +it.each([ + 'bytesExpression', + 'packetsExpression', + 'srcAddrExpression', + 'dstAddrExpression', + 'protocolExpression', +])('requires the %s mapping', field => { + expect(SourceSchema.safeParse({ ...source, [field]: '' }).success).toBe( + false, + ); +}); + +it('searches raw flows with mapped defaults without log-only fields or sampling weights', () => { + const parsed = SourceSchema.parse(source); + expect(isSearchableSource(parsed)).toBe(true); + const config = buildSearchChartConfig(parsed, { + where: 'Proto = 6', + whereLanguage: 'sql', + }); + expect(config.select).toBe(source.defaultTableSelectExpression); + expect(config.timestampValueExpression).toBe('TimeReceived'); + expect(config.where).toBe('Proto = 6'); + expect(config).not.toHaveProperty('bodyExpression'); + expect(config).not.toHaveProperty('sampleWeightExpression'); +}); + +it('searches mapped NetFlow dimensions for bare Lucene terms by default', () => { + const config = buildSearchChartConfig(SourceSchema.parse(source), { + where: 'edge-router', + whereLanguage: 'lucene', + }); + expect(config.implicitColumnExpression).toContain('toString(SrcAddr)'); + expect(config.implicitColumnExpression).toContain('toString(ExporterName)'); + expect(config.implicitColumnExpression).toContain('toString(OutIfName)'); + expect(config.implicitColumnExpression).toContain( + "ifNull(toString(Proto), '')", + ); + expect(config.implicitColumnExpression).toContain( + netflowProtocolNameExpression('Proto'), + ); + expect(netflowProtocolNameExpression('Proto')).toBe( + "transform(toString(Proto), ['1', '6', '17', '47', '50', '58'], ['ICMP', 'TCP', 'UDP', 'GRE', 'ESP', 'ICMPv6'], toString(Proto))", + ); + expect(getSourceImplicitColumnExpression(SourceSchema.parse(source))).toBe( + config.implicitColumnExpression, + ); +}); + +it('preserves and uses a custom NetFlow implicit expression', () => { + const parsed = SourceSchema.parse({ + ...source, + implicitColumnExpression: 'SearchText', + }); + expect(parsed).toHaveProperty('implicitColumnExpression', 'SearchText'); + expect(getSourceImplicitColumnExpression(parsed)).toBe('SearchText'); + expect( + buildSearchChartConfig(parsed, { where: 'router' }) + .implicitColumnExpression, + ).toBe('SearchText'); +}); + +it('derives implicit search from custom mappings and ignores blank optional dimensions', () => { + const parsed = SourceSchema.parse({ + ...source, + srcAddrExpression: 'client_ip', + implicitColumnExpression: ' ', + exporterExpression: ' ', + inIfExpression: undefined, + outIfExpression: '', + }); + const implicit = buildSearchChartConfig(parsed, { + where: 'router', + }).implicitColumnExpression; + expect(implicit).toContain('toString(client_ip)'); + expect(implicit).not.toMatch( + /SrcAddr|ExporterName|InIfName|OutIfName|toString\(\s*\)/, + ); +}); diff --git a/packages/common-utils/src/clickhouse/__tests__/browser.test.ts b/packages/common-utils/src/clickhouse/__tests__/browser.test.ts index eab74e43e6..9eb4b2ad36 100644 --- a/packages/common-utils/src/clickhouse/__tests__/browser.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/browser.test.ts @@ -1,4 +1,8 @@ -import { consoleLogger } from '@/clickhouse/browser'; +import { createClient } from '@clickhouse/client-web'; + +import { ClickhouseClient, consoleLogger } from '@/clickhouse/browser'; + +jest.mock('@clickhouse/client-web', () => ({ createClient: jest.fn() })); describe('consoleLogger', () => { afterEach(() => { @@ -63,3 +67,56 @@ describe('consoleLogger', () => { expect(errorSpy).toHaveBeenCalledWith('[clickhouse] Query failed', err); }); }); + +describe('browser connection routing', () => { + const query = jest.fn().mockResolvedValue({}); + beforeEach(() => { + Object.defineProperty(globalThis, 'window', { + value: { origin: 'http://localhost:3000' }, + configurable: true, + }); + jest + .mocked(createClient) + .mockReturnValue({ query } as unknown as ReturnType); + query.mockClear(); + jest.spyOn(console, 'debug').mockImplementation(() => {}); + }); + afterEach(() => { + Reflect.deleteProperty(globalThis, 'window'); + jest.restoreAllMocks(); + }); + it('omits proxy routing headers for direct ClickHouse connections with generated IDs', async () => { + const client = new ClickhouseClient({ + host: 'http://localhost:8123', + username: 'default', + password: '', + }); + await client.query({ + query: 'SELECT 1', + connectionId: 'local-generated-id', + shouldSkipApplySettings: true, + }); + expect(query).toHaveBeenCalledWith( + expect.objectContaining({ http_headers: {} }), + ); + }); + it.each([{}, { username: 'default' }, { password: '' }])( + 'retains proxy routing with incomplete credentials %j', + async credentials => { + const client = new ClickhouseClient({ + host: '/api/clickhouse-proxy', + ...credentials, + }); + await client.query({ + query: 'SELECT 1', + connectionId: 'team-connection', + shouldSkipApplySettings: true, + }); + expect(query).toHaveBeenCalledWith( + expect.objectContaining({ + http_headers: { 'x-hyperdx-connection-id': 'team-connection' }, + }), + ); + }, + ); +}); diff --git a/packages/common-utils/src/clickhouse/browser.ts b/packages/common-utils/src/clickhouse/browser.ts index fc107f0725..1a4fe60568 100644 --- a/packages/common-utils/src/clickhouse/browser.ts +++ b/packages/common-utils/src/clickhouse/browser.ts @@ -118,13 +118,16 @@ export class ClickhouseClient extends BaseClickhouseClient { return super.getClient() as WebClickHouseClient; } + private get isLocalMode() { + return this.username != null && this.password != null; + } + private buildClient() { let url = this.host!; let myFetch: typeof fetch; - const isLocalMode = this.username != null && this.password != null; const clickhouseSettings: ClickHouseSettings = {}; - if (isLocalMode) { + if (this.isLocalMode) { myFetch = localModeFetch; clickhouseSettings.add_http_cors_header = 1; } else { @@ -185,7 +188,8 @@ export class ClickhouseClient extends BaseClickhouseClient { } const httpHeaders: { [header: string]: string } = { - ...(connectionId && connectionId !== 'local' + // Only the API proxy uses this header; direct requests must avoid a CORS preflight. + ...(!this.isLocalMode && connectionId && connectionId !== 'local' ? { 'x-hyperdx-connection-id': connectionId } : {}), }; diff --git a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts index 99d28a5c53..9fc080ee22 100644 --- a/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts +++ b/packages/common-utils/src/core/__tests__/searchChartConfig.test.ts @@ -1,6 +1,7 @@ import { ALERT_COUNT_DEFAULT_SELECT, buildSearchChartConfig, + getSourceImplicitColumnExpression, } from '@/core/searchChartConfig'; import { DisplayType, Filter, SourceKind, TSource } from '@/types'; @@ -44,6 +45,43 @@ const makeMetricSource = (overrides: Record = {}): TSource => ...overrides, }) as unknown as TSource; +describe('getSourceImplicitColumnExpression', () => { + it.each([makeLogSource, makeTraceSource])( + 'preserves the configured expression on existing searchable kinds', + makeSource => { + const expression = " concatWithSeparator(';', Body, Message) "; + expect( + getSourceImplicitColumnExpression( + makeSource({ + implicitColumnExpression: expression, + }), + ), + ).toBe(expression); + expect( + getSourceImplicitColumnExpression( + makeSource({ + implicitColumnExpression: undefined, + }), + ), + ).toBeUndefined(); + }, + ); + + it.each([SourceKind.Metric, SourceKind.Session, SourceKind.Promql])( + 'does not add implicit search to %s sources', + kind => { + expect( + getSourceImplicitColumnExpression( + makeMetricSource({ + kind, + implicitColumnExpression: 'unexpected', + }), + ), + ).toBeUndefined(); + }, + ); +}); + describe('buildSearchChartConfig', () => { describe('tableFilterExpression', () => { it('prepends tableFilterExpression as a SQL filter on Log sources', () => { diff --git a/packages/common-utils/src/core/searchChartConfig.ts b/packages/common-utils/src/core/searchChartConfig.ts index 066c0f9760..a87da1af6f 100644 --- a/packages/common-utils/src/core/searchChartConfig.ts +++ b/packages/common-utils/src/core/searchChartConfig.ts @@ -4,6 +4,7 @@ import { DisplayType, Filter, isLogSource, + isNetflowSource, isTraceSource, pickSampleWeightExpressionProps, SearchCondition, @@ -11,6 +12,7 @@ import { SelectList, SortSpecificationList, SQLInterval, + TNetflowSource, TSource, } from '@/types'; @@ -40,6 +42,59 @@ export function getSourceTableFilters(source: TSource): Filter[] { */ export type SearchChartConfig = BuilderChartConfig & Partial; +export const NETFLOW_PROTOCOL_NAMES = { + 1: 'ICMP', + 6: 'TCP', + 17: 'UDP', + 47: 'GRE', + 50: 'ESP', + 58: 'ICMPv6', +}; + +/** Shows well-known IP protocol numbers by name, e.g. 6 as TCP. */ +export function netflowProtocolNameExpression(expression: string): string { + return `transform(toString(${expression}), [${Object.keys( + NETFLOW_PROTOCOL_NAMES, + ) + .map(value => `'${value}'`) + .join(', ')}], [${Object.values(NETFLOW_PROTOCOL_NAMES) + .map(value => `'${value}'`) + .join(', ')}], toString(${expression}))`; +} + +/** Bare terms search mapped dimensions even on sources saved before this option existed. */ +function getNetflowImplicitColumnExpression(source: TNetflowSource): string { + if (source.implicitColumnExpression?.trim()) { + return source.implicitColumnExpression.trim(); + } + const protocol = source.protocolExpression?.trim(); + const expressions = [ + source.srcAddrExpression, + source.dstAddrExpression, + source.srcPortExpression, + source.dstPortExpression, + protocol, + protocol && netflowProtocolNameExpression(protocol), + source.exporterExpression, + source.inIfExpression, + source.outIfExpression, + ] + .map(expression => expression?.trim()) + .filter(Boolean); + return `concatWithSeparator(' ', ${expressions.map(expression => `ifNull(toString(${expression}), '')`).join(', ')})`; +} + +export function getSourceImplicitColumnExpression( + source: TSource, +): string | undefined { + if (isNetflowSource(source)) { + return getNetflowImplicitColumnExpression(source); + } + return isLogSource(source) || isTraceSource(source) + ? source.implicitColumnExpression + : undefined; +} + /** * Default SELECT used by alert evaluators when no caller-supplied SELECT * is provided: a single `count()` aggregate. @@ -96,7 +151,7 @@ export type SearchChartConfigInput = { /** * Resolve the SELECT list, preferring caller-provided `select`, then the - * source's `defaultTableSelectExpression` (for Log / Trace sources), falling + * source's `defaultTableSelectExpression` (for searchable sources), falling * back to an empty string. * * Both `string` and `DerivedColumn[]` SELECT shapes have a `.length` property, @@ -108,7 +163,7 @@ function resolveSelect( select: SelectList | null | undefined, ): BuilderChartConfig['select'] { if (select != null && select.length > 0) return select; - if (isLogSource(source) || isTraceSource(source)) { + if (isLogSource(source) || isTraceSource(source) || isNetflowSource(source)) { return source.defaultTableSelectExpression ?? ''; } return ''; @@ -147,10 +202,7 @@ export function buildSearchChartConfig( const userFilters: Filter[] = input.filters ?? []; const mergedFilters: Filter[] = [...tableFilter, ...userFilters]; - const implicitColumnExpression = - isLogSource(source) || isTraceSource(source) - ? source.implicitColumnExpression - : undefined; + const implicitColumnExpression = getSourceImplicitColumnExpression(source); const useTextIndexForImplicitColumn = isLogSource(source) || isTraceSource(source) ? source.useTextIndexForImplicitColumn diff --git a/packages/common-utils/src/iacEligibility.ts b/packages/common-utils/src/iacEligibility.ts index 54a7b7bd79..ee040e3159 100644 --- a/packages/common-utils/src/iacEligibility.ts +++ b/packages/common-utils/src/iacEligibility.ts @@ -214,11 +214,13 @@ export function isImportableDashboard(dashboard: { } /** - * A PromQL source has no `clickhouse_clickstack_source` representation — the + * PromQL and NetFlow sources have no supported `clickhouse_clickstack_source` representation — the * provider models the ClickHouse-backed kinds. Emitting one produces an import * block for a resource the provider cannot read, which fails the plan. Same * shape as the tile-alert rule: eligibility decided once, here. */ export function isImportableSource(source: { kind?: string }): boolean { - return source.kind !== SourceKind.Promql; + return ( + source.kind !== SourceKind.Promql && source.kind !== SourceKind.Netflow + ); } diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 8991efe487..c1eef64f1b 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2608,6 +2608,7 @@ export type Team = z.infer; // -------------------------- export enum SourceKind { Log = 'log', + Netflow = 'netflow', Trace = 'trace', Session = 'session', Metric = 'metric', @@ -2856,6 +2857,38 @@ export const PromqlSourceSchema = BaseSourceSchema.extend({ minAutoGranularity: MinAutoGranularitySchema, }); +// NetFlow mappings support Akvorado and other flow tables without reingestion. +export const NETFLOW_COLUMN_EXPRESSIONS = { + bytesExpression: 'Bytes', + packetsExpression: 'Packets', + srcAddrExpression: 'SrcAddr', + dstAddrExpression: 'DstAddr', + srcPortExpression: 'SrcPort', + dstPortExpression: 'DstPort', + protocolExpression: 'Proto', + samplingRateExpression: 'SamplingRate', + exporterExpression: 'ExporterName', + inIfExpression: 'InIfName', + outIfExpression: 'OutIfName', +} as const; + +export const NetflowSourceSchema = BaseSourceSchema.extend({ + kind: z.literal(SourceKind.Netflow), + defaultTableSelectExpression: z.string().min(1), + implicitColumnExpression: z.string().optional(), + bytesExpression: z.string().min(1), + packetsExpression: z.string().min(1), + srcAddrExpression: z.string().min(1), + dstAddrExpression: z.string().min(1), + srcPortExpression: z.string().min(1), + dstPortExpression: z.string().min(1), + protocolExpression: z.string().min(1), + samplingRateExpression: z.string().optional(), + exporterExpression: z.string().optional(), + inIfExpression: z.string().optional(), + outIfExpression: z.string().optional(), +}); + // Union of all source form schemas for validation export const SourceSchema = z.discriminatedUnion('kind', [ LogSourceSchema, @@ -2863,6 +2896,7 @@ export const SourceSchema = z.discriminatedUnion('kind', [ SessionSourceSchema, MetricSourceSchema, PromqlSourceSchema, + NetflowSourceSchema, ]); export type TSource = z.infer; @@ -2872,10 +2906,12 @@ export const SourceSchemaNoId = z.discriminatedUnion('kind', [ SessionSourceSchema.omit({ id: true }), MetricSourceSchema.omit({ id: true }), PromqlSourceSchema.omit({ id: true }), + NetflowSourceSchema.omit({ id: true }), ]); export type TSourceNoId = z.infer; // Per-kind source types extracted from the Zod discriminated union +export type TNetflowSource = Extract; export type TLogSource = Extract; export type TTraceSource = Extract; export type TSessionSource = Extract; @@ -2883,6 +2919,9 @@ export type TMetricSource = Extract; export type TPromqlSource = Extract; // Type guards for narrowing TSource by kind +export function isNetflowSource(source: TSource): source is TNetflowSource { + return source.kind === SourceKind.Netflow; +} export function isLogSource(source: TSource): source is TLogSource { return source.kind === SourceKind.Log; } @@ -2899,7 +2938,9 @@ export function isPromqlSource(source: TSource): source is TPromqlSource { return source.kind === SourceKind.Promql; } export function isSearchableSource(source: TSource): boolean { - return isLogSource(source) || isTraceSource(source); + return ( + isLogSource(source) || isTraceSource(source) || isNetflowSource(source) + ); } type SourceLikeForSampleWeight = { diff --git a/scripts/netflow-dev.py b/scripts/netflow-dev.py new file mode 100644 index 0000000000..eeb3bb6a73 --- /dev/null +++ b/scripts/netflow-dev.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Run HyperDX locally with a configured NetFlow demo source.""" + +import json +import os +from pathlib import Path +import socket + + +repo = Path(__file__).resolve().parent.parent +port = None +for candidate in (3000, 3001): + with socket.socket() as probe: + try: + probe.bind(("127.0.0.1", candidate)) + port = candidate + break + except OSError: + pass +if port is None: + raise SystemExit("Ports 3000 and 3001 are occupied; stop an existing demo first") + +connection = { + "id": "netflow-local", + "name": "Local ClickHouse", + "host": os.getenv("CLICKHOUSE_URL", "http://127.0.0.1:8123"), + "username": os.getenv("CLICKHOUSE_USER", "default"), + "password": os.getenv("CLICKHOUSE_PASSWORD", ""), +} +source = { + "id": "netflow-demo", + "kind": "netflow", + "name": "NetFlow demo", + "connection": connection["id"], + "from": {"databaseName": "netflow_demo", "tableName": "flows"}, + "timestampValueExpression": "TimeReceived", + "defaultTableSelectExpression": "TimeReceived, SrcAddr, DstAddr, Proto, Bytes, Packets", + "bytesExpression": "Bytes", + "packetsExpression": "Packets", + "samplingRateExpression": "SamplingRate", + "srcAddrExpression": "SrcAddr", + "dstAddrExpression": "DstAddr", + "srcPortExpression": "SrcPort", + "dstPortExpression": "DstPort", + "protocolExpression": "Proto", + "exporterExpression": "ExporterName", + "inIfExpression": "InIfName", + "outIfExpression": "OutIfName", +} +env = { + **os.environ, + "NEXT_PUBLIC_IS_LOCAL_MODE": "true", + "NEXT_PUBLIC_HDX_LOCAL_DEFAULT_CONNECTIONS": json.dumps([connection]), + "NEXT_PUBLIC_HDX_LOCAL_DEFAULT_SOURCES": json.dumps([source]), +} +print(f"NetFlow demo: http://127.0.0.1:{port}/netflow", flush=True) +os.chdir(repo / "packages/app") +os.execvpe( + "node", + [ + "node", + str(repo / "node_modules/next/dist/bin/next"), + "dev", + "--turbopack", + "--hostname", + "127.0.0.1", + "--port", + str(port), + ], + env, +) diff --git a/scripts/netflow-schema.sql b/scripts/netflow-schema.sql new file mode 100644 index 0000000000..820d44f431 --- /dev/null +++ b/scripts/netflow-schema.sql @@ -0,0 +1,34 @@ +CREATE TABLE IF NOT EXISTS netflow_demo.flows +( + TimeReceived DateTime('UTC'), + SamplingRate UInt64, + ExporterAddress IPv6, + ExporterName LowCardinality(String), + ExporterSite LowCardinality(String), + SrcAddr IPv6, + DstAddr IPv6, + SrcAS UInt32, + DstAS UInt32, + SrcCountry FixedString(2), + DstCountry FixedString(2), + InIfName LowCardinality(String), + OutIfName LowCardinality(String), + InIfConnectivity LowCardinality(String), + OutIfConnectivity LowCardinality(String), + InIfProvider LowCardinality(String), + OutIfProvider LowCardinality(String), + InIfSpeed UInt32, + OutIfSpeed UInt32, + InIfBoundary Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), + OutIfBoundary Enum8('undefined' = 0, 'external' = 1, 'internal' = 2), + EType UInt32, + Proto UInt32, + SrcPort UInt16, + DstPort UInt16, + Bytes UInt64, + Packets UInt64 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeReceived) +ORDER BY (TimeReceived, ExporterName, SrcAddr, DstAddr) +COMMENT 'hyperdx-netflow-demo-v1' diff --git a/scripts/netflow-seed.py b/scripts/netflow-seed.py new file mode 100644 index 0000000000..d3ceb2f5bd --- /dev/null +++ b/scripts/netflow-seed.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Seed and verify synthetic Akvorado-compatible flows in local ClickHouse.""" + +import argparse +import base64 +from datetime import datetime, timedelta, timezone +import json +import os +from pathlib import Path +from urllib.error import HTTPError +from urllib.parse import urlparse +from urllib.request import Request, urlopen + + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--url", default=os.getenv("CLICKHOUSE_URL", "http://127.0.0.1:8123") +) +parser.add_argument( + "--end", help="Exclusive UTC end time, e.g. 2026-10-09T20:00:00Z (default: now)" +) +args = parser.parse_args() +if urlparse(args.url).hostname not in ("localhost", "127.0.0.1", "::1"): + parser.error("The demo seeder only writes to a local ClickHouse server") + +end = ( + datetime.fromisoformat(args.end.replace("Z", "+00:00")) + if args.end + else datetime.now(timezone.utc) +) +if end.tzinfo is None: + parser.error("--end must include a timezone") +end = end.astimezone(timezone.utc).replace(microsecond=0) +start = end - timedelta(hours=2) +credentials = base64.b64encode( + f"{os.getenv('CLICKHOUSE_USER', 'default')}:{os.getenv('CLICKHOUSE_PASSWORD', '')}".encode() +).decode() + + +def query(sql): + request = Request( + args.url, + data=sql.encode(), + headers={"Authorization": f"Basic {credentials}"}, + method="POST", + ) + try: + with urlopen(request, timeout=60) as response: + return response.read().decode().strip() + except HTTPError as error: + raise SystemExit(error.read().decode()) from error + + +rows = [] +for i in range(14400): + exporter = i % 3 + scenario = i % 10 + proto = 6 if scenario < 6 else 17 if scenario < 9 else 58 if (i // 10) % 2 else 1 + ipv6 = proto == 58 or (proto != 1 and i % 7 == 0) + inbound = i % 4 == 0 + internal = ( + f"2001:db8:1::{i % 48 + 1:x}" + if ipv6 + else f"::ffff:192.0.2.{i % 48 + 1}" + ) + external = ( + f"2001:db8:2::{i % 24 + 1:x}" + if ipv6 + else f"::ffff:203.0.113.{i % 24 + 1}" + ) + packets = 20 + (i * 17) % 980 + # A short bulk-transfer burst makes the time series useful for investigation. + if 9000 <= i < 10200 and proto == 6: + packets *= 6 + packet_size = 1100 if proto == 6 else 300 if proto == 17 else 84 + service_port = 443 if proto == 6 else 53 if proto == 17 else 0 + client_port = 32768 + i % 28000 if proto in (6, 17) else 0 + connectivity = ("transit", "ix", "pni")[exporter] + provider = ("transit-west", "ix-frankfurt", "peer-east")[exporter] + rows.append( + { + "TimeReceived": (start + timedelta(seconds=i // 2)).strftime( + "%Y-%m-%d %H:%M:%S" + ), + "SamplingRate": (1, 100, 1000)[exporter], + "ExporterAddress": f"::ffff:198.51.100.{exporter + 1}", + "ExporterName": ("edge-sfo-01", "edge-fra-01", "core-iad-01")[exporter], + "ExporterSite": ("sfo", "fra", "iad")[exporter], + "SrcAddr": external if inbound else internal, + "DstAddr": internal if inbound else external, + "SrcAS": 64501 if inbound else 64500, + "DstAS": 64500 if inbound else 64501, + "SrcCountry": "DE" if inbound else "US", + "DstCountry": "US" if inbound else "DE", + "InIfName": "xe-0/0/0" if inbound else "ae1", + "OutIfName": "ae1" if inbound else "xe-0/0/0", + "InIfConnectivity": connectivity if inbound else "", + "OutIfConnectivity": "" if inbound else connectivity, + "InIfProvider": provider if inbound else "", + "OutIfProvider": "" if inbound else provider, + "InIfSpeed": 10000, + "OutIfSpeed": 10000, + "InIfBoundary": "external" if inbound else "internal", + "OutIfBoundary": "internal" if inbound else "external", + "EType": 0x86DD if ipv6 else 0x800, + "Proto": proto, + "SrcPort": service_port if inbound else client_port, + "DstPort": client_port if inbound else service_port, + "Bytes": packets * packet_size, + "Packets": packets, + } + ) + +query("CREATE DATABASE IF NOT EXISTS netflow_demo") +query(Path(__file__).with_name("netflow-schema.sql").read_text()) +owner = query( + "SELECT comment FROM system.tables " + "WHERE database = 'netflow_demo' AND name = 'flows' FORMAT TabSeparated" +) +if owner != "hyperdx-netflow-demo-v1": + raise SystemExit("Refusing to replace a table not owned by this demo seeder") + +query( + "ALTER TABLE netflow_demo.flows " + + ", ".join( + f"ADD COLUMN IF NOT EXISTS {column} LowCardinality(String)" + for column in ( + "InIfConnectivity", "OutIfConnectivity", "InIfProvider", "OutIfProvider" + ) + ) +) + +# This isolated table contains only disposable generated fixtures. +query("TRUNCATE TABLE netflow_demo.flows") +query( + "INSERT INTO netflow_demo.flows FORMAT JSONEachRow\n" + + "\n".join(json.dumps(row) for row in rows) +) +actual = json.loads( + query( + "SELECT count() AS records, sum(Bytes) AS raw_bytes, " + "sum(Packets) AS raw_packets, sum(Bytes * SamplingRate) AS estimated_bytes, " + "sum(Packets * SamplingRate) AS estimated_packets, " + "uniqExact(ExporterName) AS exporters, uniqExact(Proto) AS protocols, " + "countIf(EType = 34525) AS ipv6_records, " + "uniqExact(InIfConnectivity) AS input_connectivity_classes, " + "uniqExact(OutIfConnectivity) AS output_connectivity_classes, " + "uniqExact(InIfProvider) AS input_provider_classes, " + "uniqExact(OutIfProvider) AS output_provider_classes, " + "min(TimeReceived) AS first, max(TimeReceived) AS last " + "FROM netflow_demo.flows FORMAT JSONEachRow" + ) +) +expected = { + "records": len(rows), + "raw_bytes": sum(row["Bytes"] for row in rows), + "raw_packets": sum(row["Packets"] for row in rows), + "estimated_bytes": sum(row["Bytes"] * row["SamplingRate"] for row in rows), + "estimated_packets": sum(row["Packets"] * row["SamplingRate"] for row in rows), + "exporters": 3, + "protocols": 4, + "ipv6_records": sum(row["EType"] == 0x86DD for row in rows), + "input_connectivity_classes": 4, + "output_connectivity_classes": 4, + "input_provider_classes": 4, + "output_provider_classes": 4, +} +for field, value in expected.items(): + if int(actual[field]) != value: + raise SystemExit(f"Verification failed for {field}: {actual[field]} != {value}") +print(json.dumps({"verified": True, "table": "netflow_demo.flows", **actual}, indent=2))