diff --git a/.changeset/denoise-body-column-match.md b/.changeset/denoise-body-column-match.md new file mode 100644 index 0000000000..01987ed9a8 --- /dev/null +++ b/.changeset/denoise-body-column-match.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/app': patch +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +--- + +fix: "Denoise results" now removes noise on typical data. A pattern now counts as noisy above 5% of sampled events instead of 10%, which often matched nothing. Patterns that include error or warning lines are never hidden. The MCP `denoise` option uses the same 5% threshold. Denoise also mines the source's body column instead of the last table column, removes ANSI-colored rows that match a noisy pattern, says when results can't be denoised, and shows hidden patterns as a short summary with a shortened, expandable list. diff --git a/packages/api/src/mcp/__tests__/queryTool.int.test.ts b/packages/api/src/mcp/__tests__/queryTool.int.test.ts index ec654379e6..e47b9cbc2e 100644 --- a/packages/api/src/mcp/__tests__/queryTool.int.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.int.test.ts @@ -568,7 +568,7 @@ describe('MCP Query Tools', () => { beforeEach(async () => { const logs: Parameters[0] = []; - // Noisy pattern: "Health check OK from " — 80 rows (>10% threshold) + // Noisy pattern: "Health check OK from " — 80 of 85 rows for (let i = 0; i < 80; i++) { logs.push({ Body: `Health check OK from 10.0.${Math.floor(i / 256)}.${i % 256}`, @@ -578,7 +578,7 @@ describe('MCP Query Tools', () => { }); } - // Unique/rare events — 5 rows (well below 10% threshold) + // Rare events — 1 row each, well below the 5% threshold for (let i = 0; i < 5; i++) { logs.push({ Body: `Rare event type ${String.fromCharCode(65 + i)} occurred in subsystem`, diff --git a/packages/api/src/mcp/tools/query/denoise.ts b/packages/api/src/mcp/tools/query/denoise.ts index 231cad2401..73a4951a4a 100644 --- a/packages/api/src/mcp/tools/query/denoise.ts +++ b/packages/api/src/mcp/tools/query/denoise.ts @@ -1,10 +1,10 @@ import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; import { - DENOISE_NOISE_THRESHOLD, + buildNoisyBodyMatcher, DENOISE_SAMPLE_SIZE, - flattenBody, minePatterns, + selectNoisyPatterns, } from '@hyperdx/common-utils/dist/drain'; import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { DisplayType } from '@hyperdx/common-utils/dist/types'; @@ -16,8 +16,6 @@ import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout'; import { resolveBodyExpression } from './helpers'; -const PATTERN_WILDCARD = '<*>'; - // ─── Types ─────────────────────────────────────────────────────────────────── export interface DenoiseResult { @@ -40,7 +38,7 @@ export interface DenoiseResult { /** * Denoise search results by mining patterns from a random sample, identifying - * "noisy" patterns (those accounting for >10% of the sample), and filtering + * "noisy" patterns (those accounting for >5% of the sample), and filtering * them out of the result rows. * * This mirrors the web app's "Denoise Results" feature @@ -203,21 +201,17 @@ export async function denoiseSearchResults( return { rows, removedPatterns: [] }; } - // ── Identify noisy patterns (>10% of sampled events) ── - const sampledRowCount = sampleRows.length; - const noisyTemplates = new Set(); - const removedPatterns: DenoiseResult['removedPatterns'] = []; - - for (const p of patterns) { - if (p.sampleCount / sampledRowCount > DENOISE_NOISE_THRESHOLD) { - noisyTemplates.add(p.pattern); - removedPatterns.push({ - pattern: p.pattern, - estimatedCount: p.estimatedCount, - sampleCount: p.sampleCount, - }); - } - } + const noisy = selectNoisyPatterns( + patterns, + p => p.sampleCount, + sampleRows.length, + ); + const noisyTemplates = new Set(noisy.map(p => p.pattern)); + const removedPatterns: DenoiseResult['removedPatterns'] = noisy.map(p => ({ + pattern: p.pattern, + estimatedCount: p.estimatedCount, + sampleCount: p.sampleCount, + })); if (noisyTemplates.size === 0) { return { rows, removedPatterns: [] }; @@ -250,54 +244,6 @@ export async function denoiseSearchResults( // ─── Helpers ───────────────────────────────────────────────────────────────── -/** - * Return a predicate that is true when the most specific template a body - * fits (fewest <*>) is noisy. Fitting means same token count, same literal - * tokens, any token at each <*>. Taking the most specific fit keeps rows of - * a rare `user login` cluster when a broad `<*> <*>` cluster is noisy; ties - * go to the noisy template. Re-running Drain's `match` instead can land a - * member on a different same-length cluster than the one it was counted in. - */ -export function buildNoisyBodyMatcher( - templates: string[], - noisyTemplates: Set, -): (body: string) => boolean { - const candidates = templates.map(template => { - const tokens = tokenize(template); - return { - tokens, - wildcards: tokens.filter(t => t === PATTERN_WILDCARD).length, - noisy: noisyTemplates.has(template), - }; - }); - return body => { - const tokens = tokenize(flattenBody(body)); - let best: (typeof candidates)[number] | undefined; - for (const c of candidates) { - const fits = - c.tokens.length === tokens.length && - c.tokens.every( - (tok, i) => tok === PATTERN_WILDCARD || tok === tokens.at(i), - ); - if (!fits) continue; - if ( - best == null || - c.wildcards < best.wildcards || - (c.wildcards === best.wildcards && c.noisy) - ) { - best = c; - } - } - return best?.noisy ?? false; - }; -} - -// Drain tokenizes with trim + split(/\s+/). -function tokenize(text: string): string[] { - const trimmed = text.trim(); - return trimmed === '' ? [] : trimmed.split(/\s+/); -} - /** * Find the key in a result row that corresponds to the body column expression. * ClickHouse may return the column under its expression name or a simplified alias. diff --git a/packages/api/src/mcp/tools/query/search.ts b/packages/api/src/mcp/tools/query/search.ts index 8b77927fe1..9608eecb43 100644 --- a/packages/api/src/mcp/tools/query/search.ts +++ b/packages/api/src/mcp/tools/query/search.ts @@ -46,7 +46,7 @@ const searchSchema = z.object({ .default(false) .describe( 'When true, automatically removes events matching high-frequency patterns ' + - '(those accounting for >10% of sampled events) from the results. ' + + '(those accounting for >5% of sampled events) from the results. ' + 'Useful for cutting through log noise to find unusual or interesting events. ' + 'Adds ~1-2s of latency for the pattern sampling queries.', ), diff --git a/packages/app/src/components/DBRowTable.tsx b/packages/app/src/components/DBRowTable.tsx index 51ed633ac2..2c90cbf204 100644 --- a/packages/app/src/components/DBRowTable.tsx +++ b/packages/app/src/components/DBRowTable.tsx @@ -10,7 +10,6 @@ import cx from 'classnames'; import { formatDistance } from 'date-fns'; import { isString } from 'lodash'; import curry from 'lodash/curry'; -import ms from 'ms'; import { useHotkeys } from 'react-hotkeys-hook'; import { Bar, @@ -30,10 +29,6 @@ import { JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; -import { - DENOISE_NOISE_THRESHOLD, - DENOISE_SAMPLE_SIZE, -} from '@hyperdx/common-utils/dist/drain'; import { BuilderChartConfigWithDateRange, SelectList, @@ -57,7 +52,7 @@ import { IconTextWrap, IconTextWrapDisabled, } from '@tabler/icons-react'; -import { FetchNextPageOptions, useQuery } from '@tanstack/react-query'; +import { FetchNextPageOptions } from '@tanstack/react-query'; import { ColumnDef, ColumnResizeMode, @@ -78,13 +73,13 @@ import { useRenderedSqlChartConfig, } from '@/hooks/useChartConfig'; import { CsvColumn, useCsvExport } from '@/hooks/useCsvExport'; +import { useDenoisedRows } from '@/hooks/useDenoisedRows'; import { useColumns, useMergeTreeSettings, useTableMetadata, } from '@/hooks/useMetadata'; import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; -import { useGroupedPatterns } from '@/hooks/usePatterns'; import { useRowSelection } from '@/hooks/useRowSelection'; import useRowWhere, { getRowId, @@ -93,7 +88,7 @@ import useRowWhere, { WithClause, } from '@/hooks/useRowWhere'; import { useTableSearch } from '@/hooks/useTableSearch'; -import { getLevelExpression, useSource } from '@/source'; +import { useSource } from '@/source'; import { MIN_COLUMN_WIDTH, MIN_LAST_COLUMN_WIDTH, @@ -137,6 +132,7 @@ import { import { SQLPreview } from './ChartSQLPreview'; import { CsvExportButton } from './CsvExportButton'; import { RowSidePanelContext } from './DBRowSidePanel'; +import { DenoiseSummary } from './DenoiseSummary'; import { createExpandButtonColumn, ExpandedLogRow, @@ -1948,87 +1944,15 @@ function DBSqlRowTableComponent({ }, [data?.meta, onResolvedColumnsChange]); const { data: source } = useSource({ id: sourceId }); - const patternColumn = columns[columns.length - 1]; - const groupedPatterns = useGroupedPatterns({ + const denoised = useDenoisedRows({ config, - samples: DENOISE_SAMPLE_SIZE, - bodyValueExpression: patternColumn ?? '', - levelExpression: getLevelExpression(source), - totalCount: undefined, + source, + columns, + rows: processedRows, enabled: denoiseResults, }); - const noisyPatterns = useQuery({ - queryKey: ['noisy-patterns', config], - queryFn: async () => { - return Object.values(groupedPatterns.data).filter( - p => - p.count / (groupedPatterns.sampledRowCount ?? 1) > - DENOISE_NOISE_THRESHOLD, - ); - }, - enabled: - denoiseResults && - groupedPatterns.data != null && - Object.values(groupedPatterns.data).length > 0 && - groupedPatterns.miner != null, - }); - const noisyPatternIds = useMemo(() => { - return noisyPatterns.data?.map(p => p.id) ?? []; - }, [noisyPatterns.data]); - - const denoisedRows = useQuery({ - queryKey: [ - 'denoised-rows', - config, - denoiseResults, - // Only include processed rows if denoising is enabled - // This helps prevent the queryKey from getting extremely large - // and causing memory issues, when it's not used. - ...(denoiseResults ? [processedRows] : []), - noisyPatternIds, - patternColumn, - ], - queryFn: async () => { - if (!denoiseResults) { - return []; - } - // No noisy patterns, so no need to denoise - if (noisyPatternIds.length === 0) { - return processedRows; - } - - const matchedLogs = await groupedPatterns.miner?.matchLogs( - processedRows.map(row => row[patternColumn]), - ); - return processedRows.filter((row, i) => { - const match = matchedLogs?.[i]; - return !noisyPatternIds.includes(`${match}`); - }); - }, - placeholderData: (previousData, previousQuery) => { - // If it's the same search, but new data, return the previous data while we load - if ( - previousQuery?.queryKey?.[0] === 'denoised-rows' && - previousQuery?.queryKey?.[1] === config - ) { - return previousData; - } - return undefined; - }, - gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data - enabled: - denoiseResults && - noisyPatterns.isSuccess && - processedRows.length > 0 && - groupedPatterns.miner != null, - }); - const isLoading = denoiseResults - ? isFetching || - denoisedRows.isFetching || - noisyPatterns.isFetching || - groupedPatterns.isLoading - : isFetching; + const isLoading = isFetching || denoised.isLoading; const loadingDate = data?.window?.direction === 'ASC' @@ -2038,28 +1962,20 @@ function DBSqlRowTableComponent({ return ( <> {denoiseResults && ( - - - Removed Noisy Event Patterns - - - {noisyPatterns.data?.map(p => ( - - {p.pattern} - - ))} - {noisyPatternIds.length === 0 && ( - No noisy patterns found - )} - - + )} + `${Math.max(Math.round((count / total) * 100), 1)}%`; + +export function DenoiseSummary({ + noisyPatterns, + sampledRowCount, + skipped, + bodyExpression, + isLoading, +}: { + noisyPatterns: { pattern: string; count: number }[]; + sampledRowCount: number; + skipped: DenoiseSkipReason | undefined; + bodyExpression: string | undefined; + isLoading: boolean; +}) { + const [opened, { toggle }] = useDisclosure(false); + + // Patterns that read the same once simplified are shown once + const lines = useMemo(() => { + const byText = new Map(); + for (const p of noisyPatterns) { + const text = simplifyPattern(p.pattern); + const line = byText.get(text); + if (line) line.count += p.count; + else byText.set(text, { full: p.pattern, count: p.count }); + } + return [...byText.entries()] + .map(([text, line]) => ({ text, ...line })) + .sort((a, b) => b.count - a.count); + }, [noisyPatterns]); + + const hiddenCount = noisyPatterns.reduce((sum, p) => sum + p.count, 0); + + let summary: string; + if (skipped != null) { + summary = skipMessage(skipped, bodyExpression); + } else if (isLoading) { + summary = 'Finding common patterns…'; + } else if (lines.length === 0) { + summary = 'No common patterns to hide.'; + } else { + summary = + `Hiding ${lines.length} common ${lines.length === 1 ? 'pattern' : 'patterns'}, ` + + `about ${percent(hiddenCount, sampledRowCount)} of events. ` + + 'Errors and warnings are never hidden.'; + } + + const canExpand = skipped == null && !isLoading && lines.length > 0; + + return ( + + + + {summary} + + {canExpand && ( + + )} + + {canExpand && ( + + + {lines.map(line => ( + + + {percent(line.count, sampledRowCount)} + + + {line.text} + + + ))} + + + )} + + ); +} diff --git a/packages/app/src/components/Patterns/__tests__/simplifyPattern.test.ts b/packages/app/src/components/Patterns/__tests__/simplifyPattern.test.ts new file mode 100644 index 0000000000..de708d8936 --- /dev/null +++ b/packages/app/src/components/Patterns/__tests__/simplifyPattern.test.ts @@ -0,0 +1,29 @@ +import { simplifyPattern } from '@/components/Patterns/simplifyPattern'; + +describe('simplifyPattern', () => { + it('collapses generalized key=value pairs into one ellipsis', () => { + expect( + simplifyPattern( + 'msg="Product found" catalog.brand=<*> <*>.<*>_<*>=<*> <*>.<*>_<*>="<*> <*> <*>" catalog.rating=<*>.<*> grpc.status_code=OK', + ), + ).toBe( + 'msg="Product found" catalog.brand=… catalog.rating=… grpc.status_code=OK', + ); + }); + + it('keeps quotes and text around a wildcard run', () => { + expect( + simplifyPattern( + '[2026-04-16T20:<*>:<*>.<*>] "GET /api/products/<*> HTTP/1.1" 200 0 <*> <*> <*> "-" "python-requests/2.32.3" "<*>-<*>-<*>-<*>-<*>" 10.116.<*>.<*>:<*> - -', + ), + ).toBe( + '[2026-04-16T20:…] "GET /api/products/… HTTP/1.1" 200 0 … "-" "python-requests/2.32.3" "…" 10.116.… - -', + ); + }); + + it('leaves patterns without wildcards unchanged', () => { + expect(simplifyPattern('GetCartAsync called with userId={userId}')).toBe( + 'GetCartAsync called with userId={userId}', + ); + }); +}); diff --git a/packages/app/src/components/Patterns/simplifyPattern.ts b/packages/app/src/components/Patterns/simplifyPattern.ts new file mode 100644 index 0000000000..b8537f492f --- /dev/null +++ b/packages/app/src/components/Patterns/simplifyPattern.ts @@ -0,0 +1,20 @@ +// A run of wildcards joined only by spaces and punctuation, plus a closing +// quote right after it. +const WILDCARD_RUN = /<\*>(?:[^A-Za-z0-9<]*<\*>)*("?)/g; + +/** + * A pattern template shortened for display: each run of wildcards, such as + * `<*>.<*>_<*>=<*> <*>` (key=value pairs Drain fully generalized), becomes a + * single `…`. + */ +export function simplifyPattern(pattern: string): string { + return pattern + .replace(WILDCARD_RUN, (run, closingQuote: string) => { + const innerQuotes = (run.match(/"/g)?.length ?? 0) - closingQuote.length; + // An odd count means the run opened a quote, so its closing quote + // goes with it. Otherwise the closing quote belongs to the text. + return innerQuotes % 2 === 1 ? '…' : `…${closingQuote}`; + }) + .replace(/\s+/g, ' ') + .trim(); +} diff --git a/packages/app/src/components/__tests__/DenoiseSummary.test.tsx b/packages/app/src/components/__tests__/DenoiseSummary.test.tsx new file mode 100644 index 0000000000..b9e158d81e --- /dev/null +++ b/packages/app/src/components/__tests__/DenoiseSummary.test.tsx @@ -0,0 +1,55 @@ +import { fireEvent, screen } from '@testing-library/react'; + +import { DenoiseSummary } from '@/components/DenoiseSummary'; + +describe('DenoiseSummary', () => { + const base = { + sampledRowCount: 100, + skipped: undefined, + bodyExpression: 'Body', + isLoading: false, + }; + + it('summarizes hidden patterns and lists them on demand', () => { + renderWithMantine( + <*> logged in', count: 10 }, + { pattern: 'user <*> logged in', count: 12 }, + ]} + />, + ); + expect( + screen.getByText( + 'Hiding 2 common patterns, about 52% of events. Errors and warnings are never hidden.', + ), + ).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Show patterns' })); + expect(screen.getByText('user … logged in')).toBeInTheDocument(); + expect(screen.getByText('22%')).toBeInTheDocument(); + expect(screen.getByText('30%')).toBeInTheDocument(); + }); + + it('says when it is still loading', () => { + renderWithMantine( + , + ); + expect(screen.getByText('Finding common patterns…')).toBeInTheDocument(); + expect(screen.queryByRole('button')).not.toBeInTheDocument(); + }); + + it('explains why results were not denoised', () => { + renderWithMantine( + , + ); + expect( + screen.getByText('Add Body to the table columns to denoise results.'), + ).toBeInTheDocument(); + }); +}); diff --git a/packages/app/src/hooks/__tests__/useDenoisedRows.test.tsx b/packages/app/src/hooks/__tests__/useDenoisedRows.test.tsx new file mode 100644 index 0000000000..252fe5f714 --- /dev/null +++ b/packages/app/src/hooks/__tests__/useDenoisedRows.test.tsx @@ -0,0 +1,176 @@ +import { + BuilderChartConfigWithDateRange, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { renderHook } from '@testing-library/react'; + +import { findBodyResultColumn, useDenoisedRows } from '@/hooks/useDenoisedRows'; +import { useGroupedPatterns } from '@/hooks/usePatterns'; + +jest.mock('@/hooks/usePatterns', () => ({ + LEVEL_COLUMN_ALIAS: '__hdx_level', + DRAIN_EXTRA_DELIMITERS: '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'.split(''), + useGroupedPatterns: jest.fn(), +})); + +const mockUseGroupedPatterns = jest.mocked(useGroupedPatterns); + +const source = { + id: 'logs', + kind: SourceKind.Log, + bodyExpression: 'Body', + severityTextExpression: 'SeverityText', +} as unknown as TSource; + +const config = (select: string): BuilderChartConfigWithDateRange => ({ + select, + from: { databaseName: 'default', tableName: 'otel_logs' }, + where: '', + timestampValueExpression: 'Timestamp', + connection: 'local', + dateRange: [new Date('2026-01-01'), new Date('2026-01-02')], +}); + +describe('findBodyResultColumn', () => { + it('finds the body by its position in the select list', () => { + expect( + findBodyResultColumn( + "Timestamp, Body AS msg, LogAttributes['k'] AS k", + ['Timestamp', 'msg', 'k'], + 'Body', + ), + ).toBe('msg'); + }); + + it('falls back to the column name for SELECT *', () => { + expect(findBodyResultColumn('*', ['Timestamp', 'Body'], 'Body')).toBe( + 'Body', + ); + }); + + it('returns undefined when the body is not selected', () => { + expect( + findBodyResultColumn( + 'Timestamp, ServiceName', + ['Timestamp', 'ServiceName'], + 'Body', + ), + ).toBeUndefined(); + }); +}); + +const patternQueryState = { + isLoading: false, + error: null, + miner: undefined, + patternQueryConfig: undefined, +}; + +describe('useDenoisedRows', () => { + beforeEach(() => { + mockUseGroupedPatterns.mockReturnValue({ + data: { + '1': { + id: '1', + pattern: 'info cache refreshed for <*>', + count: 80, + samples: [{ __hdx_level: 'info' }], + }, + '2': { + id: '2', + pattern: 'user <*> logged in', + count: 5, + samples: [{ __hdx_level: 'info' }], + }, + }, + sampledRowCount: 100, + clusterTemplates: { + '1': 'info cache refreshed for <*>', + '2': 'user <*> logged in', + }, + ...patternQueryState, + }); + }); + + it('removes noisy rows by the body column, not the last column', () => { + const rows = [ + { Body: '\x1b[32minfo\x1b[39m: cache refreshed for ab12', k: 'x' }, + { Body: 'user bob logged in', k: 'y' }, + { Body: null, k: 'z' }, + ]; + const { result } = renderHook(() => + useDenoisedRows({ + config: config("Timestamp, Body, LogAttributes['k'] AS k"), + source, + columns: ['Timestamp', 'Body', 'k'], + rows, + enabled: true, + }), + ); + expect(mockUseGroupedPatterns).toHaveBeenLastCalledWith( + expect.objectContaining({ bodyValueExpression: 'Body', enabled: true }), + ); + expect(result.current.skipped).toBeUndefined(); + expect(result.current.noisyPatterns.map(p => p.id)).toEqual(['1']); + expect(result.current.rows).toEqual([rows[1], rows[2]]); + }); + + it('never hides a pattern with error or warning samples', () => { + mockUseGroupedPatterns.mockReturnValue({ + ...patternQueryState, + sampledRowCount: 100, + data: { + '1': { + id: '1', + pattern: 'connection refused to <*>', + count: 80, + samples: [{ __hdx_level: 'INFO' }, { __hdx_level: 'ERROR' }], + }, + '2': { + id: '2', + pattern: 'user <*> logged in', + count: 20, + samples: [{ __hdx_level: 'info' }], + }, + }, + clusterTemplates: { + '1': 'connection refused to <*>', + '2': 'user <*> logged in', + }, + }); + const rows = [ + { Body: 'connection refused to db1' }, + { Body: 'user bob logged in' }, + ]; + const { result } = renderHook(() => + useDenoisedRows({ + config: config('Timestamp, Body'), + source, + columns: ['Timestamp', 'Body'], + rows, + enabled: true, + }), + ); + expect(result.current.noisyPatterns.map(p => p.id)).toEqual(['2']); + expect(result.current.rows).toEqual([rows[0]]); + }); + + it('skips denoising when the body is not in the results', () => { + const rows = [{ ServiceName: 'api' }]; + const { result } = renderHook(() => + useDenoisedRows({ + config: config('Timestamp, ServiceName'), + source, + columns: ['Timestamp', 'ServiceName'], + rows, + enabled: true, + }), + ); + expect(mockUseGroupedPatterns).toHaveBeenLastCalledWith( + expect.objectContaining({ enabled: false }), + ); + expect(result.current.skipped).toBe('body_not_in_results'); + expect(result.current.rows).toBe(rows); + }); +}); diff --git a/packages/app/src/hooks/useDenoisedRows.ts b/packages/app/src/hooks/useDenoisedRows.ts new file mode 100644 index 0000000000..7ba421cc59 --- /dev/null +++ b/packages/app/src/hooks/useDenoisedRows.ts @@ -0,0 +1,148 @@ +import { useMemo } from 'react'; +import stripAnsi from 'strip-ansi'; +import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; +import { + buildNoisyBodyMatcher, + DENOISE_SAMPLE_SIZE, + selectNoisyPatterns, +} from '@hyperdx/common-utils/dist/drain'; +import { + BuilderChartConfigWithDateRange, + TSource, +} from '@hyperdx/common-utils/dist/types'; + +import { + DRAIN_EXTRA_DELIMITERS, + LEVEL_COLUMN_ALIAS, + useGroupedPatterns, +} from '@/hooks/usePatterns'; +import { getEventBody, getLevelExpression } from '@/source'; +import { getLogLevelClass, selectItemExpression } from '@/utils'; + +function isErrorOrWarning(row: Record) { + const level = row[LEVEL_COLUMN_ALIAS]; + if (typeof level !== 'string') return false; + const levelClass = getLogLevelClass(level); + return levelClass === 'error' || levelClass === 'warn'; +} + +export type DenoiseSkipReason = + | 'no_body_expression' + | 'body_not_in_results' + | 'patterns_failed'; + +/** + * The result column holding the source's body. `columns` follow the order of + * the SELECT items, and ClickHouse may rename a column, so the body is located + * by its position in the SELECT list, then by name (e.g. for `SELECT *`). + */ +export function findBodyResultColumn( + select: BuilderChartConfigWithDateRange['select'], + columns: string[], + bodyExpression: string | undefined, +): string | undefined { + if (!bodyExpression) return undefined; + if (typeof select === 'string') { + const index = splitAndTrimWithBracket(select).findIndex( + item => selectItemExpression(item) === bodyExpression, + ); + if (index !== -1 && index < columns.length) return columns[index]; + } + return columns.includes(bodyExpression) ? bodyExpression : undefined; +} + +export function useDenoisedRows>({ + config, + source, + columns, + rows, + enabled, +}: { + config: BuilderChartConfigWithDateRange; + source: TSource | undefined; + columns: string[]; + rows: TRow[]; + enabled: boolean; +}) { + const bodyExpression = source ? getEventBody(source) : undefined; + const bodyColumn = findBodyResultColumn( + config.select, + columns, + bodyExpression, + ); + const groupedPatterns = useGroupedPatterns({ + config, + samples: DENOISE_SAMPLE_SIZE, + bodyValueExpression: bodyExpression ?? '', + levelExpression: getLevelExpression(source), + totalCount: undefined, + enabled: enabled && bodyColumn != null, + }); + + const { + data: patternGroups, + sampledRowCount, + clusterTemplates, + } = groupedPatterns; + + let skipped: DenoiseSkipReason | undefined; + if (!enabled) { + // Not denoising, nothing to skip + } else if (!bodyExpression) { + skipped = 'no_body_expression'; + } else if (columns.length > 0 && bodyColumn == null) { + // Columns are unknown until the first page arrives + skipped = 'body_not_in_results'; + } else if (groupedPatterns.error != null) { + skipped = 'patterns_failed'; + } + + // Frequency is not importance: a flood of errors during an incident must + // stay visible, so patterns with any error or warning sample are kept. + const noisyPatterns = useMemo( + () => + selectNoisyPatterns( + Object.values(patternGroups).filter( + p => !p.samples.some(isErrorOrWarning), + ), + p => p.count, + sampledRowCount ?? 0, + ), + [patternGroups, sampledRowCount], + ); + + const isNoisy = useMemo(() => { + if (clusterTemplates == null) return undefined; + const noisyTemplates = new Set( + noisyPatterns + .map(p => clusterTemplates[p.id]) + .filter((t): t is string => t != null), + ); + return buildNoisyBodyMatcher( + Object.values(clusterTemplates), + noisyTemplates, + { extraDelimiters: DRAIN_EXTRA_DELIMITERS }, + ); + }, [clusterTemplates, noisyPatterns]); + + const denoisedRows = useMemo(() => { + if (!enabled) return rows; + if (skipped != null) return rows; + if (isNoisy == null || bodyColumn == null) return []; + if (noisyPatterns.length === 0) return rows; + return rows.filter(row => { + const body = row[bodyColumn]; + if (body == null) return true; + return !isNoisy(stripAnsi(String(body))); + }); + }, [enabled, skipped, isNoisy, bodyColumn, noisyPatterns, rows]); + + return { + rows: denoisedRows, + bodyExpression, + noisyPatterns, + sampledRowCount: sampledRowCount ?? 0, + skipped, + isLoading: enabled && skipped == null && clusterTemplates == null, + }; +} diff --git a/packages/app/src/hooks/usePatterns.tsx b/packages/app/src/hooks/usePatterns.tsx index ece24e70cd..57a42e7267 100644 --- a/packages/app/src/hooks/usePatterns.tsx +++ b/packages/app/src/hooks/usePatterns.tsx @@ -43,6 +43,11 @@ function usePyodide(options: { enabled: boolean }) { }); } +// Python's string.punctuation. Shared with the denoise matcher, which must +// split bodies into the same tokens the miner built its templates from. +export const DRAIN_EXTRA_DELIMITERS = + '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'.split(''); + class Miner { private minerVariableName; private id; @@ -55,12 +60,11 @@ class Miner { await this.pyodide.runPythonAsync(` import js import json -import string from drain3 import TemplateMiner from drain3.template_miner_config import TemplateMinerConfig _config = TemplateMinerConfig() -_config.drain_extra_delimiters = list(string.punctuation) +_config.drain_extra_delimiters = json.loads(${JSON.stringify(JSON.stringify(DRAIN_EXTRA_DELIMITERS))}) ${this.minerVariableName} = TemplateMiner(None, _config) `); @@ -80,18 +84,11 @@ ${this.minerVariableName} = TemplateMiner(None, _config) ); } - async matchLogs(logs: string[]) { - const tempLogs = `temp_logs_${this.id}`; - const tempResults = `temp_results_${this.id}`; - this.pyodide.globals.set(tempLogs, logs); - + /** Final template of every cluster, keyed by cluster id. */ + async getTemplates(): Promise> { return JSON.parse( await this.pyodide.runPythonAsync(` - ${tempResults} = [] - for log in ${tempLogs}: - match = ${this.minerVariableName}.match(log) - ${tempResults}.append(match.cluster_id if match else None) - json.dumps(${tempResults}) + json.dumps({str(c.cluster_id): c.get_template() for c in ${this.minerVariableName}.drain.clusters}) `), ); } @@ -100,9 +97,11 @@ ${this.minerVariableName} = TemplateMiner(None, _config) async function mineEventPatterns(logs: string[], pyodide: any) { const miner = new Miner(pyodide); await miner.init(); + const patterns = await miner.minePatterns(logs); return { miner, - patterns: await miner.minePatterns(logs), + patterns, + templates: await miner.getTemplates(), }; } @@ -205,6 +204,7 @@ function usePatterns({ additionalKeysLength: configWithPrimaryAndPartitionKey.additionalKeysLength, miner: response.miner, + clusterTemplates: response.templates, }; }); }, @@ -335,6 +335,7 @@ export function useGroupedPatterns({ isLoading, error, miner: results?.miner, + clusterTemplates: results?.clusterTemplates, sampledRowCount, patternQueryConfig, }; diff --git a/packages/api/src/mcp/tools/query/__tests__/denoise.test.ts b/packages/common-utils/src/__tests__/noisy-matcher.test.ts similarity index 73% rename from packages/api/src/mcp/tools/query/__tests__/denoise.test.ts rename to packages/common-utils/src/__tests__/noisy-matcher.test.ts index 20393f6d4a..524518f99d 100644 --- a/packages/api/src/mcp/tools/query/__tests__/denoise.test.ts +++ b/packages/common-utils/src/__tests__/noisy-matcher.test.ts @@ -1,6 +1,8 @@ -import { minePatterns } from '@hyperdx/common-utils/dist/drain'; - -import { buildNoisyBodyMatcher } from '@/mcp/tools/query/denoise'; +import { minePatterns } from '@/drain/mine-patterns'; +import { + buildNoisyBodyMatcher, + selectNoisyPatterns, +} from '@/drain/noisy-matcher'; function lcg(seed: number) { let s = seed; @@ -105,4 +107,37 @@ describe('buildNoisyBodyMatcher', () => { ); expect(isNoisy('a b c')).toBe(true); }); + + it('splits bodies on the miner delimiters', () => { + const isNoisy = buildNoisyBodyMatcher( + ['GET api v1 <*> 200 <*>', 'user <*> logged in'], + new Set(['GET api v1 <*> 200 <*>']), + { extraDelimiters: ['/', '.', ':'] }, + ); + expect(isNoisy('GET /api/v1/search 200 12ms')).toBe(true); + expect(isNoisy('user bob logged in')).toBe(false); + expect(isNoisy('GET /api/v1/search 500 12ms')).toBe(false); + }); +}); + +describe('selectNoisyPatterns', () => { + const select = (counts: number[], sampleSize = 100) => + selectNoisyPatterns( + counts.map((count, i) => ({ id: i, count })), + p => p.count, + sampleSize, + ).map(p => p.id); + + it('takes patterns above 5% of the sample, most frequent first', () => { + // No pattern passes 10% on its own, as in the demo logs + expect(select([6, 9, 5, 7, 4, 3, 66])).toEqual([6, 1, 3, 0]); + }); + + it('keeps patterns at exactly 5%', () => { + expect(select([5, 95])).toEqual([1]); + }); + + it('returns nothing for an empty sample', () => { + expect(select([], 0)).toEqual([]); + }); }); diff --git a/packages/common-utils/src/drain/index.ts b/packages/common-utils/src/drain/index.ts index f735377f06..7615f11099 100644 --- a/packages/common-utils/src/drain/index.ts +++ b/packages/common-utils/src/drain/index.ts @@ -17,5 +17,6 @@ export { minePatterns, } from './mine-patterns'; export { Node } from './node'; +export { buildNoisyBodyMatcher, selectNoisyPatterns } from './noisy-matcher'; export type { AddLogMessageResult, ExtractedParameter } from './template-miner'; export { TemplateMiner } from './template-miner'; diff --git a/packages/common-utils/src/drain/mine-patterns.ts b/packages/common-utils/src/drain/mine-patterns.ts index f5f663962d..c33fa7dc1b 100644 --- a/packages/common-utils/src/drain/mine-patterns.ts +++ b/packages/common-utils/src/drain/mine-patterns.ts @@ -14,8 +14,12 @@ import { TemplateMiner } from './template-miner'; /** Number of random rows to sample for pattern learning. */ export const DENOISE_SAMPLE_SIZE = 10_000; -/** Patterns matching more than this fraction of sampled events are "noisy". */ -export const DENOISE_NOISE_THRESHOLD = 0.1; +/** + * Patterns matching more than this fraction of sampled events are "noisy". + * Drain splits one logical pattern across clusters (by token count and + * first token), so dominant patterns often sit well below 10% each. + */ +export const DENOISE_NOISE_THRESHOLD = 0.05; // ─── Body normalization ────────────────────────────────────────────────────── diff --git a/packages/common-utils/src/drain/noisy-matcher.ts b/packages/common-utils/src/drain/noisy-matcher.ts new file mode 100644 index 0000000000..45ef61f5c0 --- /dev/null +++ b/packages/common-utils/src/drain/noisy-matcher.ts @@ -0,0 +1,77 @@ +import { DENOISE_NOISE_THRESHOLD, flattenBody } from './mine-patterns'; + +const PATTERN_WILDCARD = '<*>'; + +/** + * The patterns to remove when denoising: those matching more than + * `DENOISE_NOISE_THRESHOLD` of the sample, most frequent first. + */ +export function selectNoisyPatterns( + patterns: T[], + getCount: (pattern: T) => number, + sampleSize: number, +): T[] { + if (sampleSize <= 0) return []; + return patterns + .filter(p => getCount(p) / sampleSize > DENOISE_NOISE_THRESHOLD) + .sort((a, b) => getCount(b) - getCount(a)); +} + +/** + * Return a predicate that is true when the most specific template a body + * fits (fewest <*>) is noisy. Fitting means same token count, same literal + * tokens, any token at each <*>. Taking the most specific fit keeps rows of + * a rare `user login` cluster when a broad `<*> <*>` cluster is noisy; ties + * go to the noisy template. Re-running Drain's `match` instead can land a + * member on a different same-length cluster than the one it was counted in. + * + * `extraDelimiters` must be the ones the miner was configured with, so the + * body splits into the same tokens the templates were built from. + */ +export function buildNoisyBodyMatcher( + templates: string[], + noisyTemplates: Set, + options?: { extraDelimiters?: string[] }, +): (body: string) => boolean { + const extraDelimiters = options?.extraDelimiters ?? []; + const candidates = templates.map(template => { + const tokens = tokenize(template, []); + return { + tokens, + wildcards: tokens.filter(t => t === PATTERN_WILDCARD).length, + noisy: noisyTemplates.has(template), + }; + }); + return body => { + const tokens = tokenize(flattenBody(body), extraDelimiters); + let best: (typeof candidates)[number] | undefined; + for (const c of candidates) { + const fits = + c.tokens.length === tokens.length && + c.tokens.every( + (tok, i) => tok === PATTERN_WILDCARD || tok === tokens.at(i), + ); + if (!fits) continue; + if ( + best == null || + c.wildcards < best.wildcards || + (c.wildcards === best.wildcards && c.noisy) + ) { + best = c; + } + } + return best?.noisy ?? false; + }; +} + +// Mirrors Drain's tokenizer: delimiters become spaces, then split(/\s+/). +// Templates are already space-joined tokens, and splitting one on a +// delimiter would break up `<*>`. +function tokenize(text: string, extraDelimiters: string[]): string[] { + let c = text.trim(); + for (const delimiter of extraDelimiters) { + c = c.split(delimiter).join(' '); + } + c = c.trim(); + return c === '' ? [] : c.split(/\s+/); +}