fix: make "Denoise results" remove noise reliably

.changeset/denoise-body-column-match.md

@@ -0,0 +1,7 @@
+---
+'@hyperdx/app': patch
+'@hyperdx/common-utils': patch
+'@hyperdx/api': patch
+---
+
+fix: "Denoise results" now removes noise on typical data. Noisy patterns are the most frequent ones until they cover half of the sampled events, instead of only patterns above 10% each, which often matched nothing. Patterns that include error or warning lines are never hidden. The same coverage rule applies to the MCP `denoise` option. Denoise also mines the source's body column instead of the last table column, removes ANSI-colored rows that match a noisy pattern, says when results can't be denoised, and shows hidden patterns as a short summary with a shortened, expandable list.

packages/api/src/mcp/__tests__/queryTool.int.test.ts

@@ -568,7 +568,7 @@ describe('MCP Query Tools', () => {
       beforeEach(async () => {
         const logs: Parameters<typeof bulkInsertLogs>[0] = [];
 
-        // Noisy pattern: "Health check OK from <ip>" — 80 rows (>10% threshold)
+        // Noisy pattern: "Health check OK from <ip>" — 80 of 85 rows
         for (let i = 0; i < 80; i++) {
           logs.push({
             Body: `Health check OK from 10.0.${Math.floor(i / 256)}.${i % 256}`,
@@ -578,7 +578,7 @@ describe('MCP Query Tools', () => {
           });
         }
 
-        // Unique/rare events — 5 rows (well below 10% threshold)
+        // Rare events — kept, as the health check alone covers half the sample
         for (let i = 0; i < 5; i++) {
           logs.push({
             Body: `Rare event type ${String.fromCharCode(65 + i)} occurred in subsystem`,

packages/api/src/mcp/tools/query/denoise.ts

@@ -1,10 +1,10 @@
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils';
 import {
-  DENOISE_NOISE_THRESHOLD,
+  buildNoisyBodyMatcher,
   DENOISE_SAMPLE_SIZE,
-  flattenBody,
   minePatterns,
+  selectNoisyPatterns,
 } from '@hyperdx/common-utils/dist/drain';
 import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
 import { DisplayType } from '@hyperdx/common-utils/dist/types';
@@ -16,8 +16,6 @@ import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout';
 
 import { resolveBodyExpression } from './helpers';
 
-const PATTERN_WILDCARD = '<*>';
-
 // ─── Types ───────────────────────────────────────────────────────────────────
 
 export interface DenoiseResult {
@@ -40,8 +38,8 @@ export interface DenoiseResult {
 
 /**
  * Denoise search results by mining patterns from a random sample, identifying
- * "noisy" patterns (those accounting for >10% of the sample), and filtering
- * them out of the result rows.
+ * "noisy" patterns (the most frequent ones, up to half of the sample), and
+ * filtering them out of the result rows.
  *
  * This mirrors the web app's "Denoise Results" feature
  * (packages/app/src/components/DBRowTable.tsx) but runs server-side using
@@ -203,21 +201,17 @@ export async function denoiseSearchResults(
     return { rows, removedPatterns: [] };
   }
 
-  // ── Identify noisy patterns (>10% of sampled events) ──
-  const sampledRowCount = sampleRows.length;
-  const noisyTemplates = new Set<string>();
-  const removedPatterns: DenoiseResult['removedPatterns'] = [];
-
-  for (const p of patterns) {
-    if (p.sampleCount / sampledRowCount > DENOISE_NOISE_THRESHOLD) {
-      noisyTemplates.add(p.pattern);
-      removedPatterns.push({
-        pattern: p.pattern,
-        estimatedCount: p.estimatedCount,
-        sampleCount: p.sampleCount,
-      });
-    }
-  }
+  const noisy = selectNoisyPatterns(
+    patterns,
+    p => p.sampleCount,
+    sampleRows.length,
+  );
+  const noisyTemplates = new Set(noisy.map(p => p.pattern));
+  const removedPatterns: DenoiseResult['removedPatterns'] = noisy.map(p => ({
+    pattern: p.pattern,
+    estimatedCount: p.estimatedCount,
+    sampleCount: p.sampleCount,
+  }));
 
   if (noisyTemplates.size === 0) {
     return { rows, removedPatterns: [] };
@@ -250,54 +244,6 @@ export async function denoiseSearchResults(
 
 // ─── Helpers ─────────────────────────────────────────────────────────────────
 
-/**
- * Return a predicate that is true when the most specific template a body
- * fits (fewest <*>) is noisy. Fitting means same token count, same literal
- * tokens, any token at each <*>. Taking the most specific fit keeps rows of
- * a rare `user login` cluster when a broad `<*> <*>` cluster is noisy; ties
- * go to the noisy template. Re-running Drain's `match` instead can land a
- * member on a different same-length cluster than the one it was counted in.
- */
-export function buildNoisyBodyMatcher(
-  templates: string[],
-  noisyTemplates: Set<string>,
-): (body: string) => boolean {
-  const candidates = templates.map(template => {
-    const tokens = tokenize(template);
-    return {
-      tokens,
-      wildcards: tokens.filter(t => t === PATTERN_WILDCARD).length,
-      noisy: noisyTemplates.has(template),
-    };
-  });
-  return body => {
-    const tokens = tokenize(flattenBody(body));
-    let best: (typeof candidates)[number] | undefined;
-    for (const c of candidates) {
-      const fits =
-        c.tokens.length === tokens.length &&
-        c.tokens.every(
-          (tok, i) => tok === PATTERN_WILDCARD || tok === tokens.at(i),
-        );
-      if (!fits) continue;
-      if (
-        best == null ||
-        c.wildcards < best.wildcards ||
-        (c.wildcards === best.wildcards && c.noisy)
-      ) {
-        best = c;
-      }
-    }
-    return best?.noisy ?? false;
-  };
-}
-
-// Drain tokenizes with trim + split(/\s+/).
-function tokenize(text: string): string[] {
-  const trimmed = text.trim();
-  return trimmed === '' ? [] : trimmed.split(/\s+/);
-}
-
 /**
  * Find the key in a result row that corresponds to the body column expression.
  * ClickHouse may return the column under its expression name or a simplified alias.

packages/api/src/mcp/tools/query/search.ts

@@ -46,7 +46,7 @@ const searchSchema = z.object({
     .default(false)
     .describe(
       'When true, automatically removes events matching high-frequency patterns ' +
-        '(those accounting for >10% of sampled events) from the results. ' +
+        '(the most frequent patterns, together covering about half of sampled events) from the results. ' +
         'Useful for cutting through log noise to find unusual or interesting events. ' +
         'Adds ~1-2s of latency for the pattern sampling queries.',
     ),

packages/app/src/components/DBRowTable.tsx

@@ -10,7 +10,6 @@ import cx from 'classnames';
 import { formatDistance } from 'date-fns';
 import { isString } from 'lodash';
 import curry from 'lodash/curry';
-import ms from 'ms';
 import { useHotkeys } from 'react-hotkeys-hook';
 import {
   Bar,
@@ -30,10 +29,6 @@ import {
   JSDataType,
 } from '@hyperdx/common-utils/dist/clickhouse';
 import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils';
-import {
-  DENOISE_NOISE_THRESHOLD,
-  DENOISE_SAMPLE_SIZE,
-} from '@hyperdx/common-utils/dist/drain';
 import {
   BuilderChartConfigWithDateRange,
   SelectList,
@@ -57,7 +52,7 @@ import {
   IconTextWrap,
   IconTextWrapDisabled,
 } from '@tabler/icons-react';
-import { FetchNextPageOptions, useQuery } from '@tanstack/react-query';
+import { FetchNextPageOptions } from '@tanstack/react-query';
 import {
   ColumnDef,
   ColumnResizeMode,
@@ -78,13 +73,13 @@ import {
   useRenderedSqlChartConfig,
 } from '@/hooks/useChartConfig';
 import { CsvColumn, useCsvExport } from '@/hooks/useCsvExport';
+import { useDenoisedRows } from '@/hooks/useDenoisedRows';
 import {
   useColumns,
   useMergeTreeSettings,
   useTableMetadata,
 } from '@/hooks/useMetadata';
 import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery';
-import { useGroupedPatterns } from '@/hooks/usePatterns';
 import { useRowSelection } from '@/hooks/useRowSelection';
 import useRowWhere, {
   getRowId,
@@ -93,7 +88,7 @@ import useRowWhere, {
   WithClause,
 } from '@/hooks/useRowWhere';
 import { useTableSearch } from '@/hooks/useTableSearch';
-import { getLevelExpression, useSource } from '@/source';
+import { useSource } from '@/source';
 import {
   MIN_COLUMN_WIDTH,
   MIN_LAST_COLUMN_WIDTH,
@@ -137,6 +132,7 @@ import {
 import { SQLPreview } from './ChartSQLPreview';
 import { CsvExportButton } from './CsvExportButton';
 import { RowSidePanelContext } from './DBRowSidePanel';
+import { DenoiseSummary } from './DenoiseSummary';
 import {
   createExpandButtonColumn,
   ExpandedLogRow,
@@ -1948,87 +1944,15 @@ function DBSqlRowTableComponent({
   }, [data?.meta, onResolvedColumnsChange]);
 
   const { data: source } = useSource({ id: sourceId });
-  const patternColumn = columns[columns.length - 1];
-  const groupedPatterns = useGroupedPatterns({
+  const denoised = useDenoisedRows({
     config,
-    samples: DENOISE_SAMPLE_SIZE,
-    bodyValueExpression: patternColumn ?? '',
-    levelExpression: getLevelExpression(source),
-    totalCount: undefined,
+    source,
+    columns,
+    rows: processedRows,
     enabled: denoiseResults,
   });
-  const noisyPatterns = useQuery({
-    queryKey: ['noisy-patterns', config],
-    queryFn: async () => {
-      return Object.values(groupedPatterns.data).filter(
-        p =>
-          p.count / (groupedPatterns.sampledRowCount ?? 1) >
-          DENOISE_NOISE_THRESHOLD,
-      );
-    },
-    enabled:
-      denoiseResults &&
-      groupedPatterns.data != null &&
-      Object.values(groupedPatterns.data).length > 0 &&
-      groupedPatterns.miner != null,
-  });
-  const noisyPatternIds = useMemo(() => {
-    return noisyPatterns.data?.map(p => p.id) ?? [];
-  }, [noisyPatterns.data]);
-
-  const denoisedRows = useQuery({
-    queryKey: [
-      'denoised-rows',
-      config,
-      denoiseResults,
-      // Only include processed rows if denoising is enabled
-      // This helps prevent the queryKey from getting extremely large
-      // and causing memory issues, when it's not used.
-      ...(denoiseResults ? [processedRows] : []),
-      noisyPatternIds,
-      patternColumn,
-    ],
-    queryFn: async () => {
-      if (!denoiseResults) {
-        return [];
-      }
-      // No noisy patterns, so no need to denoise
-      if (noisyPatternIds.length === 0) {
-        return processedRows;
-      }
-
-      const matchedLogs = await groupedPatterns.miner?.matchLogs(
-        processedRows.map(row => row[patternColumn]),
-      );
-      return processedRows.filter((row, i) => {
-        const match = matchedLogs?.[i];
-        return !noisyPatternIds.includes(`${match}`);
-      });
-    },
-    placeholderData: (previousData, previousQuery) => {
-      // If it's the same search, but new data, return the previous data while we load
-      if (
-        previousQuery?.queryKey?.[0] === 'denoised-rows' &&
-        previousQuery?.queryKey?.[1] === config
-      ) {
-        return previousData;
-      }
-      return undefined;
-    },
-    gcTime: isLive ? ms('30s') : ms('5m'), // more aggressive gc for live data, since it can end up holding lots of data
-    enabled:
-      denoiseResults &&
-      noisyPatterns.isSuccess &&
-      processedRows.length > 0 &&
-      groupedPatterns.miner != null,
-  });
 
-  const isLoading = denoiseResults
-    ? isFetching ||
-      denoisedRows.isFetching ||
-      noisyPatterns.isFetching ||
-      groupedPatterns.isLoading
-    : isFetching;
+  const isLoading = isFetching || denoised.isLoading;
 
   const loadingDate =
     data?.window?.direction === 'ASC'
@@ -2038,28 +1962,20 @@ function DBSqlRowTableComponent({
   return (
     <>
       {denoiseResults && (
-        <Box mb="xxs" px="sm">
-          <Text fw="bold" fz="xs" mb="xxs">
-            Removed Noisy Event Patterns
-          </Text>
-          <Box mah={100} style={{ overflow: 'auto' }}>
-            {noisyPatterns.data?.map(p => (
-              <Text fz="xs" key={p.id}>
-                {p.pattern}
-              </Text>
-            ))}
-            {noisyPatternIds.length === 0 && (
-              <Text fz="xs">No noisy patterns found</Text>
-            )}
-          </Box>
-        </Box>
+        <DenoiseSummary
+          noisyPatterns={denoised.noisyPatterns}
+          sampledRowCount={denoised.sampledRowCount}
+          isLoading={denoised.isLoading}
+          skipped={denoised.skipped}
+          bodyExpression={denoised.bodyExpression}
+        />
       )}
       <RawLogTable
         isLive={isLive}
         wrapLines={false}
         displayedColumns={columns}
         highlightedLineId={highlightedLineId}
-        rows={denoiseResults ? (denoisedRows?.data ?? []) : processedRows}
+        rows={denoised.rows}
         renderRowDetails={renderRowDetails}
         isLoading={isLoading}
         fetchNextPage={fetchNextPage}