From f93793ff33e9ea9ccd65acebc5dfc518b82ec78f Mon Sep 17 00:00:00 2001 From: Karl Power Date: Tue, 3 Nov 2026 12:44:24 +0100 Subject: [PATCH 1/2] feat: Run PromQL tiles from MCP query_tile clickstack_query_tile and clickstack_query_tiles now return data for PromQL tiles instead of an error. Each expression runs against the source's backend: a Prometheus endpoint, ClickHouse's Prometheus HTTP API, or the ClickHouse table function on servers older than 26.6. Granularity, alignment, instant versus range queries and reducers follow the dashboard, so the agent sees the numbers the tile shows. Rows match the app's chart rows, with each series' labels listed once in seriesLabels. Number, pie and bar tiles return one reduced value per series. Dashboard variables apply. The backend choice moves from the Prometheus router into controllers/promql.ts so MCP can share it. The router's streaming proxy and the buffered fetch MCP uses now build their requests with one helper, buildPrometheusRequest. The router behaves as before. --- .changeset/mcp-promql-query-tile.md | 5 + .../src/controllers/__tests__/promql.test.ts | 185 +++++++ packages/api/src/controllers/promql.ts | 482 +++++++++++++++++- .../api/src/controllers/timeseriesEngine.ts | 6 + .../dashboards/promqlQueryTile.int.test.ts | 348 +++++++++++++ .../tools/query/__tests__/promqlTile.test.ts | 264 ++++++++++ packages/api/src/mcp/tools/query/helpers.ts | 22 +- .../api/src/mcp/tools/query/promqlTile.ts | 274 ++++++++++ .../api/__tests__/prometheus.int.test.ts | 12 +- packages/api/src/routers/api/prometheus.ts | 326 +++--------- 10 files changed, 1644 insertions(+), 280 deletions(-) create mode 100644 .changeset/mcp-promql-query-tile.md create mode 100644 packages/api/src/controllers/__tests__/promql.test.ts create mode 100644 packages/api/src/mcp/__tests__/dashboards/promqlQueryTile.int.test.ts create mode 100644 packages/api/src/mcp/tools/query/__tests__/promqlTile.test.ts create mode 100644 packages/api/src/mcp/tools/query/promqlTile.ts diff --git a/.changeset/mcp-promql-query-tile.md b/.changeset/mcp-promql-query-tile.md new file mode 100644 index 0000000000..19ada9d6df --- /dev/null +++ b/.changeset/mcp-promql-query-tile.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/api': patch +--- + +feat: MCP `query_tile` and `query_tiles` run PromQL tiles against Prometheus endpoints and ClickHouse TimeSeries tables diff --git a/packages/api/src/controllers/__tests__/promql.test.ts b/packages/api/src/controllers/__tests__/promql.test.ts new file mode 100644 index 0000000000..c992b85312 --- /dev/null +++ b/packages/api/src/controllers/__tests__/promql.test.ts @@ -0,0 +1,185 @@ +import { buildPrometheusRequest, runPromqlQuery } from '@/controllers/promql'; + +const mockFetch = jest.mocked(global.fetch); + +const respond = (status: number, body: unknown) => + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify(body), { status }), + ); + +const matrix = { + resultType: 'matrix', + result: [{ metric: { job: 'api' }, values: [[1700000000, '1']] }], +}; + +const prometheus = { kind: 'prometheus' as const, host: 'http://prom:9090' }; +const range = { + kind: 'range' as const, + expr: 'up', + startSec: 1700000000, + endSec: 1700003600, + stepSec: 60, +}; + +const requestedUrl = () => new URL(String(mockFetch.mock.calls[0][0])); + +describe('buildPrometheusRequest', () => { + const host = + 'http://user:p%40ss@vm:8481/select/0/prometheus?extra_label=env%3Dprod&limit=50&step=15s'; + + it('replaces host-pinned params it sets and keeps the rest', () => { + const { url } = buildPrometheusRequest(host, '/api/v1/query_range', { + step: '60', + 'match[]': ['up', 'down'], + limit: [], + }); + + const sent = new URL(url); + expect(`${sent.origin}${sent.pathname}`).toBe( + 'http://vm:8481/select/0/prometheus/api/v1/query_range', + ); + expect(sent.searchParams.get('extra_label')).toBe('env=prod'); + expect(sent.searchParams.getAll('step')).toEqual(['60']); + expect(sent.searchParams.getAll('match[]')).toEqual(['up', 'down']); + expect(sent.searchParams.has('limit')).toBe(false); + }); + + it('moves host credentials into a Basic auth header next to the given headers', () => { + const { url, headers, redactedTarget } = buildPrometheusRequest( + host, + '/api/v1/query', + {}, + { 'X-Extra': '1' }, + ); + + expect(url).not.toContain('user'); + expect(headers).toEqual({ + Authorization: `Basic ${Buffer.from('user:p@ss').toString('base64')}`, + 'X-Extra': '1', + }); + expect(redactedTarget).toBe( + 'http://vm:8481/select/0/prometheus/api/v1/query', + ); + }); + + it('throws on a host that is not http(s)', () => { + expect(() => + buildPrometheusRequest('prometheus:9090', '/api/v1/query', {}), + ).toThrow(/http\(s\)/); + }); +}); + +describe('runPromqlQuery', () => { + afterEach(() => { + mockFetch.mockReset(); + }); + + it('sends a range query to a Prometheus endpoint and returns its data', async () => { + respond(200, { status: 'success', data: matrix }); + + await expect(runPromqlQuery(prometheus, range)).resolves.toEqual(matrix); + + const url = requestedUrl(); + expect(`${url.origin}${url.pathname}`).toBe( + 'http://prom:9090/api/v1/query_range', + ); + expect(Object.fromEntries(url.searchParams)).toEqual({ + query: 'up', + start: '1700000000', + end: '1700003600', + step: '60', + }); + }); + + const clickhouseHttp = { + kind: 'clickhouse-http' as const, + connection: { host: 'http://ch:8123', username: 'default', password: 'pw' }, + databaseName: 'default', + tableName: 'metrics', + }; + const instant = { kind: 'instant' as const, expr: 'up', timeSec: 1700000000 }; + const emptyVector = { + status: 'success', + data: { resultType: 'vector', result: [] }, + }; + + it("sends an instant query to ClickHouse's Prometheus API with the table, limits and credentials", async () => { + respond(200, emptyVector); + + await runPromqlQuery(clickhouseHttp, instant); + + const url = requestedUrl(); + expect(url.pathname).toBe('/prometheus/api/v1/query'); + expect(Object.fromEntries(url.searchParams)).toMatchObject({ + max_execution_time: '30', + max_result_rows: '100000', + query: 'up', + time: '1700000000', + database: 'default', + table: 'metrics', + }); + expect(mockFetch.mock.calls[0][1]?.headers).toMatchObject({ + 'X-ClickHouse-User': 'default', + 'X-ClickHouse-Key': 'pw', + }); + }); + + it('gives each query on one ClickHouse backend its own query ID', async () => { + respond(200, emptyVector); + respond(200, emptyVector); + + await Promise.all([ + runPromqlQuery(clickhouseHttp, instant), + runPromqlQuery(clickhouseHttp, instant), + ]); + + const queryIds = mockFetch.mock.calls.map(([, init]) => + new Headers(init?.headers).get('X-ClickHouse-Query-Id'), + ); + expect(queryIds[0]).toBeTruthy(); + expect(queryIds[0]).not.toBe(queryIds[1]); + }); + + it('reports a rejected expression as a user error', async () => { + respond(400, { + status: 'error', + errorType: 'bad_data', + error: 'parse error: unexpected end of input', + }); + + await expect(runPromqlQuery(prometheus, range)).rejects.toMatchObject({ + message: 'parse error: unexpected end of input', + category: 'user', + }); + }); + + it.each([ + ['a 5xx', 503, 'internal'], + ['a timeout', 400, 'timeout'], + ])('reports %s as a server error', async (_, status, errorType) => { + respond(status, { status: 'error', errorType, error: 'failed' }); + + await expect(runPromqlQuery(prometheus, range)).rejects.toMatchObject({ + category: 'server', + }); + }); + + it('reports an unreachable endpoint as a server error, without its credentials', async () => { + mockFetch.mockRejectedValueOnce(new TypeError('fetch failed')); + + await expect( + runPromqlQuery({ ...prometheus, host: 'http://u:p@prom:9090' }, range), + ).rejects.toMatchObject({ + message: + 'Failed to reach Prometheus at http://prom:9090/api/v1/query_range', + category: 'server', + }); + }); + + it('rejects a range query over the resolution cap before sending it', async () => { + await expect( + runPromqlQuery(prometheus, { ...range, stepSec: 0.1 }), + ).rejects.toMatchObject({ category: 'user' }); + expect(mockFetch).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/controllers/promql.ts b/packages/api/src/controllers/promql.ts index 79e8a943d5..4f9fd28989 100644 --- a/packages/api/src/controllers/promql.ts +++ b/packages/api/src/controllers/promql.ts @@ -1,15 +1,34 @@ +import { + buildLogComment, + buildQueryId, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { getCurrentQueryAttribution } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { z } from 'zod'; import { ClickhouseClient } from '@/clickhouse'; import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; import { + connectionSupportsPrometheusHttpApi, + PROMETHEUS_MAX_EXECUTION_SEC, + PROMETHEUS_MAX_RESULT_ROWS, + queryInstantViaTableFunction, queryLabelNames, queryLabelValues, + queryRangeViaTableFunction, } from '@/controllers/timeseriesEngine'; +import { withSpan } from '@/utils/instrumentation'; type PromqlBackend = 'prometheus' | 'clickhouse'; const PROMETHEUS_FETCH_TIMEOUT_MS = 30_000; +const PROMETHEUS_CH_TIMEOUT_MS = 30_000; +export const PROMETHEUS_MAX_RESOLUTION = 11_000; + +// ClickHouse serves the Prometheus HTTP API under this prefix on its main HTTP +// port when the `prometheus_api_v1` handler is configured (26.6+). +export const CLICKHOUSE_PROMETHEUS_API_PREFIX = '/prometheus/api/v1'; /** * Join a Connection host with an absolute Prometheus API path. @@ -50,7 +69,7 @@ export function joinPrometheusUpstreamUrl( * Removes `user:pass@` from `url` and returns it as a Basic auth header. fetch * rejects a URL that carries credentials, so they cannot stay in the host. */ -export function takeBasicAuthHeader(url: URL): Record { +function takeBasicAuthHeader(url: URL): Record { if (!url.username && !url.password) return {}; const credentials = `${decodeURIComponent(url.username)}:${decodeURIComponent(url.password)}`; url.username = ''; @@ -60,19 +79,74 @@ export function takeBasicAuthHeader(url: URL): Record { }; } +/** + * Builds a GET to `path` on a Prometheus-compatible host. Each key in `params` + * replaces whatever the host pins at that key: an array sends every item and an + * empty array clears the key. Keys not in `params`, such as a VictoriaMetrics + * `extra_label` scope, stay as the host set them. Credentials in the host's + * userinfo become a Basic auth header, sent alongside `headers`. Throws when + * the host is not an http(s) URL. + */ +export function buildPrometheusRequest( + host: string, + path: string, + params: Record, + headers?: Record, +): { + url: string; + headers: Record; + /** Origin and path only, safe for error messages shown to the user. */ + redactedTarget: string; +} { + const url = joinPrometheusUpstreamUrl(host, path); + for (const [key, value] of Object.entries(params)) { + url.searchParams.delete(key); + for (const item of Array.isArray(value) ? value : [value]) { + url.searchParams.append(key, item); + } + } + const authHeader = takeBasicAuthHeader(url); + return { + url: url.toString(), + headers: { ...authHeader, ...headers }, + // `origin` never includes userinfo, and the query string may hold a + // host-pinned secret such as VictoriaMetrics' `authKey`. + redactedTarget: `${url.origin}${url.pathname}`, + }; +} + +/** + * A failed PromQL evaluation. `user` means the query itself was rejected (bad + * syntax, unknown function); `server` means the backend was unreachable, timed + * out or failed. + */ +export class PromQLQueryError extends Error { + constructor( + message: string, + readonly category: 'user' | 'server', + ) { + super(message); + this.name = 'PromQLQueryError'; + } +} + const prometheusBodySchema = z.object({ status: z.string(), data: z.unknown(), + errorType: z.string().optional(), error: z.string().optional(), }); +// Prometheus reports these as 4xx/5xx alike; neither says the query was wrong. +const SERVER_ERROR_TYPES = new Set(['timeout', 'canceled', 'unavailable']); + const labelListSchema = z.array(z.string()); /** - * GETs a Prometheus API path from an external Prometheus-compatible endpoint - * and returns the response's `data`, checked against `dataSchema`. Errors name - * only the origin and path: the host may carry credentials in its userinfo or - * query string. + * GETs a Prometheus API path from a Prometheus-compatible endpoint and returns + * the response's `data`, checked against `dataSchema`. Errors name only the + * origin and path: the host may carry credentials in its userinfo or query + * string. */ async function fetchPrometheusData( host: string, @@ -80,25 +154,24 @@ async function fetchPrometheusData( params: Record, dataSchema: T, signal?: AbortSignal, + headers?: Record, ): Promise> { - const url = joinPrometheusUpstreamUrl(host, path); - for (const [key, value] of Object.entries(params)) { - url.searchParams.set(key, value); - } - const redactedTarget = `${url.origin}${url.pathname}`; + const request = buildPrometheusRequest(host, path, params, headers); + const { redactedTarget } = request; const timeout = AbortSignal.timeout(PROMETHEUS_FETCH_TIMEOUT_MS); let resp: Response; try { - resp = await fetch(url, { - headers: takeBasicAuthHeader(url), + resp = await fetch(request.url, { + headers: request.headers, signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }); } catch (err) { - throw new Error( + throw new PromQLQueryError( err instanceof Error && err.name === 'TimeoutError' ? `Prometheus request to ${redactedTarget} timed out` : `Failed to reach Prometheus at ${redactedTarget}`, + 'server', ); } @@ -106,15 +179,21 @@ async function fetchPrometheusData( await resp.json().catch(() => undefined), ); if (!resp.ok || !body.success || body.data.status !== 'success') { - throw new Error( + const isUserError = + resp.status >= 400 && + resp.status < 500 && + !SERVER_ERROR_TYPES.has((body.success && body.data.errorType) || ''); + throw new PromQLQueryError( (body.success && body.data.error) || `Prometheus at ${redactedTarget} returned HTTP ${resp.status}`, + isUserError ? 'user' : 'server', ); } const data = dataSchema.safeParse(body.data.data); if (!data.success) { - throw new Error( + throw new PromQLQueryError( `Prometheus at ${redactedTarget} returned an unexpected response`, + 'server', ); } return data.data; @@ -204,3 +283,376 @@ export async function getPromqlLabelCatalog({ ]); return { backend: 'clickhouse', labelNames, metricNames }; } + +type PromqlTarget = { + connection: NonNullable>>; + databaseName: string; + tableName: string; + /** The source's scrape interval floor, e.g. "1 minute". */ + minAutoGranularity?: string; +}; + +/** + * Resolve the PromQL source a request names into the connection and table to + * query. The table always comes from the source, never from the request, so a + * caller can only reach data through a source. + */ +export async function resolvePromqlTarget( + teamId: string, + sourceId: string | undefined, + { selectPassword }: { selectPassword: boolean }, +): Promise { + if (!sourceId) { + return { status: 400, error: 'missing required parameter: sourceId' }; + } + + const source = await getSource(teamId, sourceId); + + if (!source || source.kind !== SourceKind.Promql) { + return { status: 404, error: 'Source not found' }; + } + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + selectPassword, + ); + + if (!connection) { + return { status: 404, error: 'Connection not found' }; + } + + return { + connection, + databaseName: source.from.databaseName, + tableName: source.from.tableName, + minAutoGranularity: source.minAutoGranularity, + }; +} + +/** + * The ClickHouse connection host with the query limits pinned as ClickHouse + * HTTP settings. The prometheus_api_v1 handler honours settings given in the + * URL and reports a breach as a Prometheus 400 (pinned by the live int test), + * so this is the only place to bound a PromQL evaluation — the request carries + * no SQL to attach SETTINGS to. Pinned on the host rather than passed as params + * so the merge in proxyToPrometheus (which lets the caller override any + * allowlisted key) can't loosen them. + */ +export function clickhousePrometheusUpstream( + host: string, + { + maxExecutionSec = PROMETHEUS_MAX_EXECUTION_SEC, + maxResultRows = PROMETHEUS_MAX_RESULT_ROWS, + logComment, + }: { + maxExecutionSec?: number; + maxResultRows?: number; + logComment?: string; + } = {}, +): string { + const url = new URL(host); + url.searchParams.set('max_execution_time', String(maxExecutionSec)); + url.searchParams.set('max_result_rows', String(maxResultRows)); + if (logComment) url.searchParams.set('log_comment', logComment); + return url.toString(); +} + +function clickhouseAuthHeaders(connection: { + username: string; + password?: string; +}): Record { + return { + 'X-ClickHouse-User': connection.username, + 'X-ClickHouse-Key': connection.password ?? '', + }; +} + +type ClickhouseCredentials = { + host: string; + username: string; + password?: string; +}; + +/** + * Where and how to send one PromQL request to the connection's + * prometheus_api_v1 handler, tagged with the query attribution in scope. The + * handler runs the query itself rather than through our ClickHouse client, so + * the tags are set here. `log_comment` goes in the URL like the limits. + * `query_id` goes in a header: the handler reads every URL param as a setting, + * and `query_id` in the URL fails the request as an unknown one. + * + * Call once per request: ClickHouse rejects a query whose ID is already + * running, so concurrent requests must not share the result. + */ +export function clickhousePrometheusRequest( + connection: ClickhouseCredentials, +): { upstream: string; headers: Record } { + const attribution = getCurrentQueryAttribution(); + return { + upstream: clickhousePrometheusUpstream(connection.host, { + logComment: buildLogComment(attribution), + }), + headers: { + ...clickhouseAuthHeaders(connection), + 'X-ClickHouse-Query-Id': buildQueryId(attribution), + }, + }; +} + +export function newPromqlClickhouseClient(connection: { + host: string; + username: string; + password?: string; +}) { + return new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + requestTimeout: PROMETHEUS_CH_TIMEOUT_MS, + }); +} + +/** + * Whether a ClickHouse connection answers PromQL over HTTP. Two conditions, + * cheapest first: the server is new enough (26.6+), and the prometheus_api_v1 + * handler is actually configured — a self-managed server needs an + * `` rule for that, which a stock install lacks. The handler + * answers every request, error or not, with Prometheus JSON (`{status, ...}`); + * a server without it answers a plain-text 404. `format_query` is used as the + * probe because it is the one endpoint that needs no table. + * + * Probed on every request, like the version check, so a handler enabled or + * removed on the server is reflected immediately. + */ +const PROBE_TIMEOUT_MS = 5_000; + +async function clickhouseServesPrometheusHttpApi( + client: ClickhouseClient, + connection: { id: string; host: string; username: string; password?: string }, +): Promise { + if ( + !(await connectionSupportsPrometheusHttpApi({ + client, + connectionId: connection.id, + })) + ) { + return false; + } + + try { + const { url, headers } = buildPrometheusRequest( + connection.host, + `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/format_query`, + { query: 'up' }, + clickhouseAuthHeaders(connection), + ); + const resp = await fetch(url, { + headers, + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + return typeof JSON.parse(await resp.text())?.status === 'string'; + } catch { + return false; + } +} + +type ClickhouseQueryBackend = + | { + kind: 'clickhouse-http'; + connection: ClickhouseCredentials; + databaseName: string; + tableName: string; + } + | { + kind: 'table-function'; + client: ClickhouseClient; + connectionId: string; + databaseName: string; + tableName: string; + }; + +/** How a PromQL query against a source is evaluated. */ +type PromqlQueryBackend = + | { kind: 'prometheus'; host: string } + | ClickhouseQueryBackend; + +/** + * How to evaluate PromQL on a ClickHouse-backed target. The Prometheus HTTP + * API is the only surface ClickHouse holds forward-compatible while TimeSeries + * is in preview, so it is preferred wherever the server has it (26.6+); the + * table function is the fallback. + */ +export async function resolveClickhouseQueryBackend({ + connection, + databaseName, + tableName, +}: PromqlTarget): Promise { + const client = newPromqlClickhouseClient(connection); + if (await clickhouseServesPrometheusHttpApi(client, connection)) { + return { + kind: 'clickhouse-http', + connection: { + host: connection.host, + username: connection.username, + password: connection.password, + }, + databaseName, + tableName, + }; + } + return { + kind: 'table-function', + client, + connectionId: connection.id, + databaseName, + tableName, + }; +} + +/** How to evaluate PromQL on a target. Probes ClickHouse targets for the HTTP API. */ +export async function resolvePromqlQueryBackend( + target: PromqlTarget, +): Promise { + if (target.connection.isPrometheusEndpoint) { + return { kind: 'prometheus', host: target.connection.host }; + } + return resolveClickhouseQueryBackend(target); +} + +export type PromqlQuery = + | { + kind: 'range'; + expr: string; + startSec: number; + endSec: number; + stepSec: number; + } + | { kind: 'instant'; expr: string; timeSec: number }; + +const sampleSchema = z.tuple([z.number(), z.string()]); +const metricSchema = z.record(z.string()); + +const promqlQueryDataSchema = z.discriminatedUnion('resultType', [ + z.object({ + resultType: z.literal('matrix'), + result: z.array( + z.object({ metric: metricSchema, values: z.array(sampleSchema) }), + ), + }), + z.object({ + resultType: z.literal('vector'), + result: z.array(z.object({ metric: metricSchema, value: sampleSchema })), + }), + z.object({ resultType: z.literal('scalar'), result: sampleSchema }), + z.object({ resultType: z.literal('string'), result: sampleSchema }), +]); + +/** The `data` payload of a Prometheus query or query_range response. */ +export type PromqlQueryData = z.infer; + +function promqlQueryParams(query: PromqlQuery): Record { + return query.kind === 'range' + ? { + query: query.expr, + start: String(query.startSec), + end: String(query.endSec), + step: String(query.stepSec), + } + : { query: query.expr, time: String(query.timeSec) }; +} + +async function evaluatePromqlQuery( + backend: PromqlQueryBackend, + query: PromqlQuery, + signal?: AbortSignal, +): Promise { + const endpoint = query.kind === 'range' ? 'query_range' : 'query'; + switch (backend.kind) { + case 'prometheus': + return fetchPrometheusData( + backend.host, + `/api/v1/${endpoint}`, + promqlQueryParams(query), + promqlQueryDataSchema, + signal, + ); + case 'clickhouse-http': { + const { upstream, headers } = clickhousePrometheusRequest( + backend.connection, + ); + return fetchPrometheusData( + upstream, + `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/${endpoint}`, + { + ...promqlQueryParams(query), + database: backend.databaseName, + table: backend.tableName, + }, + promqlQueryDataSchema, + signal, + headers, + ); + } + case 'table-function': + return query.kind === 'range' + ? { + resultType: 'matrix', + result: await queryRangeViaTableFunction({ + client: backend.client, + connectionId: backend.connectionId, + databaseName: backend.databaseName, + tableName: backend.tableName, + expr: query.expr, + startMs: Math.floor(query.startSec * 1000), + endMs: Math.floor(query.endSec * 1000), + stepSec: Math.max(Math.floor(query.stepSec), 1), + signal, + }), + } + : { + resultType: 'vector', + result: await queryInstantViaTableFunction({ + client: backend.client, + databaseName: backend.databaseName, + tableName: backend.tableName, + expr: query.expr, + evalMs: Math.floor(query.timeSec * 1000), + signal, + }), + }; + } +} + +/** + * Evaluates one PromQL query and returns the Prometheus `data` payload. Throws + * PromQLQueryError for a rejected or failed query; the table-function backend + * throws ClickHouse errors as they come. + */ +export async function runPromqlQuery( + backend: PromqlQueryBackend, + query: PromqlQuery, + signal?: AbortSignal, +): Promise { + if ( + query.kind === 'range' && + (query.stepSec <= 0 || + (query.endSec - query.startSec) / query.stepSec > + PROMETHEUS_MAX_RESOLUTION) + ) { + throw new PromQLQueryError( + `exceeded maximum resolution of ${PROMETHEUS_MAX_RESOLUTION.toLocaleString('en-US')} points per timeseries. Try a coarser granularity.`, + 'user', + ); + } + return withSpan( + 'promql.query', + () => evaluatePromqlQuery(backend, query, signal), + { + attributes: { + 'hyperdx.promql.backend': backend.kind, + 'hyperdx.promql.query_kind': query.kind, + }, + }, + ); +} diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts index f99172826e..3f63e8dc17 100644 --- a/packages/api/src/controllers/timeseriesEngine.ts +++ b/packages/api/src/controllers/timeseriesEngine.ts @@ -378,6 +378,7 @@ export async function queryRangeViaTableFunction({ startMs, endMs, stepSec, + signal, }: { client: ClickhouseClient; connectionId: string; @@ -387,6 +388,7 @@ export async function queryRangeViaTableFunction({ startMs: number; endMs: number; stepSec: number; + signal?: AbortSignal; }): Promise { const samplesColumn = await timeSeriesSamplesColumn({ client, @@ -405,6 +407,7 @@ export async function queryRangeViaTableFunction({ stepSec, }, format: 'JSON', + abort_signal: signal, clickhouse_settings: PROMQL_TABLE_FUNCTION_SETTINGS, }); const json = await resp.json(); @@ -417,17 +420,20 @@ export async function queryInstantViaTableFunction({ tableName, expr, evalMs, + signal, }: { client: ClickhouseClient; databaseName: string; tableName: string; expr: string; evalMs: number; + signal?: AbortSignal; }): Promise { const resp = await client.query({ query: `SELECT tags, timestamp, value FROM prometheusQuery({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({evalMs:Int64})) SETTINGS allow_experimental_time_series_table = 1`, query_params: { db: databaseName, table: tableName, expr, evalMs }, format: 'JSON', + abort_signal: signal, clickhouse_settings: PROMQL_TABLE_FUNCTION_SETTINGS, }); const json = await resp.json(); diff --git a/packages/api/src/mcp/__tests__/dashboards/promqlQueryTile.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/promqlQueryTile.int.test.ts new file mode 100644 index 0000000000..31960fe982 --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/promqlQueryTile.int.test.ts @@ -0,0 +1,348 @@ +import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { + DEFAULT_DATABASE, + dropTimeSeriesTable, + seedTimeSeriesTagsTable, +} from '@/fixtures'; +import { callTool, getFirstText } from '@/mcp/__tests__/mcpTestUtils'; +import Connection from '@/models/connection'; +import Dashboard from '@/models/dashboard'; +import { Source } from '@/models/source'; + +import { setupDashboardTests } from './setup'; + +const mockFetch = jest.mocked(global.fetch); +// Set by jest.setup.ts before it stubs `fetch`. +const realFetch: typeof fetch = globalThis.realFetch; + +const TABLE = 'mcp_promql_query_tile'; +const START = 1700000000; +const startTime = new Date(START * 1000).toISOString(); +const endTime = new Date((START + 60) * 1000).toISOString(); + +describe('MCP query_tile - PromQL tiles', () => { + const ctx = setupDashboardTests(); + + beforeAll(async () => { + await seedTimeSeriesTagsTable({ + table: TABLE, + series: ['checkout-0', 'checkout-1'].map(instance => ({ + metricName: 'mcp_up', + tags: { job: 'checkout', instance }, + startSec: START, + endSec: START + 60, + })), + withSamples: true, + }); + }); + + afterAll(async () => { + await dropTimeSeriesTable({ table: TABLE }); + }); + + beforeEach(() => ctx.reconnect({ promqlEnabled: true })); + + afterEach(() => { + jest.restoreAllMocks(); + mockFetch.mockReset(); + }); + + const createClickhouseSource = () => + Source.create({ + kind: SourceKind.Promql, + team: ctx.team._id, + connection: ctx.connection._id, + name: 'Metrics', + from: { databaseName: DEFAULT_DATABASE, tableName: TABLE }, + timestampValueExpression: 'timestamp', + }); + + const createPrometheusSource = async () => { + const connection = await Connection.create({ + team: ctx.team._id, + name: 'Prom', + host: 'http://user:secret@prom.example.com', + username: '', + password: '', + isPrometheusEndpoint: true, + }); + return Source.create({ + kind: SourceKind.Promql, + team: ctx.team._id, + connection: connection._id, + name: 'Prometheus', + from: { databaseName: 'prometheus', tableName: 'prometheus' }, + timestampValueExpression: 'timestamp', + }); + }; + + const createDashboard = ( + sourceId: string, + config: Record, + filters: Record[] = [], + ) => + new Dashboard({ + name: 'PromQL', + tiles: [ + { + id: 'promql-tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'promql', + name: 'Up', + displayType: DisplayType.Line, + source: sourceId, + granularity: '1 minute', + ...config, + }, + }, + ], + filters, + team: ctx.team._id, + }).save(); + + const queryTile = (dashboardId: string, args: Record = {}) => + callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId, + tileId: 'promql-tile', + startTime, + endTime, + ...args, + }); + + const parseResult = (result: Awaited>) => + JSON.parse(getFirstText(result)).result; + + const lineTile = { + promqlExpression: [{ expression: 'sum by (instance) (mcp_up)' }], + legendTemplate: '{{instance}}', + }; + + const fetchedPaths = () => + mockFetch.mock.calls.map(([url]) => new URL(String(url)).pathname); + + it("returns a line tile's rows through ClickHouse's Prometheus API", async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBeFalsy(); + const { data, seriesLabels } = parseResult(result); + expect(Object.keys(seriesLabels).sort()).toEqual([ + 'checkout-0', + 'checkout-1', + ]); + expect(seriesLabels['checkout-0']).toEqual({ instance: 'checkout-0' }); + expect(data.length).toBeGreaterThan(0); + expect(data).toContainEqual({ + __hdx_time_bucket: '2023-11-14T22:14:00.000Z', + series_name: 'checkout-0', + value: 1, + }); + expect(fetchedPaths()).toContain('/prometheus/api/v1/query_range'); + }); + + it("runs every expression of a tile through ClickHouse's Prometheus API", async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), { + promqlExpression: [ + { expression: 'sum by (instance) (mcp_up)' }, + { expression: 'count(mcp_up)' }, + ], + }); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError ? getFirstText(result) : '').toBe(''); + const { data } = parseResult(result); + const seriesNames = new Set( + data.map((row: { series_name: string }) => row.series_name), + ); + expect(seriesNames.size).toBe(3); + }); + + it('returns the same rows through the table function on ClickHouse < 26.6', async () => { + jest + .spyOn(Metadata.prototype, 'getServerVersion') + .mockResolvedValue([26, 5, 0, 0]); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBeFalsy(); + expect(parseResult(result).data).toContainEqual({ + __hdx_time_bucket: '2023-11-14T22:14:00.000Z', + series_name: 'checkout-0', + value: 1, + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('reduces a number tile to one value', async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), { + displayType: DisplayType.Number, + promqlExpression: [{ expression: 'count(mcp_up)', queryType: 'instant' }], + }); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBeFalsy(); + expect(parseResult(result).data).toEqual([ + { series_name: 'count(mcp_up)', value: 2 }, + ]); + }); + + it('applies the selected dashboard variable', async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard( + source._id.toString(), + { + ...lineTile, + promqlExpression: [ + { + expression: 'sum by (instance) (mcp_up{instance=~"$instance"})', + }, + ], + }, + [ + { + id: 'instance-filter', + type: 'PROMETHEUS_LABEL', + name: 'instance', + variableName: 'instance', + source: source._id.toString(), + label: 'instance', + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + ); + + const result = await queryTile(dashboard._id.toString(), { + variableValues: [{ name: 'instance', values: ['checkout-1'] }], + }); + + expect(result.isError).toBeFalsy(); + expect(Object.keys(parseResult(result).seriesLabels)).toEqual([ + 'checkout-1', + ]); + }); + + it('queries a Prometheus endpoint with the host credentials and no table params', async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + status: 'success', + data: { + resultType: 'matrix', + result: [ + { metric: { instance: 'checkout-0' }, values: [[START, '1']] }, + ], + }, + }), + ), + ); + const source = await createPrometheusSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBeFalsy(); + expect(parseResult(result).data).toEqual([ + { + __hdx_time_bucket: '2023-11-14T22:13:20.000Z', + series_name: 'checkout-0', + value: 1, + }, + ]); + const [url, init] = mockFetch.mock.calls[0]; + const requested = new URL(String(url)); + expect(`${requested.origin}${requested.pathname}`).toBe( + 'http://prom.example.com/api/v1/query_range', + ); + expect(requested.searchParams.has('database')).toBe(false); + expect(requested.searchParams.has('table')).toBe(false); + expect(init?.headers).toMatchObject({ + Authorization: `Basic ${Buffer.from('user:secret').toString('base64')}`, + }); + }); + + it("returns the endpoint's error for a rejected expression", async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + status: 'error', + errorType: 'bad_data', + error: 'parse error: unexpected end of input', + }), + { status: 400 }, + ), + ); + const source = await createPrometheusSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('parse error'); + }); + + it('reports an unreachable endpoint without its credentials', async () => { + mockFetch.mockRejectedValueOnce(new TypeError('fetch failed')); + const source = await createPrometheusSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'Failed to reach Prometheus at http://prom.example.com/api/v1/query_range', + ); + expect(getFirstText(result)).not.toContain('secret'); + }); + + it('reports a PromQL tile with data in query_tiles', async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await callTool(ctx.client!, 'clickstack_query_tiles', { + dashboardId: dashboard._id.toString(), + startTime, + endTime, + }); + + expect(result.isError).toBeFalsy(); + const { tiles } = JSON.parse(getFirstText(result)); + expect(tiles).toEqual([ + expect.objectContaining({ + tileId: 'promql-tile', + status: 'ok', + hasData: true, + }), + ]); + }); + + it('does not find a PromQL tile with PromQL disabled', async () => { + await ctx.reconnect({ promqlEnabled: false }); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Tile not found'); + }); +}); diff --git a/packages/api/src/mcp/tools/query/__tests__/promqlTile.test.ts b/packages/api/src/mcp/tools/query/__tests__/promqlTile.test.ts new file mode 100644 index 0000000000..971857e2b0 --- /dev/null +++ b/packages/api/src/mcp/tools/query/__tests__/promqlTile.test.ts @@ -0,0 +1,264 @@ +import { DisplayType, PromqlReducer } from '@hyperdx/common-utils/dist/types'; + +import { + buildPromqlTileQueries, + toPromqlTileResult, +} from '@/mcp/tools/query/promqlTile'; + +const start = new Date('2026-11-02T10:00:07Z'); +const end = new Date('2026-11-02T11:00:07Z'); +const dateRange: [Date, Date] = [start, end]; +const sec = (iso: string) => new Date(iso).getTime() / 1000; + +describe('buildPromqlTileQueries', () => { + it('sends a range query per expression of a time chart, on the aligned auto step', () => { + const queries = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [ + { expression: 'avg_over_time(up[$__interval])' }, + { expression: 'sum(up)' }, + ], + }, + dateRange, + }); + + // One hour over 80 buckets resolves to 1 minute; the range widens to + // whole minutes. + expect(queries.map(q => q.query)).toEqual([ + { + kind: 'range', + expr: 'avg_over_time(up[60s])', + startSec: sec('2026-11-02T10:00:00Z'), + endSec: sec('2026-11-02T11:01:00Z'), + stepSec: 60, + }, + { + kind: 'range', + expr: 'sum(up)', + startSec: sec('2026-11-02T10:00:00Z'), + endSec: sec('2026-11-02T11:01:00Z'), + stepSec: 60, + }, + ]); + }); + + it('sends an instant query at the end of the unaligned range', () => { + const queries = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Number, + promqlExpression: [{ expression: 'sum(up)', queryType: 'instant' }], + }, + dateRange, + }); + + expect(queries.map(q => q.query)).toEqual([ + { kind: 'instant', expr: 'sum(up)', timeSec: end.getTime() / 1000 }, + ]); + }); + + it('runs an instant expression on a time chart as a range query', () => { + const [{ query }] = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up', queryType: 'instant' }], + }, + dateRange, + }); + + expect(query.kind).toBe('range'); + }); + + it('queries only the first expression of a pie tile', () => { + const queries = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Pie, + promqlExpression: [{ expression: 'sum(up)' }, { expression: 'up' }], + }, + dateRange, + }); + + expect(queries.map(q => q.expression)).toEqual(['sum(up)']); + }); + + it('expands dashboard variables', () => { + const [{ query }] = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up{job=~"$job"}' }], + }, + dateRange, + variables: [{ name: 'job', values: ['checkout'] }], + }); + + expect(query.expr).toContain('checkout'); + expect(query.expr).not.toContain('$job'); + }); + + it("floors an auto step at the source's minimum granularity", () => { + const [{ query }] = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up' }], + }, + dateRange, + minGranularitySeconds: 300, + }); + + expect(query).toMatchObject({ kind: 'range', stepSec: 300 }); + }); + + it('keeps a granularity the tile picked, unaligned when alignment is off', () => { + const [{ query }] = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up' }], + granularity: '15 second', + alignDateRangeToGranularity: false, + }, + dateRange, + minGranularitySeconds: 300, + }); + + expect(query).toEqual({ + kind: 'range', + expr: 'up', + startSec: start.getTime() / 1000, + endSec: end.getTime() / 1000, + stepSec: 15, + }); + }); +}); + +describe('toPromqlTileResult', () => { + const checkout0 = { job: 'checkout', instance: 'checkout-0' }; + const checkout1 = { job: 'checkout', instance: 'checkout-1' }; + const t0 = sec('2026-11-02T10:00:00Z'); + const t1 = sec('2026-11-02T10:01:00Z'); + + it('returns time-bucketed rows named by the legend template, labels listed once', () => { + const result = toPromqlTileResult( + { + displayType: DisplayType.Line, + promqlExpression: 'up', + legendTemplate: '{{instance}}', + }, + [ + { + expression: 'up', + isBucketed: true, + result: [ + { + metric: checkout0, + values: [ + [t0, 1], + [t1, 0], + ], + }, + { metric: checkout1, values: [[t0, 1]] }, + ], + }, + ], + ); + + expect(result.data).toEqual([ + { + __hdx_time_bucket: '2026-11-02T10:00:00.000Z', + series_name: 'checkout-0', + value: 1, + }, + { + __hdx_time_bucket: '2026-11-02T10:01:00.000Z', + series_name: 'checkout-0', + value: 0, + }, + { + __hdx_time_bucket: '2026-11-02T10:00:00.000Z', + series_name: 'checkout-1', + value: 1, + }, + ]); + expect(result.rows).toBe(3); + expect(result.seriesLabels).toEqual({ + 'checkout-0': checkout0, + 'checkout-1': checkout1, + }); + }); + + it('reduces each series of a number tile with its reducer, dropping series with no value', () => { + const result = toPromqlTileResult( + { + displayType: DisplayType.Number, + promqlExpression: [{ expression: 'up', reducer: PromqlReducer.Max }], + legendTemplate: '{{instance}}', + }, + [ + { + expression: 'up', + isBucketed: true, + result: [ + { + metric: checkout0, + values: [ + [t0, 2], + [t1, 5], + ], + }, + { metric: checkout1, values: [[t0, NaN]] }, + ], + }, + ], + ); + + expect(result.data).toEqual([{ series_name: 'checkout-0', value: 5 }]); + expect(result.seriesLabels).toEqual({ 'checkout-0': checkout0 }); + }); + + it('defaults a reduced tile to the last non-null sample', () => { + const result = toPromqlTileResult( + { displayType: DisplayType.Bar, promqlExpression: 'up' }, + [ + { + expression: 'up', + isBucketed: true, + result: [ + { + metric: checkout0, + values: [ + [t0, 2], + [t1, 3], + ], + }, + ], + }, + ], + ); + + expect(result.data).toEqual([{ series_name: 'up', value: 3 }]); + }); + + it('leaves out the time bucket for an instant table', () => { + const result = toPromqlTileResult( + { + displayType: DisplayType.Table, + promqlExpression: [{ expression: 'up', queryType: 'instant' }], + }, + [ + { + expression: 'up', + isBucketed: false, + result: [ + { metric: checkout0, values: [[t0, 1]] }, + { metric: checkout1, values: [[t0, 0]] }, + ], + }, + ], + ); + + expect(result.data).toEqual([ + { series_name: '{instance="checkout-0"}', value: 1 }, + { series_name: '{instance="checkout-1"}', value: 0 }, + ]); + expect(result.meta.map(m => m.name)).toEqual(['series_name', 'value']); + }); +}); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 778555f868..695e13f69f 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -30,6 +30,7 @@ import ms from 'ms'; import { ClickhouseClient } from '@/clickhouse'; import { getConnectionById } from '@/controllers/connection'; +import { PromQLQueryError } from '@/controllers/promql'; import { getSource } from '@/controllers/sources'; import type { McpErrorResult } from '@/mcp/utils/errors'; import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; @@ -47,6 +48,7 @@ import { trimToolResponse } from '@/utils/trimToolResponse'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; +import { runPromqlTile } from './promqlTile'; import { runEventPatterns } from './runEventPatterns'; // ─── Source body expression helpers ────────────────────────────────────────── @@ -709,7 +711,25 @@ export async function runConfigTile( // PromQL is evaluated by the Prometheus API, not the SQL renderer. if (isPromqlSavedChartConfig(savedConfig)) { - return mcpUserError('PromQL tiles cannot be queried through MCP yet.'); + try { + return formatQueryResult( + await runPromqlTile({ + teamId, + config: savedConfig, + dateRange: [startDate, endDate], + variables: options?.variables, + signal: options?.abortSignal, + }), + ); + } catch (e) { + if (e instanceof PromQLQueryError) { + return e.category === 'user' + ? mcpUserError(e.message) + : mcpServerError(e.message); + } + // The table-function backend throws ClickHouse errors as they come. + return clickHouseErrorResult(e); + } } // Raw SQL tile — hydrate source fields for macro support ($__sourceTable, $__filters) diff --git a/packages/api/src/mcp/tools/query/promqlTile.ts b/packages/api/src/mcp/tools/query/promqlTile.ts new file mode 100644 index 0000000000..a4e347911b --- /dev/null +++ b/packages/api/src/mcp/tools/query/promqlTile.ts @@ -0,0 +1,274 @@ +import { + displayTypeSupportsInstantQuery, + displayTypeSupportsReducer, + getQueriedPromqlSeries, + isRangeQuery, + promqlSeriesQueryType, + promqlStep, + reducePromqlSamples, +} from '@hyperdx/common-utils/dist/core/promql'; +import { renderPromqlSeriesNames } from '@hyperdx/common-utils/dist/core/seriesNameTemplate'; +import { + convertDateRangeToGranularityString, + convertGranularityToSeconds, + getAlignedDateRange, +} from '@hyperdx/common-utils/dist/core/utils'; +import { substitutePromqlChartConfigTemplates } from '@hyperdx/common-utils/dist/macros'; +import type { + ChartVariable, + PromqlSavedChartConfig, + PromqlSeries, + SQLInterval, +} from '@hyperdx/common-utils/dist/types'; + +import { + PromqlQuery, + PromqlQueryData, + PromQLQueryError, + resolvePromqlQueryBackend, + resolvePromqlTarget, + runPromqlQuery, +} from '@/controllers/promql'; + +/** The tile fields that decide what a PromQL tile queries and how its result is shaped. */ +type PromqlTileConfig = Pick< + PromqlSavedChartConfig, + | 'promqlExpression' + | 'displayType' + | 'granularity' + | 'alignDateRangeToGranularity' + | 'legendTemplate' +>; + +/** The bucket count the app's time charts size an `auto` granularity for. */ +const AUTO_GRANULARITY_MAX_BUCKETS = 80; + +/** + * The queries a PromQL tile sends, one per expression its display type plots. + * Granularity, alignment and query type follow the app's chart converters, so + * the tile returns the same samples it shows on the dashboard. + */ +export function buildPromqlTileQueries({ + config, + dateRange, + minGranularitySeconds, + variables, +}: { + config: PromqlTileConfig; + dateRange: [Date, Date]; + minGranularitySeconds?: number; + variables?: ChartVariable[]; +}): (PromqlSeries & { query: PromqlQuery })[] { + const granularity: SQLInterval = + !config.granularity || config.granularity === 'auto' + ? convertDateRangeToGranularityString( + dateRange, + AUTO_GRANULARITY_MAX_BUCKETS, + minGranularitySeconds, + ) + : config.granularity; + + const allowsInstant = displayTypeSupportsInstantQuery(config); + + const alignedRange = + (!allowsInstant || isRangeQuery(config)) && + config.alignDateRangeToGranularity !== false + ? getAlignedDateRange(dateRange, granularity) + : dateRange; + + const substituted = substitutePromqlChartConfigTemplates({ + ...config, + granularity, + dateRange: alignedRange, + minGranularitySeconds, + variables, + }); + + const [startSec, endSec] = alignedRange.map(date => date.getTime() / 1000); + + // promqlStep answers a Prometheus duration such as "60s". + const stepSec = Number.parseFloat( + promqlStep(granularity, alignedRange, minGranularitySeconds), + ); + + return getQueriedPromqlSeries(substituted).map(series => ({ + ...series, + query: + allowsInstant && promqlSeriesQueryType(series) === 'instant' + ? { kind: 'instant', expr: series.expression, timeSec: endSec } + : { + kind: 'range', + expr: series.expression, + startSec, + endSec, + stepSec, + }, + })); +} + +type SampledSeries = { + metric: Record; + values: [number, number][]; +}; + +/** One expression's result, with its samples parsed. */ +type PromqlExpressionSamples = PromqlSeries & { + /** Whether the result is bucketed by time. */ + isBucketed: boolean; + result: SampledSeries[]; +}; + +function toSamples( + data: PromqlQueryData, +): Pick { + switch (data.resultType) { + case 'matrix': + return { + isBucketed: true, + result: data.result.map(({ metric, values }) => ({ + metric, + values: values.map(([ts, value]) => [ts, Number.parseFloat(value)]), + })), + }; + case 'vector': + return { + isBucketed: false, + result: data.result.map(({ metric, value }) => ({ + metric, + values: [[value[0], Number.parseFloat(value[1])]], + })), + }; + case 'scalar': + return { + isBucketed: false, + result: [ + { + metric: {}, + values: [[data.result[0], Number.parseFloat(data.result[1])]], + }, + ], + }; + case 'string': + throw new PromQLQueryError( + 'A PromQL chart needs numeric samples; this expression returned a string.', + 'user', + ); + } +} + +/** + * A PromQL tile's result as chart rows, named like the dashboard names its + * series. Number, pie and bar tiles reduce each series to one value. Each + * series' labels are listed once in `seriesLabels` rather than on every row. + */ +export function toPromqlTileResult( + config: PromqlTileConfig, + expressions: PromqlExpressionSamples[], +) { + const names = renderPromqlSeriesNames( + expressions, + config.legendTemplate?.trim() || undefined, + ); + const reduces = displayTypeSupportsReducer(config); + const reducer = getQueriedPromqlSeries(config)[0]?.reducer; + + const isBucketed = !reduces && expressions.some(e => e.isBucketed); + + const data: Record[] = []; + + const seriesLabels = new Map>(); + + // Names come back grouped by expression in series order, so they line up + // with the series once both are flattened. + const seriesNames = names.flat(); + expressions + .flatMap(({ result }) => result) + .forEach(({ metric, values }, i) => { + const series_name = seriesNames.at(i) ?? ''; + if (reduces) { + const value = reducePromqlSamples( + values.map(([, sample]) => sample), + reducer, + ); + // A series with no usable sample reads as no value, not zero. + if (value == null) return; + seriesLabels.set(series_name, metric); + data.push({ series_name, value }); + return; + } + seriesLabels.set(series_name, metric); + for (const [ts, value] of values) { + data.push( + isBucketed + ? { + __hdx_time_bucket: new Date(ts * 1000).toISOString(), + series_name, + value, + } + : { series_name, value }, + ); + } + }); + + return { + data, + meta: [ + ...(isBucketed + ? [{ name: '__hdx_time_bucket', type: 'DateTime64(3)' }] + : []), + { name: 'series_name', type: 'String' }, + { name: 'value', type: 'Float64' }, + ], + rows: data.length, + seriesLabels: Object.fromEntries(seriesLabels), + }; +} + +/** + * Runs a PromQL tile and returns its rows. Throws PromQLQueryError for a + * missing source or a rejected or failed query, and ClickHouse errors from the + * table-function backend. + */ +export async function runPromqlTile({ + teamId, + config, + dateRange, + variables, + signal, +}: { + teamId: string; + config: PromqlTileConfig & Pick; + dateRange: [Date, Date]; + variables?: ChartVariable[]; + signal?: AbortSignal; +}) { + const target = await resolvePromqlTarget(teamId, config.source, { + selectPassword: true, + }); + if ('error' in target) { + throw new PromQLQueryError( + `${target.error}: ${config.source}. Call clickstack_list_sources to discover available source IDs.`, + 'user', + ); + } + + const queries = buildPromqlTileQueries({ + config, + dateRange, + minGranularitySeconds: target.minAutoGranularity + ? convertGranularityToSeconds(target.minAutoGranularity as SQLInterval) + : undefined, + variables, + }); + + // One backend for every expression: resolving a ClickHouse backend probes + // the server for its Prometheus HTTP API. + const backend = await resolvePromqlQueryBackend(target); + const expressions = await Promise.all( + queries.map(async ({ query, ...series }) => ({ + ...series, + ...toSamples(await runPromqlQuery(backend, query, signal)), + })), + ); + return toPromqlTileResult(config, expressions); +} diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts index 836b9bdbb8..d82be84618 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts @@ -7,7 +7,11 @@ import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { Types } from 'mongoose'; import * as config from '@/config'; -import { joinPrometheusUpstreamUrl } from '@/controllers/promql'; +import { + CLICKHOUSE_PROMETHEUS_API_PREFIX, + clickhousePrometheusUpstream, + joinPrometheusUpstreamUrl, +} from '@/controllers/promql'; import { DEFAULT_DATABASE, dropTimeSeriesTable, @@ -18,11 +22,7 @@ import { } from '@/fixtures'; import Connection from '@/models/connection'; import { Source } from '@/models/source'; -import { - CLICKHOUSE_PROMETHEUS_API_PREFIX, - clickhousePrometheusUpstream, - PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC, -} from '@/routers/api/prometheus'; +import { PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC } from '@/routers/api/prometheus'; const mockFetch = jest.mocked(global.fetch); diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 65faa1c461..8ad5bc86a2 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -1,14 +1,8 @@ import { - buildLogComment, - buildQueryId, parseLogComment, QUERY_ATTRIBUTION_HEADER, } from '@hyperdx/common-utils/dist/clickhouse'; -import { - getCurrentQueryAttribution, - withQueryAttribution, -} from '@hyperdx/common-utils/dist/clickhouse/node'; -import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { withQueryAttribution } from '@hyperdx/common-utils/dist/clickhouse/node'; import express from 'express'; import { omit } from 'lodash'; import { performance } from 'perf_hooks'; @@ -16,17 +10,16 @@ import { Readable } from 'stream'; import { pipeline } from 'stream/promises'; import z from 'zod'; -import { ClickhouseClient } from '@/clickhouse'; -import { getConnectionById } from '@/controllers/connection'; import { - joinPrometheusUpstreamUrl, - takeBasicAuthHeader, + buildPrometheusRequest, + CLICKHOUSE_PROMETHEUS_API_PREFIX, + clickhousePrometheusRequest, + newPromqlClickhouseClient, + PROMETHEUS_MAX_RESOLUTION, + resolveClickhouseQueryBackend, + resolvePromqlTarget, } from '@/controllers/promql'; -import { getSource } from '@/controllers/sources'; import { - connectionSupportsPrometheusHttpApi, - PROMETHEUS_MAX_EXECUTION_SEC, - PROMETHEUS_MAX_RESULT_ROWS, queryInstantViaTableFunction, queryLabelNames, queryLabelValues, @@ -134,50 +127,9 @@ function getParams(req: express.Request): Record { }; } -type PromqlTarget = { - connection: NonNullable>>; - databaseName: string; - tableName: string; -}; - -/** - * Resolve the PromQL source a request names into the connection and table to - * query. The table always comes from the source, never from the request, so a - * caller can only reach data through a source. - */ -async function resolvePromqlTarget( - req: express.Request, - sourceId: string | undefined, - { selectPassword }: { selectPassword: boolean }, -): Promise { - if (!sourceId) { - return { status: 400, error: 'missing required parameter: sourceId' }; - } - - const { teamId } = getNonNullUserWithTeam(req); - - const source = await getSource(teamId.toString(), sourceId); - - if (!source || source.kind !== SourceKind.Promql) { - return { status: 404, error: 'Source not found' }; - } - - const connection = await getConnectionById( - teamId.toString(), - source.connection.toString(), - selectPassword, - ); - - if (!connection) { - return { status: 404, error: 'Connection not found' }; - } - - return { - connection, - databaseName: source.from.databaseName, - tableName: source.from.tableName, - }; -} +/** The request's team as the string the PromQL controller takes. */ +const requestTeamId = (req: express.Request): string => + getNonNullUserWithTeam(req).teamId.toString(); /** * Params for an external Prometheus endpoint. `database` and `table` select a @@ -196,8 +148,6 @@ function prometheusEndpointParams>( // -------------------------- const PROMETHEUS_PROXY_TIMEOUT_MS = 90_000; -const PROMETHEUS_CH_TIMEOUT_MS = 30_000; -const PROMETHEUS_MAX_RESOLUTION = 11_000; // Widest window /query_exemplars will proxy. Prometheus's exemplar store is a // small circular buffer, so a wider range mostly costs a bigger streamed body // for no extra markers — the chart's own thinning caps what's rendered anyway. @@ -245,132 +195,6 @@ const CALLER_SETTABLE_PARAM_KEYS = new Set([ 'table', ]); -// ClickHouse serves the Prometheus HTTP API under this prefix on its main HTTP -// port when the `prometheus_api_v1` handler is configured (26.6+). -export const CLICKHOUSE_PROMETHEUS_API_PREFIX = '/prometheus/api/v1'; - -/** - * The ClickHouse connection host with the query limits pinned as ClickHouse - * HTTP settings. The prometheus_api_v1 handler honours settings given in the - * URL and reports a breach as a Prometheus 400 (pinned by the live int test), - * so this is the only place to bound a PromQL evaluation — the request carries - * no SQL to attach SETTINGS to. Pinned on the host rather than passed as params - * so the merge in proxyToPrometheus (which lets the caller override any - * allowlisted key) can't loosen them. - */ -export function clickhousePrometheusUpstream( - host: string, - { - maxExecutionSec = PROMETHEUS_MAX_EXECUTION_SEC, - maxResultRows = PROMETHEUS_MAX_RESULT_ROWS, - logComment, - }: { - maxExecutionSec?: number; - maxResultRows?: number; - logComment?: string; - } = {}, -): string { - const url = new URL(host); - url.searchParams.set('max_execution_time', String(maxExecutionSec)); - url.searchParams.set('max_result_rows', String(maxResultRows)); - if (logComment) url.searchParams.set('log_comment', logComment); - return url.toString(); -} - -/** - * Where and how to proxy a PromQL request to the connection's prometheus_api_v1 - * handler, tagged with the query attribution in scope. The handler runs the - * query itself rather than through our ClickHouse client, so the tags are set - * here. `log_comment` goes in the URL like the limits. `query_id` goes in a - * header: the handler reads every URL param as a setting, and `query_id` in - * the URL fails the request as an unknown one. - */ -function clickhousePrometheusRequest(connection: { - host: string; - username: string; - password?: string; -}): { upstream: string; headers: Record } { - const attribution = getCurrentQueryAttribution(); - return { - upstream: clickhousePrometheusUpstream(connection.host, { - logComment: buildLogComment(attribution), - }), - headers: { - ...clickhouseAuthHeaders(connection), - 'X-ClickHouse-Query-Id': buildQueryId(attribution), - }, - }; -} - -function newClickhouseClient(connection: { - host: string; - username: string; - password?: string; -}) { - return new ClickhouseClient({ - host: connection.host, - username: connection.username, - password: connection.password, - requestTimeout: PROMETHEUS_CH_TIMEOUT_MS, - }); -} - -/** - * Whether a ClickHouse connection answers PromQL over HTTP. Two conditions, - * cheapest first: the server is new enough (26.6+), and the prometheus_api_v1 - * handler is actually configured — a self-managed server needs an - * `` rule for that, which a stock install lacks. The handler - * answers every request, error or not, with Prometheus JSON (`{status, ...}`); - * a server without it answers a plain-text 404. `format_query` is used as the - * probe because it is the one endpoint that needs no table. - * - * Probed on every request, like the version check, so a handler enabled or - * removed on the server is reflected immediately. - */ -const PROBE_TIMEOUT_MS = 5_000; - -async function clickhouseServesPrometheusHttpApi( - client: ClickhouseClient, - connection: { id: string; host: string; username: string; password?: string }, -): Promise { - if ( - !(await connectionSupportsPrometheusHttpApi({ - client, - connectionId: connection.id, - })) - ) { - return false; - } - - try { - const url = joinPrometheusUpstreamUrl( - connection.host, - `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/format_query`, - ); - url.searchParams.set('query', 'up'); - const resp = await fetch(url, { - headers: { - ...takeBasicAuthHeader(url), - ...clickhouseAuthHeaders(connection), - }, - signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), - }); - return typeof JSON.parse(await resp.text())?.status === 'string'; - } catch { - return false; - } -} - -function clickhouseAuthHeaders(connection: { - username: string; - password?: string; -}): Record { - return { - 'X-ClickHouse-User': connection.username, - 'X-ClickHouse-Key': connection.password ?? '', - }; -} - // Forwards the response straight from the upstream Prometheus to the // HyperDX client. Returns the HTTP status it wrote, so callers can record an // error metric: this helper handles its own failures by writing 400/502/504 and @@ -388,17 +212,33 @@ async function proxyToPrometheus( res: express.Response, headers?: Record, ): Promise { - let url: URL; + // For a key in CALLER_SETTABLE_PARAM_KEYS, the request always wins + // outright, including repeatable ones like `match[]` -- so a Connection + // host can never silently override (or, for `match[]`, narrow) a value + // the caller or this proxy explicitly sets. A host-only param outside + // that set (e.g. VictoriaMetrics's `extra_label`) is left as-is. + const callerParams = Object.fromEntries( + Object.entries(params).flatMap(([k, v]) => + CALLER_SETTABLE_PARAM_KEYS.has(k) && v != null + ? // Prometheus reads a limit of 0 as "unlimited", the same as omitting + // the key entirely -- forward that by omission too, rather than a + // literal "0", so upstream sees the same request shape it always + // has. An empty list still clears a host-pinned `limit`. + [[k, k === 'limit' && v === '0' ? [] : v]] + : [], + ), + ); + + let request: ReturnType; try { - url = joinPrometheusUpstreamUrl(upstreamHost, path); + request = buildPrometheusRequest(upstreamHost, path, callerParams, headers); } catch (err) { // Not echoing `upstreamHost` at all -- a redaction regex on an arbitrary // (possibly malformed, since this is the failure branch) string can // always be wrong for some shape (e.g. a `/` preceding the credentials). - // `err.message` is always safe to show: it's either this function's own - // fixed "Connection host must be http(s)", or `URL`'s parse-failure - // message, which is the fixed string "Invalid URL" and never echoes the - // input (verified against Node's URL implementation). + // `err.message` is always safe to show: it's the fixed "Connection host + // must be http(s)", `URL`'s fixed "Invalid URL", or decodeURIComponent's + // fixed "URI malformed" for bad userinfo; none echo the input. res.status(400).json({ status: 'error', errorType: 'bad_data', @@ -406,49 +246,14 @@ async function proxyToPrometheus( }); return 400; } - // For a key in CALLER_SETTABLE_PARAM_KEYS, the request always wins - // outright, including repeatable ones like `match[]` -- so a Connection - // host can never silently override (or, for `match[]`, narrow) a value - // the caller or this proxy explicitly sets. A host-only param outside - // that set (e.g. VictoriaMetrics's `extra_label`) is left as-is. - for (const [k, v] of Object.entries(params)) { - if (!CALLER_SETTABLE_PARAM_KEYS.has(k)) continue; - if (v == null) continue; - // Clear any host-pinned value at this key first: for a repeatable param - // this prevents an `append` from leaving the host's value(s) alongside - // the request's rather than replacing them. This also means a request's - // `limit=0` still clears a host-pinned `limit` (see below) even though - // it isn't itself re-set. - url.searchParams.delete(k); - // Prometheus reads a limit of 0 as "unlimited", the same as omitting the - // key entirely -- forward that by omission too, rather than a literal - // "0", so upstream sees the same request shape it always has. - if (k === 'limit' && v === '0') continue; - if (Array.isArray(v)) { - for (const item of v) url.searchParams.append(k, item); - } else { - url.searchParams.set(k, v); - } - } - // fetch rejects a URL that carries credentials, so `user:pw@` in the host - // becomes a Basic auth header instead. - const authHeader = takeBasicAuthHeader(url); - const target = url.toString(); - - // A connection host may carry basic-auth credentials (`http://user:pw@host`) - // or a secret pinned in its own query string (e.g. VictoriaMetrics - // `?authKey=...`, preserved by the merge above for any key outside - // `CALLER_SETTABLE_PARAM_KEYS`), and the error bodies below are shown in - // the browser. `url.origin` never includes userinfo (WHATWG URL spec), and - // dropping the query/hash entirely -- rather than trying to redact only the - // secret-shaped parts of it -- means there's no query key to ever miss. - // `target` still carries the query string, and `authHeader` the userinfo. - const redactedTarget = `${url.origin}${url.pathname}`; + // The error bodies below are shown in the browser, so they name only + // `redactedTarget`: never the host's credentials or query string. + const { redactedTarget } = request; let upstreamResp: Response; try { - upstreamResp = await fetch(target, { - headers: { ...authHeader, ...headers }, + upstreamResp = await fetch(request.url, { + headers: request.headers, signal: AbortSignal.timeout(PROMETHEUS_PROXY_TIMEOUT_MS), }); } catch (err) { @@ -562,9 +367,11 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { }); } - const target = await resolvePromqlTarget(req, params.sourceId, { - selectPassword: true, - }); + const target = await resolvePromqlTarget( + requestTeamId(req), + params.sourceId, + { selectPassword: true }, + ); if ('error' in target) { return res.status(target.status).json({ @@ -602,13 +409,11 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { }); } - const client = newClickhouseClient(connection); - - // The Prometheus HTTP API is the only surface ClickHouse holds - // forward-compatible while TimeSeries is in preview, so it is preferred - // wherever the server has it (26.6+); the table function is the fallback. - if (await clickhouseServesPrometheusHttpApi(client, connection)) { - const { upstream, headers } = clickhousePrometheusRequest(connection); + const chBackend = await resolveClickhouseQueryBackend(target); + if (chBackend.kind === 'clickhouse-http') { + const { upstream, headers } = clickhousePrometheusRequest( + chBackend.connection, + ); const status = await proxyToPrometheus( upstream, `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`, @@ -622,8 +427,8 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { // fallback for clickhouse version < 26.6 const result = await queryRangeViaTableFunction({ - client, - connectionId: connection.id, + client: chBackend.client, + connectionId: chBackend.connectionId, databaseName, tableName, expr: query, @@ -672,9 +477,11 @@ const queryHandler: express.RequestHandler = async (req, res) => { }); } - const target = await resolvePromqlTarget(req, params.sourceId, { - selectPassword: true, - }); + const target = await resolvePromqlTarget( + requestTeamId(req), + params.sourceId, + { selectPassword: true }, + ); if ('error' in target) { return res.status(target.status).json({ status: 'error', @@ -697,10 +504,11 @@ const queryHandler: express.RequestHandler = async (req, res) => { } backend = 'clickhouse'; - const client = newClickhouseClient(connection); - - if (await clickhouseServesPrometheusHttpApi(client, connection)) { - const { upstream, headers } = clickhousePrometheusRequest(connection); + const chBackend = await resolveClickhouseQueryBackend(target); + if (chBackend.kind === 'clickhouse-http') { + const { upstream, headers } = clickhousePrometheusRequest( + chBackend.connection, + ); const status = await proxyToPrometheus( upstream, `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query`, @@ -714,7 +522,7 @@ const queryHandler: express.RequestHandler = async (req, res) => { // fallback for clickhouse version < 26.6 const result = await queryInstantViaTableFunction({ - client, + client: chBackend.client, databaseName, tableName, expr: query, @@ -800,9 +608,11 @@ const queryExemplarsHandler: express.RequestHandler = async (req, res) => { // No `selectPassword` — neither branch builds a ClickhouseClient, so there // is no reason to pull the connection secret into request scope. - const target = await resolvePromqlTarget(req, params.sourceId, { - selectPassword: false, - }); + const target = await resolvePromqlTarget( + requestTeamId(req), + params.sourceId, + { selectPassword: false }, + ); if ('error' in target) { return res.status(target.status).json({ status: 'error', @@ -989,7 +799,7 @@ async function handleLabelLookup( } const { sourceId, start, end, limit, match } = parseResult.data; - const target = await resolvePromqlTarget(req, sourceId, { + const target = await resolvePromqlTarget(requestTeamId(req), sourceId, { selectPassword: true, }); if ('error' in target) { @@ -1026,7 +836,7 @@ async function handleLabelLookup( } backend = 'clickhouse'; - const client = newClickhouseClient(connection); + const client = newPromqlClickhouseClient(connection); const startMs = start != null ? Math.floor(start * 1000) : undefined; const endMs = end != null ? Math.ceil(end * 1000) : undefined; From e03116a9eb33e3a830e9dbf9b18e355222fd9fb0 Mon Sep 17 00:00:00 2001 From: Karl Power Date: Tue, 3 Nov 2026 14:21:01 +0100 Subject: [PATCH 2/2] fix: Type PromqlTarget.minAutoGranularity as SQLInterval --- packages/api/src/controllers/promql.ts | 4 ++-- packages/api/src/mcp/tools/query/promqlTile.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/src/controllers/promql.ts b/packages/api/src/controllers/promql.ts index 4f9fd28989..24492d53a2 100644 --- a/packages/api/src/controllers/promql.ts +++ b/packages/api/src/controllers/promql.ts @@ -3,7 +3,7 @@ import { buildQueryId, } from '@hyperdx/common-utils/dist/clickhouse'; import { getCurrentQueryAttribution } from '@hyperdx/common-utils/dist/clickhouse/node'; -import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { SourceKind, type SQLInterval } from '@hyperdx/common-utils/dist/types'; import { z } from 'zod'; import { ClickhouseClient } from '@/clickhouse'; @@ -289,7 +289,7 @@ type PromqlTarget = { databaseName: string; tableName: string; /** The source's scrape interval floor, e.g. "1 minute". */ - minAutoGranularity?: string; + minAutoGranularity?: SQLInterval; }; /** diff --git a/packages/api/src/mcp/tools/query/promqlTile.ts b/packages/api/src/mcp/tools/query/promqlTile.ts index a4e347911b..432111eb0f 100644 --- a/packages/api/src/mcp/tools/query/promqlTile.ts +++ b/packages/api/src/mcp/tools/query/promqlTile.ts @@ -256,7 +256,7 @@ export async function runPromqlTile({ config, dateRange, minGranularitySeconds: target.minAutoGranularity - ? convertGranularityToSeconds(target.minAutoGranularity as SQLInterval) + ? convertGranularityToSeconds(target.minAutoGranularity) : undefined, variables, });