diff --git a/.changeset/mcp-promql-query-tile.md b/.changeset/mcp-promql-query-tile.md new file mode 100644 index 0000000000..19ada9d6df --- /dev/null +++ b/.changeset/mcp-promql-query-tile.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/api': patch +--- + +feat: MCP `query_tile` and `query_tiles` run PromQL tiles against Prometheus endpoints and ClickHouse TimeSeries tables diff --git a/packages/api/src/controllers/__tests__/promql.test.ts b/packages/api/src/controllers/__tests__/promql.test.ts new file mode 100644 index 0000000000..c992b85312 --- /dev/null +++ b/packages/api/src/controllers/__tests__/promql.test.ts @@ -0,0 +1,185 @@ +import { buildPrometheusRequest, runPromqlQuery } from '@/controllers/promql'; + +const mockFetch = jest.mocked(global.fetch); + +const respond = (status: number, body: unknown) => + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify(body), { status }), + ); + +const matrix = { + resultType: 'matrix', + result: [{ metric: { job: 'api' }, values: [[1700000000, '1']] }], +}; + +const prometheus = { kind: 'prometheus' as const, host: 'http://prom:9090' }; +const range = { + kind: 'range' as const, + expr: 'up', + startSec: 1700000000, + endSec: 1700003600, + stepSec: 60, +}; + +const requestedUrl = () => new URL(String(mockFetch.mock.calls[0][0])); + +describe('buildPrometheusRequest', () => { + const host = + 'http://user:p%40ss@vm:8481/select/0/prometheus?extra_label=env%3Dprod&limit=50&step=15s'; + + it('replaces host-pinned params it sets and keeps the rest', () => { + const { url } = buildPrometheusRequest(host, '/api/v1/query_range', { + step: '60', + 'match[]': ['up', 'down'], + limit: [], + }); + + const sent = new URL(url); + expect(`${sent.origin}${sent.pathname}`).toBe( + 'http://vm:8481/select/0/prometheus/api/v1/query_range', + ); + expect(sent.searchParams.get('extra_label')).toBe('env=prod'); + expect(sent.searchParams.getAll('step')).toEqual(['60']); + expect(sent.searchParams.getAll('match[]')).toEqual(['up', 'down']); + expect(sent.searchParams.has('limit')).toBe(false); + }); + + it('moves host credentials into a Basic auth header next to the given headers', () => { + const { url, headers, redactedTarget } = buildPrometheusRequest( + host, + '/api/v1/query', + {}, + { 'X-Extra': '1' }, + ); + + expect(url).not.toContain('user'); + expect(headers).toEqual({ + Authorization: `Basic ${Buffer.from('user:p@ss').toString('base64')}`, + 'X-Extra': '1', + }); + expect(redactedTarget).toBe( + 'http://vm:8481/select/0/prometheus/api/v1/query', + ); + }); + + it('throws on a host that is not http(s)', () => { + expect(() => + buildPrometheusRequest('prometheus:9090', '/api/v1/query', {}), + ).toThrow(/http\(s\)/); + }); +}); + +describe('runPromqlQuery', () => { + afterEach(() => { + mockFetch.mockReset(); + }); + + it('sends a range query to a Prometheus endpoint and returns its data', async () => { + respond(200, { status: 'success', data: matrix }); + + await expect(runPromqlQuery(prometheus, range)).resolves.toEqual(matrix); + + const url = requestedUrl(); + expect(`${url.origin}${url.pathname}`).toBe( + 'http://prom:9090/api/v1/query_range', + ); + expect(Object.fromEntries(url.searchParams)).toEqual({ + query: 'up', + start: '1700000000', + end: '1700003600', + step: '60', + }); + }); + + const clickhouseHttp = { + kind: 'clickhouse-http' as const, + connection: { host: 'http://ch:8123', username: 'default', password: 'pw' }, + databaseName: 'default', + tableName: 'metrics', + }; + const instant = { kind: 'instant' as const, expr: 'up', timeSec: 1700000000 }; + const emptyVector = { + status: 'success', + data: { resultType: 'vector', result: [] }, + }; + + it("sends an instant query to ClickHouse's Prometheus API with the table, limits and credentials", async () => { + respond(200, emptyVector); + + await runPromqlQuery(clickhouseHttp, instant); + + const url = requestedUrl(); + expect(url.pathname).toBe('/prometheus/api/v1/query'); + expect(Object.fromEntries(url.searchParams)).toMatchObject({ + max_execution_time: '30', + max_result_rows: '100000', + query: 'up', + time: '1700000000', + database: 'default', + table: 'metrics', + }); + expect(mockFetch.mock.calls[0][1]?.headers).toMatchObject({ + 'X-ClickHouse-User': 'default', + 'X-ClickHouse-Key': 'pw', + }); + }); + + it('gives each query on one ClickHouse backend its own query ID', async () => { + respond(200, emptyVector); + respond(200, emptyVector); + + await Promise.all([ + runPromqlQuery(clickhouseHttp, instant), + runPromqlQuery(clickhouseHttp, instant), + ]); + + const queryIds = mockFetch.mock.calls.map(([, init]) => + new Headers(init?.headers).get('X-ClickHouse-Query-Id'), + ); + expect(queryIds[0]).toBeTruthy(); + expect(queryIds[0]).not.toBe(queryIds[1]); + }); + + it('reports a rejected expression as a user error', async () => { + respond(400, { + status: 'error', + errorType: 'bad_data', + error: 'parse error: unexpected end of input', + }); + + await expect(runPromqlQuery(prometheus, range)).rejects.toMatchObject({ + message: 'parse error: unexpected end of input', + category: 'user', + }); + }); + + it.each([ + ['a 5xx', 503, 'internal'], + ['a timeout', 400, 'timeout'], + ])('reports %s as a server error', async (_, status, errorType) => { + respond(status, { status: 'error', errorType, error: 'failed' }); + + await expect(runPromqlQuery(prometheus, range)).rejects.toMatchObject({ + category: 'server', + }); + }); + + it('reports an unreachable endpoint as a server error, without its credentials', async () => { + mockFetch.mockRejectedValueOnce(new TypeError('fetch failed')); + + await expect( + runPromqlQuery({ ...prometheus, host: 'http://u:p@prom:9090' }, range), + ).rejects.toMatchObject({ + message: + 'Failed to reach Prometheus at http://prom:9090/api/v1/query_range', + category: 'server', + }); + }); + + it('rejects a range query over the resolution cap before sending it', async () => { + await expect( + runPromqlQuery(prometheus, { ...range, stepSec: 0.1 }), + ).rejects.toMatchObject({ category: 'user' }); + expect(mockFetch).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/controllers/promql.ts b/packages/api/src/controllers/promql.ts index 79e8a943d5..24492d53a2 100644 --- a/packages/api/src/controllers/promql.ts +++ b/packages/api/src/controllers/promql.ts @@ -1,15 +1,34 @@ +import { + buildLogComment, + buildQueryId, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { getCurrentQueryAttribution } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { SourceKind, type SQLInterval } from '@hyperdx/common-utils/dist/types'; import { z } from 'zod'; import { ClickhouseClient } from '@/clickhouse'; import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; import { + connectionSupportsPrometheusHttpApi, + PROMETHEUS_MAX_EXECUTION_SEC, + PROMETHEUS_MAX_RESULT_ROWS, + queryInstantViaTableFunction, queryLabelNames, queryLabelValues, + queryRangeViaTableFunction, } from '@/controllers/timeseriesEngine'; +import { withSpan } from '@/utils/instrumentation'; type PromqlBackend = 'prometheus' | 'clickhouse'; const PROMETHEUS_FETCH_TIMEOUT_MS = 30_000; +const PROMETHEUS_CH_TIMEOUT_MS = 30_000; +export const PROMETHEUS_MAX_RESOLUTION = 11_000; + +// ClickHouse serves the Prometheus HTTP API under this prefix on its main HTTP +// port when the `prometheus_api_v1` handler is configured (26.6+). +export const CLICKHOUSE_PROMETHEUS_API_PREFIX = '/prometheus/api/v1'; /** * Join a Connection host with an absolute Prometheus API path. @@ -50,7 +69,7 @@ export function joinPrometheusUpstreamUrl( * Removes `user:pass@` from `url` and returns it as a Basic auth header. fetch * rejects a URL that carries credentials, so they cannot stay in the host. */ -export function takeBasicAuthHeader(url: URL): Record { +function takeBasicAuthHeader(url: URL): Record { if (!url.username && !url.password) return {}; const credentials = `${decodeURIComponent(url.username)}:${decodeURIComponent(url.password)}`; url.username = ''; @@ -60,19 +79,74 @@ export function takeBasicAuthHeader(url: URL): Record { }; } +/** + * Builds a GET to `path` on a Prometheus-compatible host. Each key in `params` + * replaces whatever the host pins at that key: an array sends every item and an + * empty array clears the key. Keys not in `params`, such as a VictoriaMetrics + * `extra_label` scope, stay as the host set them. Credentials in the host's + * userinfo become a Basic auth header, sent alongside `headers`. Throws when + * the host is not an http(s) URL. + */ +export function buildPrometheusRequest( + host: string, + path: string, + params: Record, + headers?: Record, +): { + url: string; + headers: Record; + /** Origin and path only, safe for error messages shown to the user. */ + redactedTarget: string; +} { + const url = joinPrometheusUpstreamUrl(host, path); + for (const [key, value] of Object.entries(params)) { + url.searchParams.delete(key); + for (const item of Array.isArray(value) ? value : [value]) { + url.searchParams.append(key, item); + } + } + const authHeader = takeBasicAuthHeader(url); + return { + url: url.toString(), + headers: { ...authHeader, ...headers }, + // `origin` never includes userinfo, and the query string may hold a + // host-pinned secret such as VictoriaMetrics' `authKey`. + redactedTarget: `${url.origin}${url.pathname}`, + }; +} + +/** + * A failed PromQL evaluation. `user` means the query itself was rejected (bad + * syntax, unknown function); `server` means the backend was unreachable, timed + * out or failed. + */ +export class PromQLQueryError extends Error { + constructor( + message: string, + readonly category: 'user' | 'server', + ) { + super(message); + this.name = 'PromQLQueryError'; + } +} + const prometheusBodySchema = z.object({ status: z.string(), data: z.unknown(), + errorType: z.string().optional(), error: z.string().optional(), }); +// Prometheus reports these as 4xx/5xx alike; neither says the query was wrong. +const SERVER_ERROR_TYPES = new Set(['timeout', 'canceled', 'unavailable']); + const labelListSchema = z.array(z.string()); /** - * GETs a Prometheus API path from an external Prometheus-compatible endpoint - * and returns the response's `data`, checked against `dataSchema`. Errors name - * only the origin and path: the host may carry credentials in its userinfo or - * query string. + * GETs a Prometheus API path from a Prometheus-compatible endpoint and returns + * the response's `data`, checked against `dataSchema`. Errors name only the + * origin and path: the host may carry credentials in its userinfo or query + * string. */ async function fetchPrometheusData( host: string, @@ -80,25 +154,24 @@ async function fetchPrometheusData( params: Record, dataSchema: T, signal?: AbortSignal, + headers?: Record, ): Promise> { - const url = joinPrometheusUpstreamUrl(host, path); - for (const [key, value] of Object.entries(params)) { - url.searchParams.set(key, value); - } - const redactedTarget = `${url.origin}${url.pathname}`; + const request = buildPrometheusRequest(host, path, params, headers); + const { redactedTarget } = request; const timeout = AbortSignal.timeout(PROMETHEUS_FETCH_TIMEOUT_MS); let resp: Response; try { - resp = await fetch(url, { - headers: takeBasicAuthHeader(url), + resp = await fetch(request.url, { + headers: request.headers, signal: signal ? AbortSignal.any([signal, timeout]) : timeout, }); } catch (err) { - throw new Error( + throw new PromQLQueryError( err instanceof Error && err.name === 'TimeoutError' ? `Prometheus request to ${redactedTarget} timed out` : `Failed to reach Prometheus at ${redactedTarget}`, + 'server', ); } @@ -106,15 +179,21 @@ async function fetchPrometheusData( await resp.json().catch(() => undefined), ); if (!resp.ok || !body.success || body.data.status !== 'success') { - throw new Error( + const isUserError = + resp.status >= 400 && + resp.status < 500 && + !SERVER_ERROR_TYPES.has((body.success && body.data.errorType) || ''); + throw new PromQLQueryError( (body.success && body.data.error) || `Prometheus at ${redactedTarget} returned HTTP ${resp.status}`, + isUserError ? 'user' : 'server', ); } const data = dataSchema.safeParse(body.data.data); if (!data.success) { - throw new Error( + throw new PromQLQueryError( `Prometheus at ${redactedTarget} returned an unexpected response`, + 'server', ); } return data.data; @@ -204,3 +283,376 @@ export async function getPromqlLabelCatalog({ ]); return { backend: 'clickhouse', labelNames, metricNames }; } + +type PromqlTarget = { + connection: NonNullable>>; + databaseName: string; + tableName: string; + /** The source's scrape interval floor, e.g. "1 minute". */ + minAutoGranularity?: SQLInterval; +}; + +/** + * Resolve the PromQL source a request names into the connection and table to + * query. The table always comes from the source, never from the request, so a + * caller can only reach data through a source. + */ +export async function resolvePromqlTarget( + teamId: string, + sourceId: string | undefined, + { selectPassword }: { selectPassword: boolean }, +): Promise { + if (!sourceId) { + return { status: 400, error: 'missing required parameter: sourceId' }; + } + + const source = await getSource(teamId, sourceId); + + if (!source || source.kind !== SourceKind.Promql) { + return { status: 404, error: 'Source not found' }; + } + + const connection = await getConnectionById( + teamId, + source.connection.toString(), + selectPassword, + ); + + if (!connection) { + return { status: 404, error: 'Connection not found' }; + } + + return { + connection, + databaseName: source.from.databaseName, + tableName: source.from.tableName, + minAutoGranularity: source.minAutoGranularity, + }; +} + +/** + * The ClickHouse connection host with the query limits pinned as ClickHouse + * HTTP settings. The prometheus_api_v1 handler honours settings given in the + * URL and reports a breach as a Prometheus 400 (pinned by the live int test), + * so this is the only place to bound a PromQL evaluation — the request carries + * no SQL to attach SETTINGS to. Pinned on the host rather than passed as params + * so the merge in proxyToPrometheus (which lets the caller override any + * allowlisted key) can't loosen them. + */ +export function clickhousePrometheusUpstream( + host: string, + { + maxExecutionSec = PROMETHEUS_MAX_EXECUTION_SEC, + maxResultRows = PROMETHEUS_MAX_RESULT_ROWS, + logComment, + }: { + maxExecutionSec?: number; + maxResultRows?: number; + logComment?: string; + } = {}, +): string { + const url = new URL(host); + url.searchParams.set('max_execution_time', String(maxExecutionSec)); + url.searchParams.set('max_result_rows', String(maxResultRows)); + if (logComment) url.searchParams.set('log_comment', logComment); + return url.toString(); +} + +function clickhouseAuthHeaders(connection: { + username: string; + password?: string; +}): Record { + return { + 'X-ClickHouse-User': connection.username, + 'X-ClickHouse-Key': connection.password ?? '', + }; +} + +type ClickhouseCredentials = { + host: string; + username: string; + password?: string; +}; + +/** + * Where and how to send one PromQL request to the connection's + * prometheus_api_v1 handler, tagged with the query attribution in scope. The + * handler runs the query itself rather than through our ClickHouse client, so + * the tags are set here. `log_comment` goes in the URL like the limits. + * `query_id` goes in a header: the handler reads every URL param as a setting, + * and `query_id` in the URL fails the request as an unknown one. + * + * Call once per request: ClickHouse rejects a query whose ID is already + * running, so concurrent requests must not share the result. + */ +export function clickhousePrometheusRequest( + connection: ClickhouseCredentials, +): { upstream: string; headers: Record } { + const attribution = getCurrentQueryAttribution(); + return { + upstream: clickhousePrometheusUpstream(connection.host, { + logComment: buildLogComment(attribution), + }), + headers: { + ...clickhouseAuthHeaders(connection), + 'X-ClickHouse-Query-Id': buildQueryId(attribution), + }, + }; +} + +export function newPromqlClickhouseClient(connection: { + host: string; + username: string; + password?: string; +}) { + return new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + requestTimeout: PROMETHEUS_CH_TIMEOUT_MS, + }); +} + +/** + * Whether a ClickHouse connection answers PromQL over HTTP. Two conditions, + * cheapest first: the server is new enough (26.6+), and the prometheus_api_v1 + * handler is actually configured — a self-managed server needs an + * `` rule for that, which a stock install lacks. The handler + * answers every request, error or not, with Prometheus JSON (`{status, ...}`); + * a server without it answers a plain-text 404. `format_query` is used as the + * probe because it is the one endpoint that needs no table. + * + * Probed on every request, like the version check, so a handler enabled or + * removed on the server is reflected immediately. + */ +const PROBE_TIMEOUT_MS = 5_000; + +async function clickhouseServesPrometheusHttpApi( + client: ClickhouseClient, + connection: { id: string; host: string; username: string; password?: string }, +): Promise { + if ( + !(await connectionSupportsPrometheusHttpApi({ + client, + connectionId: connection.id, + })) + ) { + return false; + } + + try { + const { url, headers } = buildPrometheusRequest( + connection.host, + `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/format_query`, + { query: 'up' }, + clickhouseAuthHeaders(connection), + ); + const resp = await fetch(url, { + headers, + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + return typeof JSON.parse(await resp.text())?.status === 'string'; + } catch { + return false; + } +} + +type ClickhouseQueryBackend = + | { + kind: 'clickhouse-http'; + connection: ClickhouseCredentials; + databaseName: string; + tableName: string; + } + | { + kind: 'table-function'; + client: ClickhouseClient; + connectionId: string; + databaseName: string; + tableName: string; + }; + +/** How a PromQL query against a source is evaluated. */ +type PromqlQueryBackend = + | { kind: 'prometheus'; host: string } + | ClickhouseQueryBackend; + +/** + * How to evaluate PromQL on a ClickHouse-backed target. The Prometheus HTTP + * API is the only surface ClickHouse holds forward-compatible while TimeSeries + * is in preview, so it is preferred wherever the server has it (26.6+); the + * table function is the fallback. + */ +export async function resolveClickhouseQueryBackend({ + connection, + databaseName, + tableName, +}: PromqlTarget): Promise { + const client = newPromqlClickhouseClient(connection); + if (await clickhouseServesPrometheusHttpApi(client, connection)) { + return { + kind: 'clickhouse-http', + connection: { + host: connection.host, + username: connection.username, + password: connection.password, + }, + databaseName, + tableName, + }; + } + return { + kind: 'table-function', + client, + connectionId: connection.id, + databaseName, + tableName, + }; +} + +/** How to evaluate PromQL on a target. Probes ClickHouse targets for the HTTP API. */ +export async function resolvePromqlQueryBackend( + target: PromqlTarget, +): Promise { + if (target.connection.isPrometheusEndpoint) { + return { kind: 'prometheus', host: target.connection.host }; + } + return resolveClickhouseQueryBackend(target); +} + +export type PromqlQuery = + | { + kind: 'range'; + expr: string; + startSec: number; + endSec: number; + stepSec: number; + } + | { kind: 'instant'; expr: string; timeSec: number }; + +const sampleSchema = z.tuple([z.number(), z.string()]); +const metricSchema = z.record(z.string()); + +const promqlQueryDataSchema = z.discriminatedUnion('resultType', [ + z.object({ + resultType: z.literal('matrix'), + result: z.array( + z.object({ metric: metricSchema, values: z.array(sampleSchema) }), + ), + }), + z.object({ + resultType: z.literal('vector'), + result: z.array(z.object({ metric: metricSchema, value: sampleSchema })), + }), + z.object({ resultType: z.literal('scalar'), result: sampleSchema }), + z.object({ resultType: z.literal('string'), result: sampleSchema }), +]); + +/** The `data` payload of a Prometheus query or query_range response. */ +export type PromqlQueryData = z.infer; + +function promqlQueryParams(query: PromqlQuery): Record { + return query.kind === 'range' + ? { + query: query.expr, + start: String(query.startSec), + end: String(query.endSec), + step: String(query.stepSec), + } + : { query: query.expr, time: String(query.timeSec) }; +} + +async function evaluatePromqlQuery( + backend: PromqlQueryBackend, + query: PromqlQuery, + signal?: AbortSignal, +): Promise { + const endpoint = query.kind === 'range' ? 'query_range' : 'query'; + switch (backend.kind) { + case 'prometheus': + return fetchPrometheusData( + backend.host, + `/api/v1/${endpoint}`, + promqlQueryParams(query), + promqlQueryDataSchema, + signal, + ); + case 'clickhouse-http': { + const { upstream, headers } = clickhousePrometheusRequest( + backend.connection, + ); + return fetchPrometheusData( + upstream, + `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/${endpoint}`, + { + ...promqlQueryParams(query), + database: backend.databaseName, + table: backend.tableName, + }, + promqlQueryDataSchema, + signal, + headers, + ); + } + case 'table-function': + return query.kind === 'range' + ? { + resultType: 'matrix', + result: await queryRangeViaTableFunction({ + client: backend.client, + connectionId: backend.connectionId, + databaseName: backend.databaseName, + tableName: backend.tableName, + expr: query.expr, + startMs: Math.floor(query.startSec * 1000), + endMs: Math.floor(query.endSec * 1000), + stepSec: Math.max(Math.floor(query.stepSec), 1), + signal, + }), + } + : { + resultType: 'vector', + result: await queryInstantViaTableFunction({ + client: backend.client, + databaseName: backend.databaseName, + tableName: backend.tableName, + expr: query.expr, + evalMs: Math.floor(query.timeSec * 1000), + signal, + }), + }; + } +} + +/** + * Evaluates one PromQL query and returns the Prometheus `data` payload. Throws + * PromQLQueryError for a rejected or failed query; the table-function backend + * throws ClickHouse errors as they come. + */ +export async function runPromqlQuery( + backend: PromqlQueryBackend, + query: PromqlQuery, + signal?: AbortSignal, +): Promise { + if ( + query.kind === 'range' && + (query.stepSec <= 0 || + (query.endSec - query.startSec) / query.stepSec > + PROMETHEUS_MAX_RESOLUTION) + ) { + throw new PromQLQueryError( + `exceeded maximum resolution of ${PROMETHEUS_MAX_RESOLUTION.toLocaleString('en-US')} points per timeseries. Try a coarser granularity.`, + 'user', + ); + } + return withSpan( + 'promql.query', + () => evaluatePromqlQuery(backend, query, signal), + { + attributes: { + 'hyperdx.promql.backend': backend.kind, + 'hyperdx.promql.query_kind': query.kind, + }, + }, + ); +} diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts index f99172826e..3f63e8dc17 100644 --- a/packages/api/src/controllers/timeseriesEngine.ts +++ b/packages/api/src/controllers/timeseriesEngine.ts @@ -378,6 +378,7 @@ export async function queryRangeViaTableFunction({ startMs, endMs, stepSec, + signal, }: { client: ClickhouseClient; connectionId: string; @@ -387,6 +388,7 @@ export async function queryRangeViaTableFunction({ startMs: number; endMs: number; stepSec: number; + signal?: AbortSignal; }): Promise { const samplesColumn = await timeSeriesSamplesColumn({ client, @@ -405,6 +407,7 @@ export async function queryRangeViaTableFunction({ stepSec, }, format: 'JSON', + abort_signal: signal, clickhouse_settings: PROMQL_TABLE_FUNCTION_SETTINGS, }); const json = await resp.json(); @@ -417,17 +420,20 @@ export async function queryInstantViaTableFunction({ tableName, expr, evalMs, + signal, }: { client: ClickhouseClient; databaseName: string; tableName: string; expr: string; evalMs: number; + signal?: AbortSignal; }): Promise { const resp = await client.query({ query: `SELECT tags, timestamp, value FROM prometheusQuery({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({evalMs:Int64})) SETTINGS allow_experimental_time_series_table = 1`, query_params: { db: databaseName, table: tableName, expr, evalMs }, format: 'JSON', + abort_signal: signal, clickhouse_settings: PROMQL_TABLE_FUNCTION_SETTINGS, }); const json = await resp.json(); diff --git a/packages/api/src/mcp/__tests__/dashboards/promqlQueryTile.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/promqlQueryTile.int.test.ts new file mode 100644 index 0000000000..31960fe982 --- /dev/null +++ b/packages/api/src/mcp/__tests__/dashboards/promqlQueryTile.int.test.ts @@ -0,0 +1,348 @@ +import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { DisplayType, SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { + DEFAULT_DATABASE, + dropTimeSeriesTable, + seedTimeSeriesTagsTable, +} from '@/fixtures'; +import { callTool, getFirstText } from '@/mcp/__tests__/mcpTestUtils'; +import Connection from '@/models/connection'; +import Dashboard from '@/models/dashboard'; +import { Source } from '@/models/source'; + +import { setupDashboardTests } from './setup'; + +const mockFetch = jest.mocked(global.fetch); +// Set by jest.setup.ts before it stubs `fetch`. +const realFetch: typeof fetch = globalThis.realFetch; + +const TABLE = 'mcp_promql_query_tile'; +const START = 1700000000; +const startTime = new Date(START * 1000).toISOString(); +const endTime = new Date((START + 60) * 1000).toISOString(); + +describe('MCP query_tile - PromQL tiles', () => { + const ctx = setupDashboardTests(); + + beforeAll(async () => { + await seedTimeSeriesTagsTable({ + table: TABLE, + series: ['checkout-0', 'checkout-1'].map(instance => ({ + metricName: 'mcp_up', + tags: { job: 'checkout', instance }, + startSec: START, + endSec: START + 60, + })), + withSamples: true, + }); + }); + + afterAll(async () => { + await dropTimeSeriesTable({ table: TABLE }); + }); + + beforeEach(() => ctx.reconnect({ promqlEnabled: true })); + + afterEach(() => { + jest.restoreAllMocks(); + mockFetch.mockReset(); + }); + + const createClickhouseSource = () => + Source.create({ + kind: SourceKind.Promql, + team: ctx.team._id, + connection: ctx.connection._id, + name: 'Metrics', + from: { databaseName: DEFAULT_DATABASE, tableName: TABLE }, + timestampValueExpression: 'timestamp', + }); + + const createPrometheusSource = async () => { + const connection = await Connection.create({ + team: ctx.team._id, + name: 'Prom', + host: 'http://user:secret@prom.example.com', + username: '', + password: '', + isPrometheusEndpoint: true, + }); + return Source.create({ + kind: SourceKind.Promql, + team: ctx.team._id, + connection: connection._id, + name: 'Prometheus', + from: { databaseName: 'prometheus', tableName: 'prometheus' }, + timestampValueExpression: 'timestamp', + }); + }; + + const createDashboard = ( + sourceId: string, + config: Record, + filters: Record[] = [], + ) => + new Dashboard({ + name: 'PromQL', + tiles: [ + { + id: 'promql-tile', + x: 0, + y: 0, + w: 6, + h: 3, + config: { + configType: 'promql', + name: 'Up', + displayType: DisplayType.Line, + source: sourceId, + granularity: '1 minute', + ...config, + }, + }, + ], + filters, + team: ctx.team._id, + }).save(); + + const queryTile = (dashboardId: string, args: Record = {}) => + callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId, + tileId: 'promql-tile', + startTime, + endTime, + ...args, + }); + + const parseResult = (result: Awaited>) => + JSON.parse(getFirstText(result)).result; + + const lineTile = { + promqlExpression: [{ expression: 'sum by (instance) (mcp_up)' }], + legendTemplate: '{{instance}}', + }; + + const fetchedPaths = () => + mockFetch.mock.calls.map(([url]) => new URL(String(url)).pathname); + + it("returns a line tile's rows through ClickHouse's Prometheus API", async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBeFalsy(); + const { data, seriesLabels } = parseResult(result); + expect(Object.keys(seriesLabels).sort()).toEqual([ + 'checkout-0', + 'checkout-1', + ]); + expect(seriesLabels['checkout-0']).toEqual({ instance: 'checkout-0' }); + expect(data.length).toBeGreaterThan(0); + expect(data).toContainEqual({ + __hdx_time_bucket: '2023-11-14T22:14:00.000Z', + series_name: 'checkout-0', + value: 1, + }); + expect(fetchedPaths()).toContain('/prometheus/api/v1/query_range'); + }); + + it("runs every expression of a tile through ClickHouse's Prometheus API", async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), { + promqlExpression: [ + { expression: 'sum by (instance) (mcp_up)' }, + { expression: 'count(mcp_up)' }, + ], + }); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError ? getFirstText(result) : '').toBe(''); + const { data } = parseResult(result); + const seriesNames = new Set( + data.map((row: { series_name: string }) => row.series_name), + ); + expect(seriesNames.size).toBe(3); + }); + + it('returns the same rows through the table function on ClickHouse < 26.6', async () => { + jest + .spyOn(Metadata.prototype, 'getServerVersion') + .mockResolvedValue([26, 5, 0, 0]); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBeFalsy(); + expect(parseResult(result).data).toContainEqual({ + __hdx_time_bucket: '2023-11-14T22:14:00.000Z', + series_name: 'checkout-0', + value: 1, + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('reduces a number tile to one value', async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), { + displayType: DisplayType.Number, + promqlExpression: [{ expression: 'count(mcp_up)', queryType: 'instant' }], + }); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBeFalsy(); + expect(parseResult(result).data).toEqual([ + { series_name: 'count(mcp_up)', value: 2 }, + ]); + }); + + it('applies the selected dashboard variable', async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard( + source._id.toString(), + { + ...lineTile, + promqlExpression: [ + { + expression: 'sum by (instance) (mcp_up{instance=~"$instance"})', + }, + ], + }, + [ + { + id: 'instance-filter', + type: 'PROMETHEUS_LABEL', + name: 'instance', + variableName: 'instance', + source: source._id.toString(), + label: 'instance', + isBroadcastEnabled: false, + isVariableEnabled: true, + }, + ], + ); + + const result = await queryTile(dashboard._id.toString(), { + variableValues: [{ name: 'instance', values: ['checkout-1'] }], + }); + + expect(result.isError).toBeFalsy(); + expect(Object.keys(parseResult(result).seriesLabels)).toEqual([ + 'checkout-1', + ]); + }); + + it('queries a Prometheus endpoint with the host credentials and no table params', async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + status: 'success', + data: { + resultType: 'matrix', + result: [ + { metric: { instance: 'checkout-0' }, values: [[START, '1']] }, + ], + }, + }), + ), + ); + const source = await createPrometheusSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBeFalsy(); + expect(parseResult(result).data).toEqual([ + { + __hdx_time_bucket: '2023-11-14T22:13:20.000Z', + series_name: 'checkout-0', + value: 1, + }, + ]); + const [url, init] = mockFetch.mock.calls[0]; + const requested = new URL(String(url)); + expect(`${requested.origin}${requested.pathname}`).toBe( + 'http://prom.example.com/api/v1/query_range', + ); + expect(requested.searchParams.has('database')).toBe(false); + expect(requested.searchParams.has('table')).toBe(false); + expect(init?.headers).toMatchObject({ + Authorization: `Basic ${Buffer.from('user:secret').toString('base64')}`, + }); + }); + + it("returns the endpoint's error for a rejected expression", async () => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + status: 'error', + errorType: 'bad_data', + error: 'parse error: unexpected end of input', + }), + { status: 400 }, + ), + ); + const source = await createPrometheusSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('parse error'); + }); + + it('reports an unreachable endpoint without its credentials', async () => { + mockFetch.mockRejectedValueOnce(new TypeError('fetch failed')); + const source = await createPrometheusSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain( + 'Failed to reach Prometheus at http://prom.example.com/api/v1/query_range', + ); + expect(getFirstText(result)).not.toContain('secret'); + }); + + it('reports a PromQL tile with data in query_tiles', async () => { + mockFetch.mockImplementation(realFetch); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await callTool(ctx.client!, 'clickstack_query_tiles', { + dashboardId: dashboard._id.toString(), + startTime, + endTime, + }); + + expect(result.isError).toBeFalsy(); + const { tiles } = JSON.parse(getFirstText(result)); + expect(tiles).toEqual([ + expect.objectContaining({ + tileId: 'promql-tile', + status: 'ok', + hasData: true, + }), + ]); + }); + + it('does not find a PromQL tile with PromQL disabled', async () => { + await ctx.reconnect({ promqlEnabled: false }); + const source = await createClickhouseSource(); + const dashboard = await createDashboard(source._id.toString(), lineTile); + + const result = await queryTile(dashboard._id.toString()); + + expect(result.isError).toBe(true); + expect(getFirstText(result)).toContain('Tile not found'); + }); +}); diff --git a/packages/api/src/mcp/tools/query/__tests__/promqlTile.test.ts b/packages/api/src/mcp/tools/query/__tests__/promqlTile.test.ts new file mode 100644 index 0000000000..971857e2b0 --- /dev/null +++ b/packages/api/src/mcp/tools/query/__tests__/promqlTile.test.ts @@ -0,0 +1,264 @@ +import { DisplayType, PromqlReducer } from '@hyperdx/common-utils/dist/types'; + +import { + buildPromqlTileQueries, + toPromqlTileResult, +} from '@/mcp/tools/query/promqlTile'; + +const start = new Date('2026-11-02T10:00:07Z'); +const end = new Date('2026-11-02T11:00:07Z'); +const dateRange: [Date, Date] = [start, end]; +const sec = (iso: string) => new Date(iso).getTime() / 1000; + +describe('buildPromqlTileQueries', () => { + it('sends a range query per expression of a time chart, on the aligned auto step', () => { + const queries = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [ + { expression: 'avg_over_time(up[$__interval])' }, + { expression: 'sum(up)' }, + ], + }, + dateRange, + }); + + // One hour over 80 buckets resolves to 1 minute; the range widens to + // whole minutes. + expect(queries.map(q => q.query)).toEqual([ + { + kind: 'range', + expr: 'avg_over_time(up[60s])', + startSec: sec('2026-11-02T10:00:00Z'), + endSec: sec('2026-11-02T11:01:00Z'), + stepSec: 60, + }, + { + kind: 'range', + expr: 'sum(up)', + startSec: sec('2026-11-02T10:00:00Z'), + endSec: sec('2026-11-02T11:01:00Z'), + stepSec: 60, + }, + ]); + }); + + it('sends an instant query at the end of the unaligned range', () => { + const queries = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Number, + promqlExpression: [{ expression: 'sum(up)', queryType: 'instant' }], + }, + dateRange, + }); + + expect(queries.map(q => q.query)).toEqual([ + { kind: 'instant', expr: 'sum(up)', timeSec: end.getTime() / 1000 }, + ]); + }); + + it('runs an instant expression on a time chart as a range query', () => { + const [{ query }] = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up', queryType: 'instant' }], + }, + dateRange, + }); + + expect(query.kind).toBe('range'); + }); + + it('queries only the first expression of a pie tile', () => { + const queries = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Pie, + promqlExpression: [{ expression: 'sum(up)' }, { expression: 'up' }], + }, + dateRange, + }); + + expect(queries.map(q => q.expression)).toEqual(['sum(up)']); + }); + + it('expands dashboard variables', () => { + const [{ query }] = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up{job=~"$job"}' }], + }, + dateRange, + variables: [{ name: 'job', values: ['checkout'] }], + }); + + expect(query.expr).toContain('checkout'); + expect(query.expr).not.toContain('$job'); + }); + + it("floors an auto step at the source's minimum granularity", () => { + const [{ query }] = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up' }], + }, + dateRange, + minGranularitySeconds: 300, + }); + + expect(query).toMatchObject({ kind: 'range', stepSec: 300 }); + }); + + it('keeps a granularity the tile picked, unaligned when alignment is off', () => { + const [{ query }] = buildPromqlTileQueries({ + config: { + displayType: DisplayType.Line, + promqlExpression: [{ expression: 'up' }], + granularity: '15 second', + alignDateRangeToGranularity: false, + }, + dateRange, + minGranularitySeconds: 300, + }); + + expect(query).toEqual({ + kind: 'range', + expr: 'up', + startSec: start.getTime() / 1000, + endSec: end.getTime() / 1000, + stepSec: 15, + }); + }); +}); + +describe('toPromqlTileResult', () => { + const checkout0 = { job: 'checkout', instance: 'checkout-0' }; + const checkout1 = { job: 'checkout', instance: 'checkout-1' }; + const t0 = sec('2026-11-02T10:00:00Z'); + const t1 = sec('2026-11-02T10:01:00Z'); + + it('returns time-bucketed rows named by the legend template, labels listed once', () => { + const result = toPromqlTileResult( + { + displayType: DisplayType.Line, + promqlExpression: 'up', + legendTemplate: '{{instance}}', + }, + [ + { + expression: 'up', + isBucketed: true, + result: [ + { + metric: checkout0, + values: [ + [t0, 1], + [t1, 0], + ], + }, + { metric: checkout1, values: [[t0, 1]] }, + ], + }, + ], + ); + + expect(result.data).toEqual([ + { + __hdx_time_bucket: '2026-11-02T10:00:00.000Z', + series_name: 'checkout-0', + value: 1, + }, + { + __hdx_time_bucket: '2026-11-02T10:01:00.000Z', + series_name: 'checkout-0', + value: 0, + }, + { + __hdx_time_bucket: '2026-11-02T10:00:00.000Z', + series_name: 'checkout-1', + value: 1, + }, + ]); + expect(result.rows).toBe(3); + expect(result.seriesLabels).toEqual({ + 'checkout-0': checkout0, + 'checkout-1': checkout1, + }); + }); + + it('reduces each series of a number tile with its reducer, dropping series with no value', () => { + const result = toPromqlTileResult( + { + displayType: DisplayType.Number, + promqlExpression: [{ expression: 'up', reducer: PromqlReducer.Max }], + legendTemplate: '{{instance}}', + }, + [ + { + expression: 'up', + isBucketed: true, + result: [ + { + metric: checkout0, + values: [ + [t0, 2], + [t1, 5], + ], + }, + { metric: checkout1, values: [[t0, NaN]] }, + ], + }, + ], + ); + + expect(result.data).toEqual([{ series_name: 'checkout-0', value: 5 }]); + expect(result.seriesLabels).toEqual({ 'checkout-0': checkout0 }); + }); + + it('defaults a reduced tile to the last non-null sample', () => { + const result = toPromqlTileResult( + { displayType: DisplayType.Bar, promqlExpression: 'up' }, + [ + { + expression: 'up', + isBucketed: true, + result: [ + { + metric: checkout0, + values: [ + [t0, 2], + [t1, 3], + ], + }, + ], + }, + ], + ); + + expect(result.data).toEqual([{ series_name: 'up', value: 3 }]); + }); + + it('leaves out the time bucket for an instant table', () => { + const result = toPromqlTileResult( + { + displayType: DisplayType.Table, + promqlExpression: [{ expression: 'up', queryType: 'instant' }], + }, + [ + { + expression: 'up', + isBucketed: false, + result: [ + { metric: checkout0, values: [[t0, 1]] }, + { metric: checkout1, values: [[t0, 0]] }, + ], + }, + ], + ); + + expect(result.data).toEqual([ + { series_name: '{instance="checkout-0"}', value: 1 }, + { series_name: '{instance="checkout-1"}', value: 0 }, + ]); + expect(result.meta.map(m => m.name)).toEqual(['series_name', 'value']); + }); +}); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index 778555f868..695e13f69f 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -30,6 +30,7 @@ import ms from 'ms'; import { ClickhouseClient } from '@/clickhouse'; import { getConnectionById } from '@/controllers/connection'; +import { PromQLQueryError } from '@/controllers/promql'; import { getSource } from '@/controllers/sources'; import type { McpErrorResult } from '@/mcp/utils/errors'; import { mcpServerError, mcpUserError } from '@/mcp/utils/errors'; @@ -47,6 +48,7 @@ import { trimToolResponse } from '@/utils/trimToolResponse'; import type { ExternalDashboardTileWithId } from '@/utils/zod'; import { externalDashboardTileSchemaWithId } from '@/utils/zod'; +import { runPromqlTile } from './promqlTile'; import { runEventPatterns } from './runEventPatterns'; // ─── Source body expression helpers ────────────────────────────────────────── @@ -709,7 +711,25 @@ export async function runConfigTile( // PromQL is evaluated by the Prometheus API, not the SQL renderer. if (isPromqlSavedChartConfig(savedConfig)) { - return mcpUserError('PromQL tiles cannot be queried through MCP yet.'); + try { + return formatQueryResult( + await runPromqlTile({ + teamId, + config: savedConfig, + dateRange: [startDate, endDate], + variables: options?.variables, + signal: options?.abortSignal, + }), + ); + } catch (e) { + if (e instanceof PromQLQueryError) { + return e.category === 'user' + ? mcpUserError(e.message) + : mcpServerError(e.message); + } + // The table-function backend throws ClickHouse errors as they come. + return clickHouseErrorResult(e); + } } // Raw SQL tile — hydrate source fields for macro support ($__sourceTable, $__filters) diff --git a/packages/api/src/mcp/tools/query/promqlTile.ts b/packages/api/src/mcp/tools/query/promqlTile.ts new file mode 100644 index 0000000000..432111eb0f --- /dev/null +++ b/packages/api/src/mcp/tools/query/promqlTile.ts @@ -0,0 +1,274 @@ +import { + displayTypeSupportsInstantQuery, + displayTypeSupportsReducer, + getQueriedPromqlSeries, + isRangeQuery, + promqlSeriesQueryType, + promqlStep, + reducePromqlSamples, +} from '@hyperdx/common-utils/dist/core/promql'; +import { renderPromqlSeriesNames } from '@hyperdx/common-utils/dist/core/seriesNameTemplate'; +import { + convertDateRangeToGranularityString, + convertGranularityToSeconds, + getAlignedDateRange, +} from '@hyperdx/common-utils/dist/core/utils'; +import { substitutePromqlChartConfigTemplates } from '@hyperdx/common-utils/dist/macros'; +import type { + ChartVariable, + PromqlSavedChartConfig, + PromqlSeries, + SQLInterval, +} from '@hyperdx/common-utils/dist/types'; + +import { + PromqlQuery, + PromqlQueryData, + PromQLQueryError, + resolvePromqlQueryBackend, + resolvePromqlTarget, + runPromqlQuery, +} from '@/controllers/promql'; + +/** The tile fields that decide what a PromQL tile queries and how its result is shaped. */ +type PromqlTileConfig = Pick< + PromqlSavedChartConfig, + | 'promqlExpression' + | 'displayType' + | 'granularity' + | 'alignDateRangeToGranularity' + | 'legendTemplate' +>; + +/** The bucket count the app's time charts size an `auto` granularity for. */ +const AUTO_GRANULARITY_MAX_BUCKETS = 80; + +/** + * The queries a PromQL tile sends, one per expression its display type plots. + * Granularity, alignment and query type follow the app's chart converters, so + * the tile returns the same samples it shows on the dashboard. + */ +export function buildPromqlTileQueries({ + config, + dateRange, + minGranularitySeconds, + variables, +}: { + config: PromqlTileConfig; + dateRange: [Date, Date]; + minGranularitySeconds?: number; + variables?: ChartVariable[]; +}): (PromqlSeries & { query: PromqlQuery })[] { + const granularity: SQLInterval = + !config.granularity || config.granularity === 'auto' + ? convertDateRangeToGranularityString( + dateRange, + AUTO_GRANULARITY_MAX_BUCKETS, + minGranularitySeconds, + ) + : config.granularity; + + const allowsInstant = displayTypeSupportsInstantQuery(config); + + const alignedRange = + (!allowsInstant || isRangeQuery(config)) && + config.alignDateRangeToGranularity !== false + ? getAlignedDateRange(dateRange, granularity) + : dateRange; + + const substituted = substitutePromqlChartConfigTemplates({ + ...config, + granularity, + dateRange: alignedRange, + minGranularitySeconds, + variables, + }); + + const [startSec, endSec] = alignedRange.map(date => date.getTime() / 1000); + + // promqlStep answers a Prometheus duration such as "60s". + const stepSec = Number.parseFloat( + promqlStep(granularity, alignedRange, minGranularitySeconds), + ); + + return getQueriedPromqlSeries(substituted).map(series => ({ + ...series, + query: + allowsInstant && promqlSeriesQueryType(series) === 'instant' + ? { kind: 'instant', expr: series.expression, timeSec: endSec } + : { + kind: 'range', + expr: series.expression, + startSec, + endSec, + stepSec, + }, + })); +} + +type SampledSeries = { + metric: Record; + values: [number, number][]; +}; + +/** One expression's result, with its samples parsed. */ +type PromqlExpressionSamples = PromqlSeries & { + /** Whether the result is bucketed by time. */ + isBucketed: boolean; + result: SampledSeries[]; +}; + +function toSamples( + data: PromqlQueryData, +): Pick { + switch (data.resultType) { + case 'matrix': + return { + isBucketed: true, + result: data.result.map(({ metric, values }) => ({ + metric, + values: values.map(([ts, value]) => [ts, Number.parseFloat(value)]), + })), + }; + case 'vector': + return { + isBucketed: false, + result: data.result.map(({ metric, value }) => ({ + metric, + values: [[value[0], Number.parseFloat(value[1])]], + })), + }; + case 'scalar': + return { + isBucketed: false, + result: [ + { + metric: {}, + values: [[data.result[0], Number.parseFloat(data.result[1])]], + }, + ], + }; + case 'string': + throw new PromQLQueryError( + 'A PromQL chart needs numeric samples; this expression returned a string.', + 'user', + ); + } +} + +/** + * A PromQL tile's result as chart rows, named like the dashboard names its + * series. Number, pie and bar tiles reduce each series to one value. Each + * series' labels are listed once in `seriesLabels` rather than on every row. + */ +export function toPromqlTileResult( + config: PromqlTileConfig, + expressions: PromqlExpressionSamples[], +) { + const names = renderPromqlSeriesNames( + expressions, + config.legendTemplate?.trim() || undefined, + ); + const reduces = displayTypeSupportsReducer(config); + const reducer = getQueriedPromqlSeries(config)[0]?.reducer; + + const isBucketed = !reduces && expressions.some(e => e.isBucketed); + + const data: Record[] = []; + + const seriesLabels = new Map>(); + + // Names come back grouped by expression in series order, so they line up + // with the series once both are flattened. + const seriesNames = names.flat(); + expressions + .flatMap(({ result }) => result) + .forEach(({ metric, values }, i) => { + const series_name = seriesNames.at(i) ?? ''; + if (reduces) { + const value = reducePromqlSamples( + values.map(([, sample]) => sample), + reducer, + ); + // A series with no usable sample reads as no value, not zero. + if (value == null) return; + seriesLabels.set(series_name, metric); + data.push({ series_name, value }); + return; + } + seriesLabels.set(series_name, metric); + for (const [ts, value] of values) { + data.push( + isBucketed + ? { + __hdx_time_bucket: new Date(ts * 1000).toISOString(), + series_name, + value, + } + : { series_name, value }, + ); + } + }); + + return { + data, + meta: [ + ...(isBucketed + ? [{ name: '__hdx_time_bucket', type: 'DateTime64(3)' }] + : []), + { name: 'series_name', type: 'String' }, + { name: 'value', type: 'Float64' }, + ], + rows: data.length, + seriesLabels: Object.fromEntries(seriesLabels), + }; +} + +/** + * Runs a PromQL tile and returns its rows. Throws PromQLQueryError for a + * missing source or a rejected or failed query, and ClickHouse errors from the + * table-function backend. + */ +export async function runPromqlTile({ + teamId, + config, + dateRange, + variables, + signal, +}: { + teamId: string; + config: PromqlTileConfig & Pick; + dateRange: [Date, Date]; + variables?: ChartVariable[]; + signal?: AbortSignal; +}) { + const target = await resolvePromqlTarget(teamId, config.source, { + selectPassword: true, + }); + if ('error' in target) { + throw new PromQLQueryError( + `${target.error}: ${config.source}. Call clickstack_list_sources to discover available source IDs.`, + 'user', + ); + } + + const queries = buildPromqlTileQueries({ + config, + dateRange, + minGranularitySeconds: target.minAutoGranularity + ? convertGranularityToSeconds(target.minAutoGranularity) + : undefined, + variables, + }); + + // One backend for every expression: resolving a ClickHouse backend probes + // the server for its Prometheus HTTP API. + const backend = await resolvePromqlQueryBackend(target); + const expressions = await Promise.all( + queries.map(async ({ query, ...series }) => ({ + ...series, + ...toSamples(await runPromqlQuery(backend, query, signal)), + })), + ); + return toPromqlTileResult(config, expressions); +} diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts index 836b9bdbb8..d82be84618 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts @@ -7,7 +7,11 @@ import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { Types } from 'mongoose'; import * as config from '@/config'; -import { joinPrometheusUpstreamUrl } from '@/controllers/promql'; +import { + CLICKHOUSE_PROMETHEUS_API_PREFIX, + clickhousePrometheusUpstream, + joinPrometheusUpstreamUrl, +} from '@/controllers/promql'; import { DEFAULT_DATABASE, dropTimeSeriesTable, @@ -18,11 +22,7 @@ import { } from '@/fixtures'; import Connection from '@/models/connection'; import { Source } from '@/models/source'; -import { - CLICKHOUSE_PROMETHEUS_API_PREFIX, - clickhousePrometheusUpstream, - PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC, -} from '@/routers/api/prometheus'; +import { PROMETHEUS_MAX_EXEMPLAR_WINDOW_SEC } from '@/routers/api/prometheus'; const mockFetch = jest.mocked(global.fetch); diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 65faa1c461..8ad5bc86a2 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -1,14 +1,8 @@ import { - buildLogComment, - buildQueryId, parseLogComment, QUERY_ATTRIBUTION_HEADER, } from '@hyperdx/common-utils/dist/clickhouse'; -import { - getCurrentQueryAttribution, - withQueryAttribution, -} from '@hyperdx/common-utils/dist/clickhouse/node'; -import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { withQueryAttribution } from '@hyperdx/common-utils/dist/clickhouse/node'; import express from 'express'; import { omit } from 'lodash'; import { performance } from 'perf_hooks'; @@ -16,17 +10,16 @@ import { Readable } from 'stream'; import { pipeline } from 'stream/promises'; import z from 'zod'; -import { ClickhouseClient } from '@/clickhouse'; -import { getConnectionById } from '@/controllers/connection'; import { - joinPrometheusUpstreamUrl, - takeBasicAuthHeader, + buildPrometheusRequest, + CLICKHOUSE_PROMETHEUS_API_PREFIX, + clickhousePrometheusRequest, + newPromqlClickhouseClient, + PROMETHEUS_MAX_RESOLUTION, + resolveClickhouseQueryBackend, + resolvePromqlTarget, } from '@/controllers/promql'; -import { getSource } from '@/controllers/sources'; import { - connectionSupportsPrometheusHttpApi, - PROMETHEUS_MAX_EXECUTION_SEC, - PROMETHEUS_MAX_RESULT_ROWS, queryInstantViaTableFunction, queryLabelNames, queryLabelValues, @@ -134,50 +127,9 @@ function getParams(req: express.Request): Record { }; } -type PromqlTarget = { - connection: NonNullable>>; - databaseName: string; - tableName: string; -}; - -/** - * Resolve the PromQL source a request names into the connection and table to - * query. The table always comes from the source, never from the request, so a - * caller can only reach data through a source. - */ -async function resolvePromqlTarget( - req: express.Request, - sourceId: string | undefined, - { selectPassword }: { selectPassword: boolean }, -): Promise { - if (!sourceId) { - return { status: 400, error: 'missing required parameter: sourceId' }; - } - - const { teamId } = getNonNullUserWithTeam(req); - - const source = await getSource(teamId.toString(), sourceId); - - if (!source || source.kind !== SourceKind.Promql) { - return { status: 404, error: 'Source not found' }; - } - - const connection = await getConnectionById( - teamId.toString(), - source.connection.toString(), - selectPassword, - ); - - if (!connection) { - return { status: 404, error: 'Connection not found' }; - } - - return { - connection, - databaseName: source.from.databaseName, - tableName: source.from.tableName, - }; -} +/** The request's team as the string the PromQL controller takes. */ +const requestTeamId = (req: express.Request): string => + getNonNullUserWithTeam(req).teamId.toString(); /** * Params for an external Prometheus endpoint. `database` and `table` select a @@ -196,8 +148,6 @@ function prometheusEndpointParams>( // -------------------------- const PROMETHEUS_PROXY_TIMEOUT_MS = 90_000; -const PROMETHEUS_CH_TIMEOUT_MS = 30_000; -const PROMETHEUS_MAX_RESOLUTION = 11_000; // Widest window /query_exemplars will proxy. Prometheus's exemplar store is a // small circular buffer, so a wider range mostly costs a bigger streamed body // for no extra markers — the chart's own thinning caps what's rendered anyway. @@ -245,132 +195,6 @@ const CALLER_SETTABLE_PARAM_KEYS = new Set([ 'table', ]); -// ClickHouse serves the Prometheus HTTP API under this prefix on its main HTTP -// port when the `prometheus_api_v1` handler is configured (26.6+). -export const CLICKHOUSE_PROMETHEUS_API_PREFIX = '/prometheus/api/v1'; - -/** - * The ClickHouse connection host with the query limits pinned as ClickHouse - * HTTP settings. The prometheus_api_v1 handler honours settings given in the - * URL and reports a breach as a Prometheus 400 (pinned by the live int test), - * so this is the only place to bound a PromQL evaluation — the request carries - * no SQL to attach SETTINGS to. Pinned on the host rather than passed as params - * so the merge in proxyToPrometheus (which lets the caller override any - * allowlisted key) can't loosen them. - */ -export function clickhousePrometheusUpstream( - host: string, - { - maxExecutionSec = PROMETHEUS_MAX_EXECUTION_SEC, - maxResultRows = PROMETHEUS_MAX_RESULT_ROWS, - logComment, - }: { - maxExecutionSec?: number; - maxResultRows?: number; - logComment?: string; - } = {}, -): string { - const url = new URL(host); - url.searchParams.set('max_execution_time', String(maxExecutionSec)); - url.searchParams.set('max_result_rows', String(maxResultRows)); - if (logComment) url.searchParams.set('log_comment', logComment); - return url.toString(); -} - -/** - * Where and how to proxy a PromQL request to the connection's prometheus_api_v1 - * handler, tagged with the query attribution in scope. The handler runs the - * query itself rather than through our ClickHouse client, so the tags are set - * here. `log_comment` goes in the URL like the limits. `query_id` goes in a - * header: the handler reads every URL param as a setting, and `query_id` in - * the URL fails the request as an unknown one. - */ -function clickhousePrometheusRequest(connection: { - host: string; - username: string; - password?: string; -}): { upstream: string; headers: Record } { - const attribution = getCurrentQueryAttribution(); - return { - upstream: clickhousePrometheusUpstream(connection.host, { - logComment: buildLogComment(attribution), - }), - headers: { - ...clickhouseAuthHeaders(connection), - 'X-ClickHouse-Query-Id': buildQueryId(attribution), - }, - }; -} - -function newClickhouseClient(connection: { - host: string; - username: string; - password?: string; -}) { - return new ClickhouseClient({ - host: connection.host, - username: connection.username, - password: connection.password, - requestTimeout: PROMETHEUS_CH_TIMEOUT_MS, - }); -} - -/** - * Whether a ClickHouse connection answers PromQL over HTTP. Two conditions, - * cheapest first: the server is new enough (26.6+), and the prometheus_api_v1 - * handler is actually configured — a self-managed server needs an - * `` rule for that, which a stock install lacks. The handler - * answers every request, error or not, with Prometheus JSON (`{status, ...}`); - * a server without it answers a plain-text 404. `format_query` is used as the - * probe because it is the one endpoint that needs no table. - * - * Probed on every request, like the version check, so a handler enabled or - * removed on the server is reflected immediately. - */ -const PROBE_TIMEOUT_MS = 5_000; - -async function clickhouseServesPrometheusHttpApi( - client: ClickhouseClient, - connection: { id: string; host: string; username: string; password?: string }, -): Promise { - if ( - !(await connectionSupportsPrometheusHttpApi({ - client, - connectionId: connection.id, - })) - ) { - return false; - } - - try { - const url = joinPrometheusUpstreamUrl( - connection.host, - `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/format_query`, - ); - url.searchParams.set('query', 'up'); - const resp = await fetch(url, { - headers: { - ...takeBasicAuthHeader(url), - ...clickhouseAuthHeaders(connection), - }, - signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), - }); - return typeof JSON.parse(await resp.text())?.status === 'string'; - } catch { - return false; - } -} - -function clickhouseAuthHeaders(connection: { - username: string; - password?: string; -}): Record { - return { - 'X-ClickHouse-User': connection.username, - 'X-ClickHouse-Key': connection.password ?? '', - }; -} - // Forwards the response straight from the upstream Prometheus to the // HyperDX client. Returns the HTTP status it wrote, so callers can record an // error metric: this helper handles its own failures by writing 400/502/504 and @@ -388,17 +212,33 @@ async function proxyToPrometheus( res: express.Response, headers?: Record, ): Promise { - let url: URL; + // For a key in CALLER_SETTABLE_PARAM_KEYS, the request always wins + // outright, including repeatable ones like `match[]` -- so a Connection + // host can never silently override (or, for `match[]`, narrow) a value + // the caller or this proxy explicitly sets. A host-only param outside + // that set (e.g. VictoriaMetrics's `extra_label`) is left as-is. + const callerParams = Object.fromEntries( + Object.entries(params).flatMap(([k, v]) => + CALLER_SETTABLE_PARAM_KEYS.has(k) && v != null + ? // Prometheus reads a limit of 0 as "unlimited", the same as omitting + // the key entirely -- forward that by omission too, rather than a + // literal "0", so upstream sees the same request shape it always + // has. An empty list still clears a host-pinned `limit`. + [[k, k === 'limit' && v === '0' ? [] : v]] + : [], + ), + ); + + let request: ReturnType; try { - url = joinPrometheusUpstreamUrl(upstreamHost, path); + request = buildPrometheusRequest(upstreamHost, path, callerParams, headers); } catch (err) { // Not echoing `upstreamHost` at all -- a redaction regex on an arbitrary // (possibly malformed, since this is the failure branch) string can // always be wrong for some shape (e.g. a `/` preceding the credentials). - // `err.message` is always safe to show: it's either this function's own - // fixed "Connection host must be http(s)", or `URL`'s parse-failure - // message, which is the fixed string "Invalid URL" and never echoes the - // input (verified against Node's URL implementation). + // `err.message` is always safe to show: it's the fixed "Connection host + // must be http(s)", `URL`'s fixed "Invalid URL", or decodeURIComponent's + // fixed "URI malformed" for bad userinfo; none echo the input. res.status(400).json({ status: 'error', errorType: 'bad_data', @@ -406,49 +246,14 @@ async function proxyToPrometheus( }); return 400; } - // For a key in CALLER_SETTABLE_PARAM_KEYS, the request always wins - // outright, including repeatable ones like `match[]` -- so a Connection - // host can never silently override (or, for `match[]`, narrow) a value - // the caller or this proxy explicitly sets. A host-only param outside - // that set (e.g. VictoriaMetrics's `extra_label`) is left as-is. - for (const [k, v] of Object.entries(params)) { - if (!CALLER_SETTABLE_PARAM_KEYS.has(k)) continue; - if (v == null) continue; - // Clear any host-pinned value at this key first: for a repeatable param - // this prevents an `append` from leaving the host's value(s) alongside - // the request's rather than replacing them. This also means a request's - // `limit=0` still clears a host-pinned `limit` (see below) even though - // it isn't itself re-set. - url.searchParams.delete(k); - // Prometheus reads a limit of 0 as "unlimited", the same as omitting the - // key entirely -- forward that by omission too, rather than a literal - // "0", so upstream sees the same request shape it always has. - if (k === 'limit' && v === '0') continue; - if (Array.isArray(v)) { - for (const item of v) url.searchParams.append(k, item); - } else { - url.searchParams.set(k, v); - } - } - // fetch rejects a URL that carries credentials, so `user:pw@` in the host - // becomes a Basic auth header instead. - const authHeader = takeBasicAuthHeader(url); - const target = url.toString(); - - // A connection host may carry basic-auth credentials (`http://user:pw@host`) - // or a secret pinned in its own query string (e.g. VictoriaMetrics - // `?authKey=...`, preserved by the merge above for any key outside - // `CALLER_SETTABLE_PARAM_KEYS`), and the error bodies below are shown in - // the browser. `url.origin` never includes userinfo (WHATWG URL spec), and - // dropping the query/hash entirely -- rather than trying to redact only the - // secret-shaped parts of it -- means there's no query key to ever miss. - // `target` still carries the query string, and `authHeader` the userinfo. - const redactedTarget = `${url.origin}${url.pathname}`; + // The error bodies below are shown in the browser, so they name only + // `redactedTarget`: never the host's credentials or query string. + const { redactedTarget } = request; let upstreamResp: Response; try { - upstreamResp = await fetch(target, { - headers: { ...authHeader, ...headers }, + upstreamResp = await fetch(request.url, { + headers: request.headers, signal: AbortSignal.timeout(PROMETHEUS_PROXY_TIMEOUT_MS), }); } catch (err) { @@ -562,9 +367,11 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { }); } - const target = await resolvePromqlTarget(req, params.sourceId, { - selectPassword: true, - }); + const target = await resolvePromqlTarget( + requestTeamId(req), + params.sourceId, + { selectPassword: true }, + ); if ('error' in target) { return res.status(target.status).json({ @@ -602,13 +409,11 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { }); } - const client = newClickhouseClient(connection); - - // The Prometheus HTTP API is the only surface ClickHouse holds - // forward-compatible while TimeSeries is in preview, so it is preferred - // wherever the server has it (26.6+); the table function is the fallback. - if (await clickhouseServesPrometheusHttpApi(client, connection)) { - const { upstream, headers } = clickhousePrometheusRequest(connection); + const chBackend = await resolveClickhouseQueryBackend(target); + if (chBackend.kind === 'clickhouse-http') { + const { upstream, headers } = clickhousePrometheusRequest( + chBackend.connection, + ); const status = await proxyToPrometheus( upstream, `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`, @@ -622,8 +427,8 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { // fallback for clickhouse version < 26.6 const result = await queryRangeViaTableFunction({ - client, - connectionId: connection.id, + client: chBackend.client, + connectionId: chBackend.connectionId, databaseName, tableName, expr: query, @@ -672,9 +477,11 @@ const queryHandler: express.RequestHandler = async (req, res) => { }); } - const target = await resolvePromqlTarget(req, params.sourceId, { - selectPassword: true, - }); + const target = await resolvePromqlTarget( + requestTeamId(req), + params.sourceId, + { selectPassword: true }, + ); if ('error' in target) { return res.status(target.status).json({ status: 'error', @@ -697,10 +504,11 @@ const queryHandler: express.RequestHandler = async (req, res) => { } backend = 'clickhouse'; - const client = newClickhouseClient(connection); - - if (await clickhouseServesPrometheusHttpApi(client, connection)) { - const { upstream, headers } = clickhousePrometheusRequest(connection); + const chBackend = await resolveClickhouseQueryBackend(target); + if (chBackend.kind === 'clickhouse-http') { + const { upstream, headers } = clickhousePrometheusRequest( + chBackend.connection, + ); const status = await proxyToPrometheus( upstream, `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query`, @@ -714,7 +522,7 @@ const queryHandler: express.RequestHandler = async (req, res) => { // fallback for clickhouse version < 26.6 const result = await queryInstantViaTableFunction({ - client, + client: chBackend.client, databaseName, tableName, expr: query, @@ -800,9 +608,11 @@ const queryExemplarsHandler: express.RequestHandler = async (req, res) => { // No `selectPassword` — neither branch builds a ClickhouseClient, so there // is no reason to pull the connection secret into request scope. - const target = await resolvePromqlTarget(req, params.sourceId, { - selectPassword: false, - }); + const target = await resolvePromqlTarget( + requestTeamId(req), + params.sourceId, + { selectPassword: false }, + ); if ('error' in target) { return res.status(target.status).json({ status: 'error', @@ -989,7 +799,7 @@ async function handleLabelLookup( } const { sourceId, start, end, limit, match } = parseResult.data; - const target = await resolvePromqlTarget(req, sourceId, { + const target = await resolvePromqlTarget(requestTeamId(req), sourceId, { selectPassword: true, }); if ('error' in target) { @@ -1026,7 +836,7 @@ async function handleLabelLookup( } backend = 'clickhouse'; - const client = newClickhouseClient(connection); + const client = newPromqlClickhouseClient(connection); const startMs = start != null ? Math.floor(start * 1000) : undefined; const endMs = end != null ? Math.ceil(end * 1000) : undefined;