[FEATURE] Prometheus: add Exemplars support

docs/prometheus/README.md

@@ -12,6 +12,8 @@ The Prometheus data source is the base building block that enables the connectio
 
 It supports the [proxy](https://perses.dev/perses/docs/concepts/proxy/) feature of Perses that allows to restrict the access to your data source.
 
+It also provides an opt-in support for [exemplars](#exemplars).
+
 See also technical docs related to this plugin:
 
 - [Data model](./model.md#prometheusdatasource)
@@ -78,6 +80,35 @@ See also technical docs related to this plugin:
 - [Data model](./model.md#prometheustimeseriesquery)
 - [Dashboard-as-Code Go lib](./go-sdk/query.md)
 
+## Exemplars
+
+[Exemplars](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-exemplars) are references to related data — typically a trace ID — attached to individual samples of a metric. They let you go from an aggregate metric to a concrete representative request, which is handy to investigate a spike or a latency outlier directly from a dashboard.
+
+The Prometheus package supports displaying exemplars on the [Time Series Chart](../timeserieschart/README.md) panel:
+
+- When exemplars are enabled on the datasource (see below), the `PrometheusTimeSeriesQuery` plugin calls the Prometheus [`/api/v1/query_exemplars`](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-exemplars) endpoint in parallel with every range query, and attaches the returned exemplars to the series matching their labels.
+- On the chart, each exemplar is rendered as a diamond marker placed at its value and timestamp, using the color of the series it belongs to. Clicking a marker opens a dialog with its metadata: the labels of the series it belongs to, the exemplar labels (like `trace_id`), its value and its timestamp (e.g. to copy a trace ID and open the corresponding trace in your tracing UI).
+- A failed exemplars request never impacts the panel itself: the query results are displayed without exemplars.
+
+### Enabling exemplars
+
+Exemplars are opt-in and configured at the datasource level. You can either toggle the **Enable exemplars** switch in the Prometheus datasource editor, or set the following in the datasource spec:
+
+```yaml
+kind: "PrometheusDatasource"
+spec:
+  exemplars:
+    enable: true
+```
+
+When using the Dashboard-as-Code Go lib, the same can be achieved with the `datasource.EnableExemplars()` option (see [Go SDK](./go-sdk/datasource.md#exemplars)).
+
+Some requirements and caveats to be aware of:
+
+- The Prometheus instance (or any Prometheus-compatible backend) must have the [exemplar storage](https://prometheus.io/docs/prometheus/latest/feature_flags/#exemplars-storage) enabled, and your telemetry pipeline must actually attach exemplars to the samples (e.g. via the OTLP receiver).
+- Exemplars are only fetched for range queries: queries executed in instant mode never display exemplars.
+- When relying on the [proxy](https://perses.dev/perses/docs/concepts/proxy/) feature, make sure the `/api/v1/query_exemplars` endpoint is allowed so exemplars can be fetched through the Perses server.
+
 ## Annotation (`PrometheusPromQLAnnotation`)
 
 The annotation plugin overlays contextual events on the panels of a dashboard, like deployments, incidents or maintenance windows. The provided PromQL expression is executed as a range query over the time range of the dashboard, and each series it returns becomes an annotation spanning from the timestamp of its first sample to the timestamp of its last one.

docs/prometheus/go-sdk/datasource.md

@@ -58,6 +58,19 @@ Configure query parameters to be appended to all Prometheus API requests. This i
 - Resolution control (`max_source_resolution=0s`)
 - Any custom query parameters required by your Prometheus setup
 
+#### Exemplars
+
+```golang
+import "github.com/perses/plugins/prometheus/sdk/go/datasource"
+
+datasource.EnableExemplars()
+```
+
+Enable exemplar support on the datasource: the Prometheus query plugin will also query the
+`/api/v1/query_exemplars` endpoint when running range queries, so exemplars can be displayed on
+capable panels like the Time Series Chart. See the [plugin documentation](../README.md#exemplars) for
+the requirements (exemplar storage enabled on the Prometheus side, etc.).
+
 ## Examples
 
 ```golang

docs/prometheus/model.md

@@ -18,6 +18,11 @@ spec:
   proxy: <HTTP Proxy specification> # Optional
 
   scrapeInterval: <duration> # Optional
+
+  # Optional, enables the exemplars support: the Prometheus query plugin
+  # will also query the /api/v1/query_exemplars endpoint.
+  exemplars:
+    enable: <boolean>
 ```
 
 ### HTTP Proxy specification

prometheus/schemas/datasource/prometheus.cue

@@ -23,6 +23,9 @@ spec: {
 	datasource.#HTTPDatasourceSpec
 	scrapeInterval?: =~#durationRegex
 	queryParams?: {[string]: string}
+	exemplars?: {
+		enable: bool
+	}
 }
 
 #kind: "PrometheusDatasource"

prometheus/sdk/go/datasource/datasource.go

@@ -31,6 +31,13 @@ type PluginSpec struct {
 	Proxy          *http.Proxy       `json:"proxy,omitempty" yaml:"proxy,omitempty"`
 	ScrapeInterval common.Duration   `json:"scrapeInterval,omitempty" yaml:"scrapeInterval,omitempty"`
 	QueryParams    map[string]string `json:"queryParams,omitempty" yaml:"queryParams,omitempty"`
+	Exemplars      *Exemplars        `json:"exemplars,omitempty" yaml:"exemplars,omitempty"`
+}
+
+// Exemplars mirrors the exemplars section of the PrometheusDatasource CUE schema.
+// When enabled, the Prometheus query plugin also queries the /api/v1/query_exemplars endpoint.
+type Exemplars struct {
+	Enable bool `json:"enable" yaml:"enable"`
 }
 
 func (s *PluginSpec) UnmarshalJSON(data []byte) error {

prometheus/sdk/go/datasource/datasource_test.go

@@ -0,0 +1,63 @@
+// Copyright The Perses Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package datasource
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestPrometheusDatasourceBuilder(t *testing.T) {
+	builder, err := create(DirectURL("https://prometheus.demo.do.prometheus.io"))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if builder.DirectURL != "https://prometheus.demo.do.prometheus.io" {
+		t.Errorf("directUrl mismatch: %s", builder.DirectURL)
+	}
+	if builder.Exemplars != nil {
+		t.Errorf("exemplars should not be set: %v", builder.Exemplars)
+	}
+}
+
+func TestEnableExemplars(t *testing.T) {
+	builder, err := create(
+		DirectURL("https://prometheus.demo.do.prometheus.io"),
+		EnableExemplars(),
+	)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if builder.Exemplars == nil || !builder.Exemplars.Enable {
+		t.Fatalf("exemplars should be enabled: %v", builder.Exemplars)
+	}
+
+	raw, err := json.Marshal(builder.PluginSpec)
+	if err != nil {
+		t.Fatalf("marshal: %v", err)
+	}
+
+	var out map[string]any
+	if err := json.Unmarshal(raw, &out); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+
+	exemplars, ok := out["exemplars"].(map[string]any)
+	if !ok {
+		t.Fatalf("exemplars section is missing in the serialized spec: %v", out)
+	}
+	if enable, ok := exemplars["enable"].(bool); !ok || !enable {
+		t.Errorf("exemplars.enable mismatch: %v", exemplars["enable"])
+	}
+}

prometheus/sdk/go/datasource/options.go

@@ -51,3 +51,12 @@ func QueryParam(key, value string) Option {
 		return nil
 	}
 }
+
+// EnableExemplars enables exemplar support on the datasource: the Prometheus query plugin
+// will also query the /api/v1/query_exemplars endpoint when running range queries.
+func EnableExemplars() Option {
+	return func(builder *Builder) error {
+		builder.Exemplars = &Exemplars{Enable: true}
+		return nil
+	}
+}

prometheus/src/model/api-types.ts

@@ -101,6 +101,26 @@ export interface RangeQueryRequestParameters {
 
 export type RangeQueryResponse = ApiResponse<MatrixData>;
 
+// Ref https://prometheus.io/docs/prometheus/latest/querying/api/#querying-exemplars
+export interface QueryExemplarsRequestParameters {
+  query: string;
+  start: UnixTimestampSeconds;
+  end: UnixTimestampSeconds;
+}
+
+export interface ExemplarData {
+  labels: Metric;
+  value: string;
+  timestamp: UnixTimestampSeconds;
+}
+
+export interface ExemplarSeries {
+  seriesLabels: Metric;
+  exemplars: ExemplarData[];
+}
+
+export type QueryExemplarsResponse = ApiResponse<ExemplarSeries[]>;
+
 export interface SeriesRequestParameters {
   'match[]': string[];
   start?: UnixTimestampSeconds;

prometheus/src/model/prometheus-client.ts

@@ -27,6 +27,8 @@ import type {
   MetricMetadataResponse,
   ParseQueryRequestParameters,
   ParseQueryResponse,
+  QueryExemplarsRequestParameters,
+  QueryExemplarsResponse,
   RangeQueryRequestParameters,
   RangeQueryResponse,
   SeriesRequestParameters,
@@ -49,6 +51,10 @@ export interface PrometheusClient extends DatasourceClient {
   options: PrometheusClientOptions;
   instantQuery(params: InstantQueryRequestParameters, options?: ClientRequestOptions): Promise<InstantQueryResponse>;
   rangeQuery(params: RangeQueryRequestParameters, options?: ClientRequestOptions): Promise<RangeQueryResponse>;
+  queryExemplars(
+    params: QueryExemplarsRequestParameters,
+    options?: ClientRequestOptions,
+  ): Promise<QueryExemplarsResponse>;
   labelNames(params: LabelNamesRequestParameters, options?: ClientRequestOptions): Promise<LabelNamesResponse>;
   labelValues(params: LabelValuesRequestParameters, options?: ClientRequestOptions): Promise<LabelValuesResponse>;
   metricMetadata(
@@ -134,6 +140,20 @@ export function rangeQuery(
   return fetchWithPost<RangeQueryRequestParameters, RangeQueryResponse>('/api/v1/query_range', params, queryOptions);
 }
 
+/**
+ * Calls the `/api/v1/query_exemplars` endpoint to get exemplar data for a query.
+ */
+export function queryExemplars(
+  params: QueryExemplarsRequestParameters,
+  queryOptions: QueryOptions,
+): Promise<QueryExemplarsResponse> {
+  return fetchWithGet<QueryExemplarsRequestParameters, QueryExemplarsResponse>(
+    '/api/v1/query_exemplars',
+    params,
+    queryOptions,
+  );
+}
+
 /**
  * Calls the `/api/v1/labels` endpoint to get a list of label names.
  */

prometheus/src/plugins/PrometheusDatasourceEditor.tsx

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Box, IconButton, TextField, Typography } from '@mui/material';
+import { Box, FormControlLabel, IconButton, Switch, TextField, Typography } from '@mui/material';
 import type { QueryParamValues } from '@perses-dev/components';
 import type { DatasourceEditorProps } from '@perses-dev/plugin-system';
 import { HTTPSettingsEditor } from '@perses-dev/plugin-system';
@@ -133,6 +133,10 @@ export function PrometheusDatasourceEditor(props: PrometheusDatasourceEditorProp
             endpointPattern: '/api/v1/query_range',
             method: 'POST',
           },
+          {
+            endpointPattern: '/api/v1/query_exemplars',
+            method: 'GET',
+          },
           {
             endpointPattern: '/api/v1/label/([a-zA-Z0-9_-]+)/values',
             method: 'GET',
@@ -165,6 +169,24 @@ export function PrometheusDatasourceEditor(props: PrometheusDatasourceEditorProp
         onChange={(e) => onChange({ ...value, scrapeInterval: e.target.value as DurationString })}
         helperText="Set it to match the typical scrape interval used in your Prometheus instance."
       />
+      <Typography variant="h5" mt={2} mb={1}>
+        Exemplars
+      </Typography>
+      <FormControlLabel
+        control={
+          <Switch
+            checked={value.exemplars?.enable ?? false}
+            disabled={isReadonly}
+            onChange={(e) =>
+              onChange({
+                ...value,
+                exemplars: e.target.checked ? { enable: true } : undefined,
+              })
+            }
+          />
+        }
+        label="Enable exemplars"
+      />
       <HTTPSettingsEditor
         value={value}
         onChange={onChange}

prometheus/src/plugins/prometheus-datasource.test.ts

@@ -66,4 +66,38 @@ describe('PrometheusDatasource query parameters', () => {
 
     mockFetch.mockClear();
   });
+
+  it('should call the query_exemplars endpoint with query parameters', async () => {
+    const spec: PrometheusDatasourceSpec = {
+      directUrl: 'http://localhost:9090',
+      queryParams: {
+        dedup: 'false',
+      },
+      exemplars: {
+        enable: true,
+      },
+    };
+
+    const client = PrometheusDatasource.createClient(spec, {});
+
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      statusText: 'OK',
+      json: () => Promise.resolve({ data: [] }),
+    });
+    global.fetch = mockFetch;
+
+    const response = await client.queryExemplars({
+      query: 'up',
+      start: 1700000000,
+      end: 1700003600,
+    });
+
+    expect(response.data).toEqual([]);
+    expect(mockFetch).toHaveBeenCalledWith(
+      'http://localhost:9090/api/v1/query_exemplars?query=up&start=1700000000&end=1700003600&dedup=false',
+      expect.objectContaining({ method: 'GET' }),
+    );
+  });
 });

prometheus/src/plugins/prometheus-datasource.tsx

@@ -25,6 +25,7 @@ import {
   mergeQueryParams,
   metricMetadata,
   parseQuery,
+  queryExemplars,
   rangeQuery,
   series,
 } from '../model';
@@ -69,6 +70,7 @@ const createClient: DatasourcePlugin<PrometheusDatasourceSpec, PrometheusClient>
     healthCheck: healthCheck({ datasourceUrl, headers: specHeaders, queryParams }),
     instantQuery: wrapClientMethod(instantQuery, datasourceUrl, specHeaders, queryParams),
     rangeQuery: wrapClientMethod(rangeQuery, datasourceUrl, specHeaders, queryParams),
+    queryExemplars: wrapClientMethod(queryExemplars, datasourceUrl, specHeaders, queryParams),
     labelNames: wrapClientMethod(labelNames, datasourceUrl, specHeaders, queryParams),
     labelValues: wrapClientMethod(labelValues, datasourceUrl, specHeaders, queryParams),
     metricMetadata: wrapClientMethod(metricMetadata, datasourceUrl, specHeaders, queryParams),