Tempo: Add support for TraceQL Metrics exemplars (#96859)

* Add support for exemplars

* TraceQL metrics exemplars ftw

* Fix exemplars on histogram queries

* Added exemplars field for the query builder options

* Add series labels as exemplars dataframe fields so the panel can link series with exemplars

* Fix tests

* Fix lint

* Hide exemplars field from options

* Fix crash on histogram queries

* Use DataTopicAnnotations enum

* Fix test

packages/grafana-schema/src/raw/composable/tempo/dataquery/x/TempoDataQuery_types.gen.ts

@@ -13,6 +13,10 @@ import * as common from '@grafana/schema';
 export const pluginVersion = "%VERSION%";
 
 export interface TempoQuery extends common.DataQuery {
+  /**
+   * For metric queries, how many exemplars to request, 0 means no exemplars
+   */
+  exemplars?: number;
   filters: Array<TraceqlFilter>;
   /**
    * Filters that are used to query the metrics summary

pkg/tsdb/tempo/kinds/dataquery/types_dataquery_gen.go

@@ -85,6 +85,9 @@ type TempoQuery struct {
 	// TODO find a better way to do this ^ that's friendly to schema
 	// TODO this shouldn't be unknown but DataSourceRef | null
 	Datasource *any `json:"datasource,omitempty"`
+
+	// For metric queries, how many exemplars to request, 0 means no exemplars
+	Exemplars *int64          `json:"exemplars,omitempty"`
 	Filters   []TraceqlFilter `json:"filters,omitempty"`
 
 	// Filters that are used to query the metrics summary

pkg/tsdb/tempo/traceql/exemplars.go

@@ -0,0 +1,59 @@
+package traceql
+
+import (
+	"strings"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/tempo/pkg/tempopb"
+)
+
+func transformExemplarToFrame(name string, series *tempopb.TimeSeries) *data.Frame {
+	exemplars := series.Exemplars
+
+	// Setup fields for basic data
+	fields := []*data.Field{
+		data.NewField("Time", nil, []time.Time{}),
+		data.NewField("Value", nil, []float64{}),
+		data.NewField("traceId", nil, []string{}),
+	}
+
+	fields[2].Config = &data.FieldConfig{
+		DisplayName: "Trace ID",
+	}
+
+	// Add fields for each label to be able to link exemplars to the series
+	for _, label := range series.Labels {
+		fields = append(fields, data.NewField(label.GetKey(), nil, []string{}))
+	}
+
+	frame := &data.Frame{
+		RefID:  name,
+		Name:   "exemplar",
+		Fields: fields,
+		Meta: &data.FrameMeta{
+			DataTopic: data.DataTopicAnnotations,
+		},
+	}
+
+	for _, exemplar := range exemplars {
+		_, labels := transformLabelsAndGetName(exemplar.GetLabels())
+		traceId := labels["trace:id"]
+		if traceId != "" {
+			traceId = strings.ReplaceAll(traceId, "\"", "")
+		}
+
+		// Add basic data
+		frame.AppendRow(time.UnixMilli(exemplar.GetTimestampMs()), exemplar.GetValue(), traceId)
+
+		// Add labels
+		for _, label := range series.Labels {
+			field, _ := frame.FieldByName(label.GetKey())
+			if field != nil {
+				val, _ := metricsValueToString(label.GetValue())
+				field.Append(val)
+			}
+		}
+	}
+	return frame
+}

pkg/tsdb/tempo/traceql/exemplars_test.go

@@ -0,0 +1,137 @@
+package traceql
+
+import (
+	"testing"
+	"time"
+
+	"github.com/grafana/tempo/pkg/tempopb"
+	v1 "github.com/grafana/tempo/pkg/tempopb/common/v1"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTransformExemplarToFrame_EmptyExemplars(t *testing.T) {
+	frame := transformExemplarToFrame("test", &tempopb.TimeSeries{
+		Labels:     nil,
+		Samples:    nil,
+		PromLabels: "",
+		Exemplars:  make([]tempopb.Exemplar, 0),
+	})
+	assert.NotNil(t, frame)
+	assert.Equal(t, "test", frame.RefID)
+	assert.Equal(t, "exemplar", frame.Name)
+	assert.Len(t, frame.Fields, 3)
+	assert.Empty(t, frame.Fields[0].Len())
+	assert.Empty(t, frame.Fields[1].Len())
+	assert.Empty(t, frame.Fields[2].Len())
+}
+
+func TestTransformExemplarToFrame_SingleExemplar(t *testing.T) {
+	exemplars := []tempopb.Exemplar{
+		{
+			TimestampMs: 1638316800000,
+			Value:       1.23,
+			Labels: []v1.KeyValue{
+				{Key: "trace:id", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "trace-123"}}},
+			},
+		},
+	}
+	frame := transformExemplarToFrame("test", &tempopb.TimeSeries{
+		Labels:     nil,
+		Samples:    nil,
+		PromLabels: "",
+		Exemplars:  exemplars,
+	})
+	assert.NotNil(t, frame)
+	assert.Equal(t, "test", frame.RefID)
+	assert.Equal(t, "exemplar", frame.Name)
+	assert.Len(t, frame.Fields, 3)
+	assert.Equal(t, time.UnixMilli(1638316800000), frame.Fields[0].At(0))
+	assert.Equal(t, 1.23, frame.Fields[1].At(0))
+	assert.Equal(t, "trace-123", frame.Fields[2].At(0))
+}
+
+func TestTransformExemplarToFrame_SingleExemplarHistogram(t *testing.T) {
+	exemplars := []tempopb.Exemplar{
+		{
+			TimestampMs: 1638316800000,
+			Value:       1.23,
+			Labels: []v1.KeyValue{
+				{Key: "trace:id", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "trace-123"}}},
+				{Key: "__bucket", Value: &v1.AnyValue{Value: &v1.AnyValue_DoubleValue{DoubleValue: 1.23}}},
+			},
+		},
+	}
+	frame := transformExemplarToFrame("test", &tempopb.TimeSeries{
+		Labels: []v1.KeyValue{
+			{Key: "__bucket", Value: &v1.AnyValue{Value: &v1.AnyValue_DoubleValue{DoubleValue: 1.23}}},
+		},
+		Samples:    nil,
+		PromLabels: "",
+		Exemplars:  exemplars,
+	})
+	assert.NotNil(t, frame)
+	assert.Equal(t, "test", frame.RefID)
+	assert.Equal(t, "exemplar", frame.Name)
+	assert.Len(t, frame.Fields, 4)
+	assert.Equal(t, time.UnixMilli(1638316800000), frame.Fields[0].At(0))
+	assert.Equal(t, 1.23, frame.Fields[1].At(0))
+	assert.Equal(t, "trace-123", frame.Fields[2].At(0))
+}
+
+func TestTransformExemplarToFrame_MultipleExemplars(t *testing.T) {
+	exemplars := []tempopb.Exemplar{
+		{
+			TimestampMs: 1638316800000,
+			Value:       1.23,
+			Labels: []v1.KeyValue{
+				{Key: "trace:id", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "trace-123"}}},
+			},
+		},
+		{
+			TimestampMs: 1638316801000,
+			Value:       4.56,
+			Labels: []v1.KeyValue{
+				{Key: "trace:id", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "trace-456"}}},
+			},
+		},
+	}
+	frame := transformExemplarToFrame("test", &tempopb.TimeSeries{
+		Labels:     nil,
+		Samples:    nil,
+		PromLabels: "",
+		Exemplars:  exemplars,
+	})
+	assert.NotNil(t, frame)
+	assert.Equal(t, "test", frame.RefID)
+	assert.Equal(t, "exemplar", frame.Name)
+	assert.Len(t, frame.Fields, 3)
+	assert.Equal(t, time.UnixMilli(1638316800000), frame.Fields[0].At(0))
+	assert.Equal(t, 1.23, frame.Fields[1].At(0))
+	assert.Equal(t, "trace-123", frame.Fields[2].At(0))
+	assert.Equal(t, time.UnixMilli(1638316801000), frame.Fields[0].At(1))
+	assert.Equal(t, 4.56, frame.Fields[1].At(1))
+	assert.Equal(t, "trace-456", frame.Fields[2].At(1))
+}
+
+func TestTransformExemplarToFrame_ExemplarWithoutTraceId(t *testing.T) {
+	exemplars := []tempopb.Exemplar{
+		{
+			TimestampMs: 1638316800000,
+			Value:       1.23,
+			Labels:      []v1.KeyValue{},
+		},
+	}
+	frame := transformExemplarToFrame("test", &tempopb.TimeSeries{
+		Labels:     nil,
+		Samples:    nil,
+		PromLabels: "",
+		Exemplars:  exemplars,
+	})
+	assert.NotNil(t, frame)
+	assert.Equal(t, "test", frame.RefID)
+	assert.Equal(t, "exemplar", frame.Name)
+	assert.Len(t, frame.Fields, 3)
+	assert.Equal(t, time.UnixMilli(1638316800000), frame.Fields[0].At(0))
+	assert.Equal(t, 1.23, frame.Fields[1].At(0))
+	assert.Equal(t, "", frame.Fields[2].At(0))
+}

pkg/tsdb/tempo/traceql/metrics.go

@@ -2,36 +2,24 @@ package traceql
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/tsdb/tempo/kinds/dataquery"
 	"github.com/grafana/tempo/pkg/tempopb"
 	v1 "github.com/grafana/tempo/pkg/tempopb/common/v1"
 )
 
-func TransformMetricsResponse(resp tempopb.QueryRangeResponse) []*data.Frame {
+func TransformMetricsResponse(query *dataquery.TempoQuery, resp tempopb.QueryRangeResponse) []*data.Frame {
 	// prealloc frames
 	frames := make([]*data.Frame, len(resp.Series))
-	for i, series := range resp.Series {
-		labels := make(data.Labels)
-		for _, label := range series.Labels {
-			labels[label.GetKey()] = metricsValueToString(label.GetValue())
-		}
+	var exemplarFrames []*data.Frame
 
-		name := ""
-		if len(series.Labels) > 0 {
-			if len(series.Labels) == 1 {
-				name = metricsValueToString(series.Labels[0].GetValue())
-			} else {
-				var labelStrings []string
-				for key, val := range labels {
-					labelStrings = append(labelStrings, fmt.Sprintf("%s=%s", key, val))
-				}
-				name = fmt.Sprintf("{%s}", strings.Join(labelStrings, ", "))
-			}
-		}
+	for i, series := range resp.Series {
+		name, labels := transformLabelsAndGetName(series.Labels)
 
 		valueField := data.NewField(name, labels, []float64{})
 		valueField.Config = &data.FieldConfig{
@@ -42,7 +30,7 @@ func TransformMetricsResponse(resp tempopb.QueryRangeResponse) []*data.Frame {
 
 		frame := &data.Frame{
 			RefID: name,
-			Name:  "Trace",
+			Name:  name,
 			Fields: []*data.Field{
 				timeField,
 				valueField,
@@ -52,25 +40,65 @@ func TransformMetricsResponse(resp tempopb.QueryRangeResponse) []*data.Frame {
 			},
 		}
 
+		isHistogram := isHistogramQuery(*query.Query)
+		if isHistogram {
+			frame.Meta.PreferredVisualizationPluginID = "heatmap"
+		}
+
 		for _, sample := range series.Samples {
 			frame.AppendRow(time.UnixMilli(sample.GetTimestampMs()), sample.GetValue())
 		}
 
+		if len(series.Exemplars) > 0 {
+			exFrame := transformExemplarToFrame(name, series)
+			exemplarFrames = append(exemplarFrames, exFrame)
+		}
+
 		frames[i] = frame
 	}
-	return frames
+	return append(frames, exemplarFrames...)
 }
 
-func metricsValueToString(value *v1.AnyValue) string {
+func metricsValueToString(value *v1.AnyValue) (string, string) {
 	switch value.GetValue().(type) {
 	case *v1.AnyValue_DoubleValue:
-		return strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64)
+		res := strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64)
+		return res, res
 	case *v1.AnyValue_IntValue:
-		return strconv.FormatInt(value.GetIntValue(), 10)
+		res := strconv.FormatInt(value.GetIntValue(), 10)
+		return res, res
 	case *v1.AnyValue_StringValue:
-		return fmt.Sprintf("\"%s\"", value.GetStringValue())
+		// return the value wrapped in quotes since it's accurate and "1" is different from 1
+		// the second value is returned without quotes for display purposes
+		return fmt.Sprintf("\"%s\"", value.GetStringValue()), value.GetStringValue()
 	case *v1.AnyValue_BoolValue:
-		return strconv.FormatBool(value.GetBoolValue())
+		res := strconv.FormatBool(value.GetBoolValue())
+		return res, res
+	}
+	return "", ""
+}
+
+func transformLabelsAndGetName(seriesLabels []v1.KeyValue) (string, data.Labels) {
+	labels := make(data.Labels)
+	for _, label := range seriesLabels {
+		labels[label.GetKey()], _ = metricsValueToString(label.GetValue())
+	}
+	name := ""
+	if len(seriesLabels) > 0 {
+		if len(seriesLabels) == 1 {
+			_, name = metricsValueToString(seriesLabels[0].GetValue())
+		} else {
+			var labelStrings []string
+			for key, val := range labels {
+				labelStrings = append(labelStrings, fmt.Sprintf("%s=%s", key, val))
+			}
+			name = fmt.Sprintf("{%s}", strings.Join(labelStrings, ", "))
+		}
 	}
-	return ""
+	return name, labels
+}
+
+func isHistogramQuery(query string) bool {
+	match, _ := regexp.MatchString("\\|\\s*(histogram_over_time)\\s*\\(", query)
+	return match
 }

pkg/tsdb/tempo/traceql/metrics_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/tsdb/tempo/kinds/dataquery"
 	"github.com/grafana/tempo/pkg/tempopb"
 	v1 "github.com/grafana/tempo/pkg/tempopb/common/v1"
 	"github.com/stretchr/testify/assert"
@@ -12,7 +13,9 @@ import (
 
 func TestTransformMetricsResponse_EmptyResponse(t *testing.T) {
 	resp := tempopb.QueryRangeResponse{}
-	frames := TransformMetricsResponse(resp)
+	queryStr := ""
+	query := &dataquery.TempoQuery{Query: &queryStr}
+	frames := TransformMetricsResponse(query, resp)
 	assert.Empty(t, frames)
 }
 
@@ -29,13 +32,15 @@ func TestTransformMetricsResponse_SingleSeriesSingleLabel(t *testing.T) {
 			},
 		},
 	}
-	frames := TransformMetricsResponse(resp)
+	queryStr := ""
+	query := &dataquery.TempoQuery{Query: &queryStr}
+	frames := TransformMetricsResponse(query, resp)
 	assert.Len(t, frames, 1)
-	assert.Equal(t, "\"value1\"", frames[0].RefID)
-	assert.Equal(t, "Trace", frames[0].Name)
+	assert.Equal(t, "value1", frames[0].RefID)
+	assert.Equal(t, "value1", frames[0].Name)
 	assert.Len(t, frames[0].Fields, 2)
 	assert.Equal(t, "time", frames[0].Fields[0].Name)
-	assert.Equal(t, "\"value1\"", frames[0].Fields[1].Name)
+	assert.Equal(t, "value1", frames[0].Fields[1].Name)
 	assert.Equal(t, data.VisTypeGraph, frames[0].Meta.PreferredVisualization)
 	assert.Equal(t, time.UnixMilli(1638316800000), frames[0].Fields[0].At(0))
 	assert.Equal(t, 1.23, frames[0].Fields[1].At(0))
@@ -60,10 +65,12 @@ func TestTransformMetricsResponse_SingleSeriesMultipleLabels(t *testing.T) {
 			},
 		},
 	}
-	frames := TransformMetricsResponse(resp)
+	queryStr := ""
+	query := &dataquery.TempoQuery{Query: &queryStr}
+	frames := TransformMetricsResponse(query, resp)
 	assert.Len(t, frames, 1)
 	assert.Equal(t, "{label1=\"value1\", label2=123, label3=123.456, label4=true}", frames[0].RefID)
-	assert.Equal(t, "Trace", frames[0].Name)
+	assert.Equal(t, "{label1=\"value1\", label2=123, label3=123.456, label4=true}", frames[0].Name)
 	assert.Len(t, frames[0].Fields, 2)
 	assert.Equal(t, "time", frames[0].Fields[0].Name)
 	assert.Equal(t, "{label1=\"value1\", label2=123, label3=123.456, label4=true}", frames[0].Fields[1].Name)
@@ -93,19 +100,21 @@ func TestTransformMetricsResponse_MultipleSeries(t *testing.T) {
 			},
 		},
 	}
-	frames := TransformMetricsResponse(resp)
+	queryStr := ""
+	query := &dataquery.TempoQuery{Query: &queryStr}
+	frames := TransformMetricsResponse(query, resp)
 	assert.Len(t, frames, 2)
-	assert.Equal(t, "\"value1\"", frames[0].RefID)
-	assert.Equal(t, "Trace", frames[0].Name)
+	assert.Equal(t, "value1", frames[0].RefID)
+	assert.Equal(t, "value1", frames[0].Name)
 	assert.Len(t, frames[0].Fields, 2)
 	assert.Equal(t, "time", frames[0].Fields[0].Name)
-	assert.Equal(t, "\"value1\"", frames[0].Fields[1].Name)
+	assert.Equal(t, "value1", frames[0].Fields[1].Name)
 	assert.Equal(t, data.VisTypeGraph, frames[0].Meta.PreferredVisualization)
 	assert.Equal(t, time.UnixMilli(1638316800000), frames[0].Fields[0].At(0))
 	assert.Equal(t, 1.23, frames[0].Fields[1].At(0))
 
 	assert.Equal(t, "456", frames[1].RefID)
-	assert.Equal(t, "Trace", frames[1].Name)
+	assert.Equal(t, "456", frames[1].Name)
 	assert.Len(t, frames[1].Fields, 2)
 	assert.Equal(t, "time", frames[1].Fields[0].Name)
 	assert.Equal(t, "456", frames[1].Fields[1].Name)

pkg/tsdb/tempo/traceql_query.go

@@ -96,7 +96,7 @@ func (s *Service) runTraceQlQueryMetrics(ctx context.Context, pCtx backend.Plugi
 		return &backend.DataResponse{}, fmt.Errorf("failed to convert response to type: %w", err)
 	}
 
-	frames := traceql.TransformMetricsResponse(queryResponse)
+	frames := traceql.TransformMetricsResponse(tempoQuery, queryResponse)
 
 	result.Frames = frames
 	ctxLogger.Debug("Successfully performed TraceQL query", "function", logEntrypoint())
@@ -151,6 +151,9 @@ func (s *Service) createMetricsQuery(ctx context.Context, dsInfo *Datasource, qu
 	if query.Step != nil {
 		q.Set("step", *query.Step)
 	}
+	if query.Exemplars != nil {
+		q.Set("exemplars", strconv.FormatInt(*query.Exemplars, 10))
+	}
 
 	searchUrl.RawQuery = q.Encode()
 

pkg/tsdb/tempo/traceql_query_test.go

@@ -19,9 +19,11 @@ func TestCreateMetricsQuery_Success(t *testing.T) {
 	}
 	queryVal := "{attribute=\"value\"}"
 	stepVal := "14"
+	exemplarVal := int64(123)
 	query := &dataquery.TempoQuery{
 		Query:     &queryVal,
 		Step:      &stepVal,
+		Exemplars: &exemplarVal,
 	}
 	start := int64(1625097600)
 	end := int64(1625184000)
@@ -29,7 +31,7 @@ func TestCreateMetricsQuery_Success(t *testing.T) {
 	req, err := service.createMetricsQuery(context.Background(), dsInfo, query, start, end)
 	assert.NoError(t, err)
 	assert.NotNil(t, req)
-	assert.Equal(t, "http://tempo:3100/api/metrics/query_range?end=1625184000&q=%7Battribute%3D%22value%22%7D&start=1625097600&step=14", req.URL.String())
+	assert.Equal(t, "http://tempo:3100/api/metrics/query_range?end=1625184000&exemplars=123&q=%7Battribute%3D%22value%22%7D&start=1625097600&step=14", req.URL.String())
 	assert.Equal(t, "application/json", req.Header.Get("Accept"))
 }
 func TestCreateMetricsQuery_OnlyQuery(t *testing.T) {

public/app/plugins/datasource/tempo/dataquery.cue

@@ -53,6 +53,8 @@ composableKinds: DataQuery: {
 					tableType?: #SearchTableType
 					// For metric queries, the step size to use
 					step?: string
+					// For metric queries, how many exemplars to request, 0 means no exemplars
+					exemplars?: int64
 				} @cuetsy(kind="interface") @grafana(TSVeneer="type")
 
 				#TempoQueryType: "traceql" | "traceqlSearch" | "serviceMap" | "upload" | "nativeSearch" | "traceId" | "clear" @cuetsy(kind="type")

public/app/plugins/datasource/tempo/dataquery.gen.ts

@@ -11,6 +11,10 @@
 import * as common from '@grafana/schema';
 
 export interface TempoQuery extends common.DataQuery {
+  /**
+   * For metric queries, how many exemplars to request, 0 means no exemplars
+   */
+  exemplars?: number;
   filters: Array<TraceqlFilter>;
   /**
    * Filters that are used to query the metrics summary

public/app/plugins/datasource/tempo/datasource.ts

@@ -51,7 +51,12 @@ import {
 } from './graphTransform';
 import TempoLanguageProvider from './language_provider';
 import { createTableFrameFromMetricsSummaryQuery, emptyResponse, MetricsSummary } from './metricsSummary';
-import { formatTraceQLResponse, transformFromOTLP as transformFromOTEL, transformTrace } from './resultTransformer';
+import {
+  enhanceTraceQlMetricsResponse,
+  formatTraceQLResponse,
+  transformFromOTLP as transformFromOTEL,
+  transformTrace,
+} from './resultTransformer';
 import { doTempoChannelStream } from './streaming';
 import { TempoJsonData, TempoQuery } from './types';
 import { getErrorMessage, migrateFromSearchToTraceQLSearch } from './utils';
@@ -604,7 +609,7 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
     const request = { ...options, targets: validTargets };
     return super.query(request).pipe(
       map((response) => {
-        return response;
+        return enhanceTraceQlMetricsResponse(response, this.instanceSettings);
       }),
       catchError((err) => {
         return of({ error: { message: getErrorMessage(err.data.message) }, data: [] });

public/app/plugins/datasource/tempo/resultTransformer.ts

@@ -464,6 +464,42 @@ function transformToTraceData(data: TraceSearchMetadata) {
   };
 }
 
+export function enhanceTraceQlMetricsResponse(
+  data: DataQueryResponse,
+  instanceSettings: DataSourceInstanceSettings
+): DataQueryResponse {
+  data.data
+    ?.filter((f) => f.name === 'exemplar' && f.meta?.dataTopic === 'annotations')
+    .map((frame) => {
+      const traceIDField = frame.fields.find((field: Field) => field.name === 'traceId');
+      if (traceIDField) {
+        const links = getDataLinks(instanceSettings);
+        traceIDField.config.links = traceIDField.config.links?.length
+          ? [...traceIDField.config.links, ...links]
+          : links;
+      }
+      return frame;
+    });
+  return data;
+}
+
+function getDataLinks(instanceSettings: DataSourceInstanceSettings): DataLink[] {
+  const dataLinks: DataLink[] = [];
+
+  if (instanceSettings.uid) {
+    dataLinks.push({
+      title: 'View trace',
+      url: '',
+      internal: {
+        query: { query: '${__value.raw}', queryType: 'traceql' },
+        datasourceUid: instanceSettings.uid,
+        datasourceName: instanceSettings?.name ?? 'Data source not found',
+      },
+    });
+  }
+  return dataLinks;
+}
+
 export function formatTraceQLResponse(
   data: TraceSearchMetadata[],
   instanceSettings: DataSourceInstanceSettings,

public/app/plugins/datasource/tempo/traceql/TempoQueryBuilderOptions.tsx

@@ -49,11 +49,25 @@ export const TempoQueryBuilderOptions = React.memo<Props>(({ onChange, query, is
     onChange({ ...query, step: e.currentTarget.value });
   };
 
+  // There's a bug in Tempo which causes the exemplars param to be ignored. It's commented out for now.
+
+  // const onExemplarsChange = (e: React.FormEvent<HTMLInputElement>) => {
+  //   const exemplars = parseInt(e.currentTarget.value, 10);
+  //   if (!isNaN(exemplars) && exemplars >= 0) {
+  //     onChange({ ...query, exemplars });
+  //   } else {
+  //     onChange({ ...query, exemplars: undefined });
+  //   }
+  // };
+
   const collapsedInfoList = [
     `Limit: ${query.limit || DEFAULT_LIMIT}`,
     `Spans Limit: ${query.spss || DEFAULT_SPSS}`,
     `Table Format: ${query.tableType === SearchTableType.Traces ? 'Traces' : 'Spans'}`,
+    '|',
     `Step: ${query.step || 'auto'}`,
+    // `Exemplars: ${query.exemplars !== undefined ? query.exemplars : 'auto'}`,
+    '|',
     `Streaming: ${isStreaming ? 'Enabled' : 'Disabled'}`,
   ];
 
@@ -106,6 +120,19 @@ export const TempoQueryBuilderOptions = React.memo<Props>(({ onChange, query, is
               value={query.step}
             />
           </EditorField>
+          {/*<EditorField*/}
+          {/*  label="Exemplars"*/}
+          {/*  tooltip="Defines the amount of exemplars to request for metric queries. A value of 0 means no exemplars."*/}
+          {/*>*/}
+          {/*  <AutoSizeInput*/}
+          {/*    className="width-4"*/}
+          {/*    placeholder="auto"*/}
+          {/*    type="string"*/}
+          {/*    defaultValue={query.exemplars}*/}
+          {/*    onCommitChange={onExemplarsChange}*/}
+          {/*    value={query.exemplars}*/}
+          {/*  />*/}
+          {/*</EditorField>*/}
           <EditorField label="Streaming" tooltip={<StreamingTooltip />} tooltipInteractive>
             <div>{isStreaming ? 'Enabled' : 'Disabled'}</div>
           </EditorField>