Explore: Add ability to include tags in trace to metrics queries (#49433)

* Add tags input

* Tracing: add ability to include tags in trace to metrics queries

docs/sources/datasources/jaeger.md

@@ -47,11 +47,12 @@ This is a configuration for the [trace to logs feature]({{< relref "../explore/t
 To configure trace to metrics, select the target Prometheus data source and create any desired linked queries.
 
 -- **Data source -** Target data source.
+-- **Tags -** You can use tags in the linked queries. The key is the span attribute name. The optional value is the corresponding metric label name (for example, map `k8s.pod` to `pod`). You may interpolate these tags into your queries using the `$__tags` keyword.
 
 Each linked query consists of:
 
 -- **Link Label -** (Optional) Descriptive label for the linked query.
--- **Query -** Query that runs when navigating from a trace to the metrics data source.
+-- **Query -** Query that runs when navigating from a trace to the metrics data source. Interpolate tags using the `$__tags` keyword. For example, when you configure the query `requests_total{$__tags}`with the tags `k8s.pod=pod` and `cluster`, it results in `requests_total{pod="nginx-554b9", cluster="us-east-1"}`.
 
 ### Node Graph
 
@@ -169,6 +170,12 @@ datasources:
         spanEndTimeShift: '1h'
         filterByTraceID: false
         filterBySpanID: false
+      tracesToMetrics:
+        datasourceUid: 'prom'
+        tags: [{ key: 'service.name', value: 'service' }, { key: 'job' }]
+        queries:
+          - name: 'Sample query'
+            query: 'sum(rate(tempo_spanmetrics_latency_bucket{$__tags}[5m]))'
     secureJsonData:
       basicAuthPassword: my_password
 ```

docs/sources/datasources/tempo.md

@@ -46,11 +46,12 @@ This is a configuration for the [trace to logs feature]({{< relref "../explore/t
 To configure trace to metrics, select the target Prometheus data source and create any desired linked queries.
 
 -- **Data source -** Target data source.
+-- **Tags -** You can use tags in the linked queries. The key is the span attribute name. The optional value is the corresponding metric label name (for example, map `k8s.pod` to `pod`). You may interpolate these tags into your queries using the `$__tags` keyword.
 
 Each linked query consists of:
 
 -- **Link Label -** (Optional) Descriptive label for the linked query.
--- **Query -** Query that runs when navigating from a trace to the metrics data source.
+-- **Query -** Query that runs when navigating from a trace to the metrics data source. Interpolate tags using the `$__tags` keyword. For example, when you configure the query `requests_total{$__tags}`with the tags `k8s.pod=pod` and `cluster`, it results in `requests_total{pod="nginx-554b9", cluster="us-east-1"}`.
 
 ### Service Graph
 
@@ -216,6 +217,12 @@ datasources:
         spanEndTimeShift: '1h'
         filterByTraceID: false
         filterBySpanID: false
+      tracesToMetrics:
+        datasourceUid: 'prom'
+        tags: [{ key: 'service.name', value: 'service' }, { key: 'job' }]
+        queries:
+          - name: 'Sample query'
+            query: 'sum(rate(tempo_spanmetrics_latency_bucket{$__tags}[5m]))'
       serviceMap:
         datasourceUid: 'prometheus'
       search:

docs/sources/datasources/zipkin.md

@@ -47,11 +47,12 @@ This is a configuration for the [trace to logs feature]({{< relref "../explore/t
 To configure trace to metrics, select the target Prometheus data source and create any desired linked queries.
 
 -- **Data source -** Target data source.
+-- **Tags -** You can use tags in the linked queries. The key is the span attribute name. The optional value is the corresponding metric label name (for example, map `k8s.pod` to `pod`). You may interpolate these tags into your queries using the `$__tags` keyword.
 
 Each linked query consists of:
 
 -- **Link Label -** (Optional) Descriptive label for the linked query.
--- **Query -** Query that runs when navigating from a trace to the metrics data source.
+-- **Query -** Query that runs when navigating from a trace to the metrics data source. Interpolate tags using the `$__tags` keyword. For example, when you configure the query `requests_total{$__tags}`with the tags `k8s.pod=pod` and `cluster`, it results in `requests_total{pod="nginx-554b9", cluster="us-east-1"}`.
 
 ### Node Graph
 

public/app/core/components/TraceToLogs/KeyValueInput.tsx

@@ -65,6 +65,7 @@ const KeyValueInput = ({
               onClick={() => onChange([...values.slice(0, idx), ...values.slice(idx + 1)])}
               className="gf-form-label query-part"
               aria-label="Remove tag"
+              type="button"
             >
               <Icon name="times" />
             </button>
@@ -73,6 +74,7 @@ const KeyValueInput = ({
                 onClick={() => onChange([...values, { key: '', value: '' }])}
                 className="gf-form-label query-part"
                 aria-label="Add tag"
+                type="button"
               >
                 <Icon name="plus" />
               </button>
@@ -84,6 +86,7 @@ const KeyValueInput = ({
           onClick={() => onChange([...values, { key: '', value: '' }])}
           className="gf-form-label query-part"
           aria-label="Add tag"
+          type="button"
         >
           <Icon name="plus" />
         </button>

public/app/core/components/TraceToMetrics/TraceToMetricsSettings.tsx

@@ -5,19 +5,23 @@ import {
   DataSourceJsonData,
   DataSourcePluginOptionsEditorProps,
   GrafanaTheme,
+  KeyValue,
   updateDatasourcePluginJsonDataOption,
 } from '@grafana/data';
 import { DataSourcePicker } from '@grafana/runtime';
 import { Button, InlineField, InlineFieldRow, Input, useStyles } from '@grafana/ui';
 
+import KeyValueInput from '../TraceToLogs/KeyValueInput';
+
 export interface TraceToMetricsOptions {
   datasourceUid?: string;
+  tags?: Array<KeyValue<string>>;
   queries: TraceToMetricQuery[];
 }
 
 export interface TraceToMetricQuery {
   name?: string;
-  query: string;
+  query?: string;
 }
 
 export interface TraceToMetricsData extends DataSourceJsonData {
@@ -71,6 +75,21 @@ export function TraceToMetricsSettings({ options, onOptionsChange }: Props) {
         ) : null}
       </InlineFieldRow>
 
+      <InlineFieldRow>
+        <InlineField tooltip="Tags that will be used in the metrics query." label="Tags" labelWidth={26}>
+          <KeyValueInput
+            keyPlaceholder="Tag"
+            values={options.jsonData.tracesToMetrics?.tags ?? []}
+            onChange={(v) =>
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToMetrics', {
+                ...options.jsonData.tracesToMetrics,
+                tags: v,
+              })
+            }
+          />
+        </InlineField>
+      </InlineFieldRow>
+
       {options.jsonData.tracesToMetrics?.queries?.map((query, i) => (
         <div key={i} className={styles.queryRow}>
           <InlineField label="Link Label" labelWidth={10}>
@@ -92,7 +111,7 @@ export function TraceToMetricsSettings({ options, onOptionsChange }: Props) {
           <InlineField
             label="Query"
             labelWidth={10}
-            tooltip="The Prometheus query that will run when navigating from a trace to metrics"
+            tooltip="The Prometheus query that will run when navigating from a trace to metrics. Interpolate tags using the `$__tags` keyword."
             grow
           >
             <Input

public/app/features/explore/TraceView/createSpanLink.test.ts

@@ -478,6 +478,40 @@ describe('createSpanLinkFactory', () => {
     });
   });
 
+  it('correctly interpolates span attributes', () => {
+    const splitOpenFn = jest.fn();
+    const createLink = createSpanLinkFactory({
+      splitOpenFn,
+      traceToMetricsOptions: {
+        datasourceUid: 'prom1',
+        queries: [{ name: 'Named Query', query: 'metric{$__tags}[5m]' }],
+        tags: [
+          { key: 'job', value: '' },
+          { key: 'k8s.pod', value: 'pod' },
+        ],
+      } as TraceToMetricsOptions,
+    });
+    expect(createLink).toBeDefined();
+
+    const links = createLink!(
+      createTraceSpan({
+        process: {
+          serviceName: 'service',
+          tags: [
+            { key: 'job', value: 'tns/app' },
+            { key: 'k8s.pod', value: 'sample-pod' },
+          ],
+        },
+      })
+    );
+    expect(links).toBeDefined();
+    expect(links!.metricLinks![0]!.href).toBe(
+      `/explore?left=${encodeURIComponent(
+        '{"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:01.000Z"},"datasource":"prom1","queries":[{"expr":"metric{job=\\"tns/app\\", pod=\\"sample-pod\\"}[5m]","refId":"A"}],"panelsState":{}}'
+      )}`
+    );
+  });
+
   describe('should return span links', () => {
     beforeAll(() => {
       setDataSourceSrv(new DatasourceSrv());

public/app/features/explore/TraceView/createSpanLink.tsx

@@ -20,7 +20,7 @@ import { getTemplateSrv } from '@grafana/runtime';
 import { Icon } from '@grafana/ui';
 import { SpanLinkFunc, TraceSpan } from '@jaegertracing/jaeger-ui-components';
 import { TraceToLogsOptions } from 'app/core/components/TraceToLogs/TraceToLogsSettings';
-import { TraceToMetricsOptions } from 'app/core/components/TraceToMetrics/TraceToMetricsSettings';
+import { TraceToMetricQuery, TraceToMetricsOptions } from 'app/core/components/TraceToMetrics/TraceToMetricsSettings';
 import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
 import { PromQuery } from 'app/plugins/datasource/prometheus/types';
 
@@ -150,10 +150,9 @@ function legacyCreateSpanLinkFactory(
 
     // Get metrics links
     if (metricsDataSourceSettings && traceToMetricsOptions?.queries) {
-      const defaultQuery = `histogram_quantile(0.5, sum(rate(tempo_spanmetrics_latency_bucket{operation="${span.operationName}"}[5m])) by (le))`;
-
       links.metricLinks = [];
       for (const query of traceToMetricsOptions.queries) {
+        const expr = buildMetricsQuery(query, traceToMetricsOptions?.tags, span);
         const dataLink: DataLink<PromQuery> = {
           title: metricsDataSourceSettings.name,
           url: '',
@@ -161,7 +160,7 @@ function legacyCreateSpanLinkFactory(
             datasourceUid: metricsDataSourceSettings.uid,
             datasourceName: metricsDataSourceSettings.name,
             query: {
-              expr: query.query || defaultQuery,
+              expr,
               refId: 'A',
             },
           },
@@ -362,3 +361,27 @@ function getTimeRangeFromSpan(
     },
   };
 }
+
+// Interpolates span attributes into trace to metric query, or returns default query
+function buildMetricsQuery(query: TraceToMetricQuery, tags: Array<KeyValue<string>> = [], span: TraceSpan): string {
+  if (!query.query) {
+    return `histogram_quantile(0.5, sum(rate(tempo_spanmetrics_latency_bucket{operation="${span.operationName}"}[5m])) by (le))`;
+  }
+
+  let expr = query.query;
+  if (tags.length && expr.indexOf('$__tags') !== -1) {
+    const spanTags = [...span.process.tags, ...span.tags];
+    const labels = tags.reduce((acc, tag) => {
+      const tagValue = spanTags.find((t) => t.key === tag.key)?.value;
+      if (tagValue) {
+        acc.push(`${tag.value ? tag.value : tag.key}="${tagValue}"`);
+      }
+      return acc;
+    }, [] as string[]);
+
+    const labelsQuery = labels?.join(', ');
+    expr = expr.replace('$__tags', labelsQuery);
+  }
+
+  return expr;
+}