Tempo: Integrate TraceQL API (#56867)

devenv/docker/blocks/tempo/docker-compose.yaml

@@ -68,7 +68,7 @@ services:
 
   tempo:
     image: grafana/tempo:latest
-    command: 
+    command:
       - --config.file=/etc/tempo.yaml
       - --search.enabled=true
     volumes:

devenv/docker/blocks/tempo/tempo.yaml

@@ -41,6 +41,10 @@ metrics_generator:
       - url: http://prometheus:9090/api/v1/write
         send_exemplars: true
 
+query_frontend:
+  search:
+    max_duration: 0 # allow searches >1h
+
 storage:
   trace:
     backend: local                     # backend configuration to use
@@ -48,6 +52,7 @@ storage:
       bloom_filter_false_positive: .05 # bloom filter false positive rate.  lower values create larger filters but fewer false positives
       index_downsample_bytes: 1000     # number of bytes per index record
       encoding: zstd                   # block encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2
+      version: vParquet
     wal:
       path: /tmp/tempo/wal             # where to store the the wal locally
       encoding: snappy                 # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2

public/app/plugins/datasource/tempo/datasource.ts

@@ -51,7 +51,6 @@ import {
   createTableFrameFromSearch,
   createTableFrameFromTraceQlQuery,
 } from './resultTransformer';
-import { mockedSearchResponse } from './traceql/mockedSearchResponse';
 import { SearchQueryParams, TempoQuery, TempoJsonData } from './types';
 
 export const DEFAULT_LIMIT = 20;
@@ -174,9 +173,21 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
         });
 
         subQueries.push(
-          of({
-            data: [createTableFrameFromTraceQlQuery(mockedSearchResponse().traces, this.instanceSettings)],
-          })
+          this._request('/api/search', {
+            q: targets.traceql[0].query,
+            limit: options.targets[0].limit,
+            start: 0, // Currently the API doesn't return traces when using the 'From' time selected in Explore
+            end: options.range.to.unix(),
+          }).pipe(
+            map((response) => {
+              return {
+                data: [createTableFrameFromTraceQlQuery(response.data.traces, this.instanceSettings)],
+              };
+            }),
+            catchError((error) => {
+              return of({ error: { message: error.data.message }, data: [] });
+            })
+          )
         );
       } catch (error) {
         return of({ error: { message: error instanceof Error ? error.message : 'Unknown error occurred' }, data: [] });

public/app/plugins/datasource/tempo/resultTransformer.test.ts

@@ -1,14 +1,28 @@
 import { collectorTypes } from '@opentelemetry/exporter-collector';
 
-import { FieldType, MutableDataFrame, PluginType, DataSourceInstanceSettings, dateTime } from '@grafana/data';
+import {
+  FieldType,
+  MutableDataFrame,
+  PluginType,
+  DataSourceInstanceSettings,
+  dateTime,
+  ArrayVector,
+} from '@grafana/data';
 
-import { createTableFrame, transformToOTLP, transformFromOTLP, createTableFrameFromSearch } from './resultTransformer';
+import {
+  createTableFrame,
+  transformToOTLP,
+  transformFromOTLP,
+  createTableFrameFromSearch,
+  createTableFrameFromTraceQlQuery,
+} from './resultTransformer';
 import {
   badOTLPResponse,
   otlpDataFrameToResponse,
   otlpDataFrameFromResponse,
   otlpResponse,
   tempoSearchResponse,
+  traceQlResponse,
 } from './testResponse';
 import { TraceSearchMetadata } from './types';
 
@@ -111,6 +125,45 @@ describe('createTableFrameFromSearch()', () => {
   });
 });
 
+describe('createTableFrameFromTraceQlQuery()', () => {
+  test('transforms TraceQL response to DataFrame', () => {
+    const frame = createTableFrameFromTraceQlQuery(traceQlResponse.traces, defaultSettings);
+    // Trace ID field
+    expect(frame.fields[0].name).toBe('traceID');
+    expect(frame.fields[0].values.get(0)).toBe('b1586c3c8c34d');
+    expect(frame.fields[0].config.unit).toBe('string');
+    expect(frame.fields[0].values).toBeInstanceOf(ArrayVector);
+    // There should be a traceIdHidden field which is hidden
+    expect(frame.fields[1].name).toBe('traceIdHidden');
+    expect(frame.fields[1].config).toBeDefined();
+    expect(frame.fields[1].config.custom).toStrictEqual({ hidden: true });
+    expect(frame.fields[1].values).toBeInstanceOf(ArrayVector);
+    // Span ID field
+    expect(frame.fields[2].name).toBe('spanID');
+    expect(frame.fields[2].config.unit).toBe('string');
+    expect(frame.fields[2].values).toBeInstanceOf(ArrayVector);
+    // Trace name field
+    expect(frame.fields[3].name).toBe('traceName');
+    expect(frame.fields[3].type).toBe('string');
+    expect(frame.fields[3].values.get(0)).toBe('lb HTTP Client');
+    expect(frame.fields[3].values).toBeInstanceOf(ArrayVector);
+    // Start time field
+    expect(frame.fields[4].name).toBe('startTime');
+    expect(frame.fields[4].type).toBe('string');
+    expect(frame.fields[4].values.get(1)).toBe('2022-10-19 09:03:34');
+    expect(frame.fields[4].values).toBeInstanceOf(ArrayVector);
+    // Duration field
+    expect(frame.fields[5].name).toBe('duration');
+    expect(frame.fields[5].type).toBe('number');
+    expect(frame.fields[5].values.get(2)).toBe(6686000);
+    // There should be a field for each attribute
+    expect(frame.fields[6].name).toBe('http.method');
+    expect(frame.fields[6].values).toBeInstanceOf(ArrayVector);
+    expect(frame.fields[7].name).toBe('service.name');
+    expect(frame.fields[7].values).toBeInstanceOf(ArrayVector);
+  });
+});
+
 describe('transformFromOTLP()', () => {
   // Mock the console error so that running the test suite doesnt throw the error
   const origError = console.error;

public/app/plugins/datasource/tempo/resultTransformer.ts

@@ -577,7 +577,7 @@ export function createTableFrameFromSearch(data: TraceSearchMetadata[], instance
       },
       { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Trace name' } },
       { name: 'startTime', type: FieldType.string, config: { displayNameFromDS: 'Start time' } },
-      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ms' } },
+      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ns' } },
     ],
     meta: {
       preferredVisualisationType: 'table',
@@ -609,7 +609,7 @@ function transformToTraceData(data: TraceSearchMetadata) {
 
   const traceStartTime = parseInt(data.startTimeUnixNano!, 10) / 1000000;
 
-  let startTime = dateTimeFormat(traceStartTime);
+  let startTime = !isNaN(traceStartTime) ? dateTimeFormat(traceStartTime) : '';
 
   if (Math.abs(differenceInHours(new Date(traceStartTime), Date.now())) <= 1) {
     startTime = formatDistance(new Date(traceStartTime), Date.now(), {
@@ -621,7 +621,7 @@ function transformToTraceData(data: TraceSearchMetadata) {
   return {
     traceID: data.traceID,
     startTime: startTime,
-    duration: data.durationMs,
+    duration: data.durationMs?.toString(),
     traceName,
   };
 }
@@ -654,12 +654,18 @@ export function createTableFrameFromTraceQlQuery(
           ],
         },
       },
+      {
+        name: 'traceIdHidden',
+        config: {
+          custom: { hidden: true },
+        },
+      },
       {
         name: 'spanID',
         type: FieldType.string,
         config: {
           unit: 'string',
-          displayNameFromDS: 'Span',
+          displayNameFromDS: 'Span ID',
           links: [
             {
               title: 'Span: ${__value.raw}',
@@ -668,18 +674,18 @@ export function createTableFrameFromTraceQlQuery(
                 datasourceUid: instanceSettings.uid,
                 datasourceName: instanceSettings.name,
                 query: {
-                  query: '${__value.raw}',
-                  queryType: 'spanId',
+                  query: '${__data.fields.traceIdHidden}',
+                  queryType: 'traceId',
                 },
               },
             },
           ],
         },
       },
-      { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Name' } },
-      { name: 'attributes', type: FieldType.string, config: { displayNameFromDS: 'Attributes' } },
+      { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Trace name' } },
+      // { name: 'attributes', type: FieldType.string, config: { displayNameFromDS: 'Attributes' } },
       { name: 'startTime', type: FieldType.string, config: { displayNameFromDS: 'Start time' } },
-      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ms' } },
+      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ns' } },
     ],
     meta: {
       preferredVisualisationType: 'table',
@@ -688,57 +694,71 @@ export function createTableFrameFromTraceQlQuery(
   if (!data?.length) {
     return frame;
   }
-  // Show the most recent traces
-  const traceData = data
+
+  const attributesAdded: string[] = [];
+
+  data.forEach((trace) => {
+    trace.spanSet?.spans.forEach((span) => {
+      span.attributes?.forEach((attr) => {
+        if (!attributesAdded.includes(attr.key)) {
+          frame.addField({ name: attr.key, type: FieldType.string, config: { displayNameFromDS: attr.key } });
+          attributesAdded.push(attr.key);
+        }
+      });
+    });
+  });
+
+  const tableRows = data
+    // Show the most recent traces
     .sort((a, b) => parseInt(b?.startTimeUnixNano!, 10) / 1000000 - parseInt(a?.startTimeUnixNano!, 10) / 1000000)
-    .reduce((list: TraceTableData[], t) => {
-      const firstSpanSet = t.spanSets?.[0];
-      const trace: TraceTableData = transformToTraceData(t);
-      trace.attributes = firstSpanSet?.attributes
-        .map((atr) => Object.keys(atr).map((key) => `${key} = ${atr[key]}`))
-        .join(', ');
-      list.push(trace);
-      firstSpanSet?.spans.forEach((span) => list.push(transformSpanToTraceData(span)));
-      return list;
+    .reduce((rows: TraceTableData[], trace) => {
+      const traceData: TraceTableData = transformToTraceData(trace);
+      rows.push(traceData);
+      trace.spanSet?.spans.forEach((span) => {
+        rows.push(transformSpanToTraceData(span, trace.traceID));
+      });
+      return rows;
     }, []);
 
-  for (const trace of traceData) {
-    frame.add(trace);
+  for (const row of tableRows) {
+    frame.add(row);
   }
 
   return frame;
 }
 
 interface TraceTableData {
+  [key: string]: string | number | undefined; // dynamic attribute name
   traceID?: string;
-  traceName: string;
   spanID?: string;
-  attributes?: string;
-  startTime: string;
-  duration: number;
+  //attributes?: string;
+  startTime?: string;
+  duration?: string;
 }
 
-function transformSpanToTraceData(data: Span): TraceTableData {
-  const traceStartTime = data.startTimeUnixNano / 1000000;
-  const traceEndTime = data.endTimeUnixNano / 1000000;
-
-  let startTime = dateTimeFormat(traceStartTime);
+function transformSpanToTraceData(span: Span, traceID: string): TraceTableData {
+  const spanStartTimeUnixMs = parseInt(span.startTimeUnixNano, 10) / 1000000;
+  let spanStartTime = dateTimeFormat(spanStartTimeUnixMs);
 
-  if (Math.abs(differenceInHours(new Date(traceStartTime), Date.now())) <= 1) {
-    startTime = formatDistance(new Date(traceStartTime), Date.now(), {
+  if (Math.abs(differenceInHours(new Date(spanStartTime), Date.now())) <= 1) {
+    spanStartTime = formatDistance(new Date(spanStartTime), Date.now(), {
       addSuffix: true,
       includeSeconds: true,
     });
   }
 
-  return {
-    traceID: undefined,
-    traceName: data.name,
-    spanID: data.spanId,
-    attributes: data.attributes?.map((atr) => Object.keys(atr).map((key) => `${key} = ${atr[key]}`)).join(', '),
-    startTime,
-    duration: traceEndTime - traceStartTime,
+  const data: TraceTableData = {
+    traceIdHidden: traceID,
+    spanID: span.spanID,
+    startTime: spanStartTime,
+    duration: span.durationNanos,
   };
+
+  span.attributes?.forEach((attr) => {
+    data[attr.key] = attr.value.stringValue;
+  });
+
+  return data;
 }
 
 const emptyDataQueryResponse = {

public/app/plugins/datasource/tempo/testResponse.ts

@@ -2204,6 +2204,273 @@ export const tempoSearchResponse = {
   },
 };
 
+export const traceQlResponse = {
+  traces: [
+    {
+      traceID: 'b1586c3c8c34d',
+      rootServiceName: 'lb',
+      rootTraceName: 'HTTP Client',
+      spanSet: {
+        spans: [
+          {
+            spanID: '162a4adae63b61f1',
+            startTimeUnixNano: '1666188214303201000',
+            durationNanos: '545000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'db',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '15991be3a92136e6',
+            startTimeUnixNano: '1666188214300239000',
+            durationNanos: '6686000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '5e91b69dc224c240',
+            startTimeUnixNano: '1666188214300647000',
+            durationNanos: '6043000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '29f218a50b00c306',
+            startTimeUnixNano: '1666188214297891000',
+            durationNanos: '8365000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'lb',
+                },
+              },
+            ],
+          },
+        ],
+        matched: 4,
+      },
+    },
+    {
+      traceID: '9161e77388f3e',
+      rootServiceName: 'lb',
+      rootTraceName: 'HTTP Client',
+      spanSet: {
+        spans: [
+          {
+            spanID: '3b9a5c222d3ddd8f',
+            startTimeUnixNano: '1666187875397721000',
+            durationNanos: '877000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'db',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '894d90db6b5807f',
+            startTimeUnixNano: '1666187875393293000',
+            durationNanos: '11073000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: 'd3284e9c5081aab',
+            startTimeUnixNano: '1666187875393897000',
+            durationNanos: '10133000',
+            attributes: [
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '454785498fc8b1aa',
+            startTimeUnixNano: '1666187875389957000',
+            durationNanos: '13953000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'lb',
+                },
+              },
+            ],
+          },
+        ],
+        matched: 4,
+      },
+    },
+    {
+      traceID: '480691f7c6f20',
+      rootServiceName: 'lb',
+      rootTraceName: 'HTTP Client',
+      spanSet: {
+        spans: [
+          {
+            spanID: '2ab970c9db57d100',
+            startTimeUnixNano: '1666186467658853000',
+            durationNanos: '436000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'db',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '3a4070e418857cbd',
+            startTimeUnixNano: '1666186467657066000',
+            durationNanos: '5503000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '7ddf87d7a3f864c8',
+            startTimeUnixNano: '1666186467657336000',
+            durationNanos: '5005000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '241e9f31609056c5',
+            startTimeUnixNano: '1666186467655299000',
+            durationNanos: '6413000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'lb',
+                },
+              },
+            ],
+          },
+        ],
+        matched: 4,
+      },
+    },
+  ],
+  metrics: {
+    inspectedBlocks: 5,
+    totalBlockBytes: '9092129',
+  },
+};
+
 export const badOTLPResponse = {
   batches: [
     {

public/app/plugins/datasource/tempo/traceql/TempoQueryBuilderOptions.tsx

@@ -12,6 +12,10 @@ interface Props {
 }
 
 export const TempoQueryBuilderOptions = React.memo<Props>(({ onChange, query }) => {
+  if (!query.hasOwnProperty('limit')) {
+    query.limit = DEFAULT_LIMIT;
+  }
+
   const onLimitChange = (e: React.FormEvent<HTMLInputElement>) => {
     onChange({ ...query, limit: parseInt(e.currentTarget.value, 10) });
   };

public/app/plugins/datasource/tempo/traceql/mockedSearchResponse.ts

@@ -1,62 +0,0 @@
-import { v4 as uuidv4 } from 'uuid';
-
-import { SearchResponse, Span, SpanKind, TraceSearchMetadata } from '../types';
-
-export const mockedSearchResponse = (): SearchResponse => {
-  const traces: TraceSearchMetadata[] = [];
-  const attributes = [
-    { 'http.status.code': '500' },
-    { 'http.status.code': '200' },
-    { 'http.status.code': '404' },
-    { job: '"test-job"' },
-    { job: '"main-job"' },
-    { job: '"long-job"' },
-    { error: '"lorem ipsum"' },
-    { error: '"something went wrong"' },
-  ];
-
-  const tracesCount = Math.random() * 20 + 20;
-  for (let i = 0; i < tracesCount; i++) {
-    const attr = Math.floor(Math.random() * attributes.length);
-    const startTime = (Date.now() - Math.random() * (i + 1) * 100000) * 1000000;
-    const t: TraceSearchMetadata = {
-      traceID: uuidv4().replace(/-/, '').substring(0, 16),
-      rootServiceName: 'service' + i,
-      rootTraceName: 'trace' + i,
-      startTimeUnixNano: startTime.toString(10),
-      durationMs: Math.random() * 1000,
-      spanSets: [],
-    };
-
-    const spanAttributes = [];
-    for (let k = 0; k < Math.random() * 2; k++) {
-      const newAttr = Math.floor(Math.random() * attributes.length);
-      if (newAttr !== attr) {
-        spanAttributes.push(attributes[newAttr]);
-      }
-    }
-
-    const spans: Span[] = [];
-    for (let j = 0; j < Math.random() * 3 + 1; j++) {
-      spans.push({
-        traceId: t.traceID,
-        spanId: uuidv4().replace(/-/, '').substring(0, 16),
-        name: uuidv4().replace(/-/, '').substring(0, 6),
-        startTimeUnixNano: startTime,
-        endTimeUnixNano: startTime + Math.random() * 10000000,
-        kind: SpanKind.INTERNAL,
-        attributes: spanAttributes,
-      });
-    }
-    t.spanSets!.push({ spans, attributes: [attributes[attr]] });
-    traces.push(t);
-  }
-
-  return {
-    traces,
-    metrics: {
-      inspectedTraces: tracesCount,
-      inspectedBytes: 83720,
-    },
-  };
-};

public/app/plugins/datasource/tempo/types.ts

@@ -56,9 +56,9 @@ export type TraceSearchMetadata = {
   traceID: string;
   rootServiceName: string;
   rootTraceName: string;
-  startTimeUnixNano: string;
-  durationMs: number;
-  spanSets?: Spanset[];
+  startTimeUnixNano?: string;
+  durationMs?: number;
+  spanSet?: { spans: Span[] };
 };
 
 export type SearchMetrics = {
@@ -81,15 +81,16 @@ export enum SpanKind {
 }
 
 export type Span = {
-  traceId: string;
-  spanId: string;
+  durationNanos: string;
+  traceId?: string;
+  spanID: string;
   traceState?: string;
   parentSpanId?: string;
-  name: string;
-  kind: SpanKind;
-  startTimeUnixNano: number;
-  endTimeUnixNano: number;
-  attributes?: KeyValue[];
+  name?: string;
+  kind?: SpanKind;
+  startTimeUnixNano: string;
+  endTimeUnixNano?: string;
+  attributes?: Array<{ key: string; value: { stringValue: string } }>;
   dropped_attributes_count?: number;
 };