PanelData: Adds annotations and QueryResultMeta dataTopic (#27757)

* topics round two

* more props

* remove Data from enum

* PanelData: Simplify ideas a bit to only focus on the addition of annotations in panel data

* Test data scenario

* Old graph panel showing annotations from PanelData

* Fixed comment

* Fixed issues trying to remove use of event.source

* Added unit test

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

packages/grafana-data/src/types/annotations.ts

@@ -34,6 +34,7 @@ export interface AnnotationEvent {
   text?: string;
   type?: string;
   tags?: string[];
+  color?: string;
 
   // Currently used to merge annotations from alerts and dashboard
   source?: any; // source.type === 'dashboard'
@@ -49,7 +50,7 @@ export enum AnnotationEventFieldSource {
 }
 
 export interface AnnotationEventFieldMapping {
-  source?: AnnotationEventFieldSource; // defautls to 'field'
+  source?: AnnotationEventFieldSource; // defaults to 'field'
   value?: string;
   regex?: string;
 }

packages/grafana-data/src/types/data.ts

@@ -2,7 +2,7 @@ import { FieldConfig } from './dataFrame';
 import { DataTransformerConfig } from './transformations';
 import { ApplyFieldOverrideOptions } from './fieldOverrides';
 
-export type KeyValue<T = any> = { [s: string]: T };
+export type KeyValue<T = any> = Record<string, T>;
 
 /**
  * Represent panel data loading state.
@@ -15,6 +15,10 @@ export enum LoadingState {
   Error = 'Error',
 }
 
+export enum DataTopic {
+  Annotations = 'annotations',
+}
+
 export type PreferredVisualisationType = 'graph' | 'table' | 'logs' | 'trace';
 
 export interface QueryResultMeta {
@@ -33,6 +37,12 @@ export interface QueryResultMeta {
   /** Currently used to show results in Explore only in preferred visualisation option */
   preferredVisualisationType?: PreferredVisualisationType;
 
+  /**
+   * Optionally identify which topic the frame should be assigned to.
+   * A value specified in the response will override what the request asked for.
+   */
+  dataTopic?: DataTopic;
+
   /**
    * This is the raw query sent to the underlying system.  All macros and templating
    * as been applied.  When metadata contains this value, it will be shown in the query inspector

packages/grafana-data/src/types/datasource.ts

@@ -4,7 +4,7 @@ import { GrafanaPlugin, PluginMeta } from './plugin';
 import { PanelData } from './panel';
 import { LogRowModel } from './logs';
 import { AnnotationEvent, AnnotationSupport } from './annotations';
-import { KeyValue, LoadingState, TableData, TimeSeries } from './data';
+import { KeyValue, LoadingState, TableData, TimeSeries, DataTopic } from './data';
 import { DataFrame, DataFrameDTO } from './dataFrame';
 import { RawTimeRange, TimeRange } from './time';
 import { ScopedVars } from './ScopedVars';
@@ -412,6 +412,11 @@ export interface DataQuery {
    */
   queryType?: string;
 
+  /**
+   * The data topic resuls should be attached to
+   */
+  dataTopic?: DataTopic;
+
   /**
    * For mixed data sources the selected datasource is on the query level.
    * For non mixed scenarios this is undefined.

packages/grafana-data/src/types/panel.ts

@@ -27,6 +27,9 @@ export interface PanelData {
   /** Contains data frames with field overrides applied */
   series: DataFrame[];
 
+  /** A list of annotation items */
+  annotations?: DataFrame[];
+
   /** Request contains the queries and properties sent to the datasource */
   request?: DataQueryRequest;
 
@@ -43,34 +46,48 @@ export interface PanelData {
 export interface PanelProps<T = any> {
   /** ID of the panel within the current dashboard */
   id: number;
+
   /** Result set of panel queries */
   data: PanelData;
+
   /** Time range of the current dashboard */
   timeRange: TimeRange;
+
   /** Time zone of the current dashboard */
   timeZone: TimeZone;
+
   /** Panel options */
   options: T;
-  /** Panel options change handler */
+
-  onOptionsChange: (options: T) => void;
-  /** Field options configuration */
-  fieldConfig: FieldConfigSource;
-  /** Field config change handler */
-  onFieldConfigChange: (config: FieldConfigSource) => void;
   /** Indicates whether or not panel should be rendered transparent */
   transparent: boolean;
+
   /** Current width of the panel */
   width: number;
+
   /** Current height of the panel */
   height: number;
-  /** Template variables interpolation function */
+
-  replaceVariables: InterpolateFunction;
+  /** Field options configuration */
-  /** Time range change handler */
+  fieldConfig: FieldConfigSource;
-  onChangeTimeRange: (timeRange: AbsoluteTimeRange) => void;
+
   /** @internal */
   renderCounter: number;
+
   /** Panel title */
   title: string;
+
+  /** Panel options change handler */
+  onOptionsChange: (options: T) => void;
+
+  /** Field config change handler */
+  onFieldConfigChange: (config: FieldConfigSource) => void;
+
+  /** Template variables interpolation function */
+  replaceVariables: InterpolateFunction;
+
+  /** Time range change handler */
+  onChangeTimeRange: (timeRange: AbsoluteTimeRange) => void;
 }
 
 export interface PanelEditorProps<T = any> {

pkg/tsdb/testdatasource/scenarios.go

@@ -277,6 +277,14 @@ func init() {
 		},
 	})
 
+	registerScenario(&Scenario{
+		Id:   "annotations",
+		Name: "Annotations",
+		Handler: func(query *tsdb.Query, context *tsdb.TsdbQuery) *tsdb.QueryResult {
+			return tsdb.NewQueryResult()
+		},
+	})
+
 	registerScenario(&Scenario{
 		Id:   "table_static",
 		Name: "Table Static",

public/app/features/annotations/annotations_srv.ts

@@ -203,6 +203,8 @@ export class AnnotationsSrv {
 
     for (const item of results) {
       item.source = annotation;
+      item.color = annotation.iconColor;
+      item.type = annotation.name;
       item.isRegion = item.timeEnd && item.time !== item.timeEnd;
     }
 

public/app/features/annotations/event_manager.ts

@@ -115,16 +115,16 @@ export class EventManager {
         // add properties used by jquery flot events
         item.min = item.time;
         item.max = item.time;
-        item.eventType = item.source.name;
+        item.eventType = item.type;
 
         if (item.newState) {
           item.eventType = '$__' + item.newState;
           continue;
         }
 
-        if (!types[item.source.name]) {
+        if (!types[item.type]) {
-          types[item.source.name] = {
+          types[item.type] = {
-            color: item.source.iconColor,
+            color: item.color,
             position: 'BOTTOM',
             markerSize: 5,
           };

public/app/features/annotations/standardAnnotationSupport.test.ts

@@ -15,33 +15,43 @@ describe('DataFrame to annotations', () => {
     expect(events).toMatchInlineSnapshot(`
       Array [
         Object {
+          "color": "red",
           "tags": Array [
             "aaa",
             "bbb",
           ],
           "text": "t1",
           "time": 1,
+          "type": "default",
         },
         Object {
+          "color": "red",
           "tags": Array [
             "bbb",
             "ccc",
           ],
           "text": "t2",
           "time": 2,
+          "type": "default",
         },
         Object {
+          "color": "red",
           "tags": Array [
             "zyz",
           ],
           "text": "t3",
           "time": 3,
+          "type": "default",
         },
         Object {
+          "color": "red",
           "time": 4,
+          "type": "default",
         },
         Object {
+          "color": "red",
           "time": 5,
+          "type": "default",
         },
       ]
     `);
@@ -63,25 +73,32 @@ describe('DataFrame to annotations', () => {
       timeEnd: { value: 'time1' },
       title: { value: 'aaaaa' },
     });
+
     expect(events).toMatchInlineSnapshot(`
       Array [
         Object {
+          "color": "red",
           "text": "b1",
           "time": 100,
           "timeEnd": 111,
           "title": "a1",
+          "type": "default",
         },
         Object {
+          "color": "red",
           "text": "b2",
           "time": 200,
           "timeEnd": 222,
           "title": "a2",
+          "type": "default",
         },
         Object {
+          "color": "red",
           "text": "b3",
           "time": 300,
           "timeEnd": 333,
           "title": "a3",
+          "type": "default",
         },
       ]
     `);

public/app/features/annotations/standardAnnotationSupport.ts

@@ -2,7 +2,7 @@ import {
   DataFrame,
   AnnotationQuery,
   AnnotationSupport,
-  transformDataFrame,
+  standardTransformers,
   FieldType,
   Field,
   KeyValue,
@@ -54,19 +54,12 @@ export function singleFrameFromPanelData(data: DataFrame[]): DataFrame | undefin
   if (!data?.length) {
     return undefined;
   }
+
   if (data.length === 1) {
     return data[0];
   }
 
-  return transformDataFrame(
+  return standardTransformers.mergeTransformer.transformer({})(data)[0];
-    [
-      {
-        id: 'seriesToColumns',
-        options: { byField: 'Time' },
-      },
-    ],
-    data
-  )[0];
 }
 
 interface AnnotationEventFieldSetter {
@@ -109,6 +102,7 @@ export const annotationEventNames: AnnotationFieldInfo[] = [
 
 export function getAnnotationsFromData(data: DataFrame[], options?: AnnotationEventMappings): AnnotationEvent[] {
   const frame = singleFrameFromPanelData(data);
+
   if (!frame?.length) {
     return [];
   }
@@ -116,6 +110,7 @@ export function getAnnotationsFromData(data: DataFrame[], options?: AnnotationEv
   let hasTime = false;
   let hasText = false;
   const byName: KeyValue<Field> = {};
+
   for (const f of frame.fields) {
     const name = getFieldDisplayName(f, frame);
     byName[name.toLowerCase()] = f;
@@ -126,11 +121,14 @@ export function getAnnotationsFromData(data: DataFrame[], options?: AnnotationEv
   }
 
   const fields: AnnotationEventFieldSetter[] = [];
+
   for (const evts of annotationEventNames) {
     const opt = options[evts.key] || {}; //AnnotationEventFieldMapping
+
     if (opt.source === AnnotationEventFieldSource.Skip) {
       continue;
     }
+
     const setter: AnnotationEventFieldSetter = { key: evts.key, split: evts.split };
 
     if (opt.source === AnnotationEventFieldSource.Text) {
@@ -138,6 +136,7 @@ export function getAnnotationsFromData(data: DataFrame[], options?: AnnotationEv
     } else {
       const lower = (opt.value || evts.key).toLowerCase();
       setter.field = byName[lower];
+
       if (!setter.field && evts.field) {
         setter.field = evts.field(frame);
       }
@@ -159,10 +158,16 @@ export function getAnnotationsFromData(data: DataFrame[], options?: AnnotationEv
 
   // Add each value to the string
   const events: AnnotationEvent[] = [];
+
   for (let i = 0; i < frame.length; i++) {
-    const anno: AnnotationEvent = {};
+    const anno: AnnotationEvent = {
+      type: 'default',
+      color: 'red',
+    };
+
     for (const f of fields) {
       let v: any = undefined;
+
       if (f.text) {
         v = f.text; // TODO support templates!
       } else if (f.field) {
@@ -175,14 +180,16 @@ export function getAnnotationsFromData(data: DataFrame[], options?: AnnotationEv
         }
       }
 
-      if (!(v === null || v === undefined)) {
+      if (v !== null && v !== undefined) {
-        if (v && f.split) {
+        if (f.split && typeof v === 'string') {
-          v = (v as string).split(',');
+          v = v.split(',');
         }
         (anno as any)[f.key] = v;
       }
     }
+
     events.push(anno);
   }
+
   return events;
 }

public/app/features/dashboard/state/runRequest.test.ts

@@ -6,6 +6,7 @@ import {
   dateTime,
   LoadingState,
   PanelData,
+  DataTopic,
 } from '@grafana/data';
 import { Observable, Subscriber, Subscription } from 'rxjs';
 import { runRequest } from './runRequest';
@@ -251,6 +252,25 @@ describe('runRequest', () => {
       expectThatRangeHasNotMutated(ctx);
     });
   });
+
+  runRequestScenario('With annotations dataTopic', ctx => {
+    ctx.setup(() => {
+      ctx.start();
+      ctx.emitPacket({
+        data: [{ name: 'DataA-1' } as DataFrame],
+        key: 'A',
+      });
+      ctx.emitPacket({
+        data: [{ name: 'DataA-2', meta: { dataTopic: DataTopic.Annotations } } as DataFrame],
+        key: 'B',
+      });
+    });
+
+    it('should seperate annotations results', () => {
+      expect(ctx.results[1].annotations?.length).toBe(1);
+      expect(ctx.results[1].series.length).toBe(1);
+    });
+  });
 });
 
 const expectThatRangeHasNotMutated = (ctx: ScenarioCtx) => {

public/app/features/dashboard/state/runRequest.ts

@@ -1,6 +1,6 @@
 // Libraries
 import { Observable, of, timer, merge, from } from 'rxjs';
-import { flatten, map as lodashMap, isArray, isString } from 'lodash';
+import { map as isArray, isString } from 'lodash';
 import { map, catchError, takeUntil, mapTo, share, finalize, tap } from 'rxjs/operators';
 // Utils & Services
 import { backendSrv } from 'app/core/services/backend_srv';
@@ -16,6 +16,7 @@ import {
   dateMath,
   toDataFrame,
   DataFrame,
+  DataTopic,
   guessFieldTypes,
 } from '@grafana/data';
 import { toDataQueryError } from '@grafana/runtime';
@@ -54,19 +55,33 @@ export function processResponsePacket(packet: DataQueryResponse, state: RunningQ
       }
     : range;
 
-  const combinedData = flatten(
+  const series: DataQueryResponseData[] = [];
-    lodashMap(packets, (packet: DataQueryResponse) => {
+  const annotations: DataQueryResponseData[] = [];
+
+  for (const key in packets) {
+    const packet = packets[key];
+
     if (packet.error) {
       loadingState = LoadingState.Error;
       error = packet.error;
     }
-      return packet.data;
+
-    })
+    if (packet.data && packet.data.length) {
-  );
+      for (const dataItem of packet.data) {
+        if (dataItem.meta?.dataTopic === DataTopic.Annotations) {
+          annotations.push(dataItem);
+          continue;
+        }
+
+        series.push(dataItem);
+      }
+    }
+  }
 
   const panelData = {
     state: loadingState,
-    series: combinedData,
+    series,
+    annotations,
     error,
     request,
     timeRange,
@@ -77,11 +92,10 @@ export function processResponsePacket(packet: DataQueryResponse, state: RunningQ
 
 /**
  * This function handles the excecution of requests & and processes the single or multiple response packets into
- * a combined PanelData response.
+ * a combined PanelData response. It will
- * It will
+ *  Merge multiple responses into a single DataFrame array based on the packet key
- *  * Merge multiple responses into a single DataFrame array based on the packet key
+ *  Will emit a loading state if no response after 50ms
- *  * Will emit a loading state if no response after 50ms
+ *  Cancel any still running network requests on unsubscribe (using request.requestId)
- *  * Cancel any still running network requests on unsubscribe (using request.requestId)
  */
 export function runRequest(datasource: DataSourceApi, request: DataQueryRequest): Observable<PanelData> {
   let state: RunningQueryState = {
@@ -162,7 +176,7 @@ export function callQueryMethod(datasource: DataSourceApi, request: DataQueryReq
  * This is also used by PanelChrome for snapshot support
  */
 export function getProcessedDataFrames(results?: DataQueryResponseData[]): DataFrame[] {
-  if (!isArray(results)) {
+  if (!results || !isArray(results)) {
     return [];
   }
 
@@ -185,7 +199,7 @@ export function getProcessedDataFrames(results?: DataQueryResponseData[]): DataF
 }
 
 export function preProcessPanelData(data: PanelData, lastResult?: PanelData): PanelData {
-  const { series } = data;
+  const { series, annotations } = data;
 
   //  for loading states with no data, use last result
   if (data.state === LoadingState.Loading && series.length === 0) {
@@ -203,11 +217,13 @@ export function preProcessPanelData(data: PanelData, lastResult?: PanelData): Pa
   // Make sure the data frames are properly formatted
   const STARTTIME = performance.now();
   const processedDataFrames = getProcessedDataFrames(series);
+  const annotationsProcessed = getProcessedDataFrames(annotations);
   const STOPTIME = performance.now();
 
   return {
     ...data,
     series: processedDataFrames,
+    annotations: annotationsProcessed,
     timings: { dataProcessingTime: STOPTIME - STARTTIME },
   };
 }

public/app/features/panel/metrics_panel_ctrl.ts

@@ -35,6 +35,7 @@ class MetricsPanelCtrl extends PanelCtrl {
   dataList: LegacyResponseData[];
   querySubscription?: Unsubscribable | null;
   useDataFrames = false;
+  panelData?: PanelData;
 
   constructor($scope: any, $injector: any) {
     super($scope, $injector);
@@ -130,6 +131,8 @@ class MetricsPanelCtrl extends PanelCtrl {
   // Updates the response with information from the stream
   panelDataObserver = {
     next: (data: PanelData) => {
+      this.panelData = data;
+
       if (data.state === LoadingState.Error) {
         this.loading = false;
         this.processDataError(data.error);

public/app/plugins/datasource/testdata/datasource.ts

@@ -14,6 +14,9 @@ import {
   MetricFindValue,
   TableData,
   TimeSeries,
+  TimeRange,
+  DataTopic,
+  AnnotationEvent,
 } from '@grafana/data';
 import { Scenario, TestDataQuery } from './types';
 import { getBackendSrv, toDataQueryError } from '@grafana/runtime';
@@ -40,13 +43,21 @@ export class TestDataDataSource extends DataSourceApi<TestDataQuery> {
       if (target.hide) {
         continue;
       }
-      if (target.scenarioId === 'streaming_client') {
+
+      switch (target.scenarioId) {
+        case 'streaming_client':
           streams.push(runStream(target, options));
-      } else if (target.scenarioId === 'grafana_api') {
+          break;
+        case 'grafana_api':
           streams.push(runGrafanaAPI(target, options));
-      } else if (target.scenarioId === 'arrow') {
+          break;
+        case 'arrow':
           streams.push(runArrowFile(target, options));
-      } else {
+          break;
+        case 'annotations':
+          streams.push(this.annotationDataTopicTest(target, options));
+          break;
+        default:
           queries.push({
             ...target,
             intervalMs: options.intervalMs,
@@ -109,23 +120,36 @@ export class TestDataDataSource extends DataSourceApi<TestDataQuery> {
     return { data, error };
   }
 
-  annotationQuery(options: any) {
+  annotationDataTopicTest(target: TestDataQuery, req: DataQueryRequest<TestDataQuery>): Observable<DataQueryResponse> {
-    let timeWalker = options.range.from.valueOf();
+    return new Observable<DataQueryResponse>(observer => {
-    const to = options.range.to.valueOf();
+      const events = this.buildFakeAnnotationEvents(req.range, 10);
+      const dataFrame = new ArrayDataFrame(events);
+      dataFrame.meta = { dataTopic: DataTopic.Annotations };
+
+      observer.next({ key: target.refId, data: [dataFrame] });
+    });
+  }
+
+  buildFakeAnnotationEvents(range: TimeRange, count: number): AnnotationEvent[] {
+    let timeWalker = range.from.valueOf();
+    const to = range.to.valueOf();
     const events = [];
-    const eventCount = 10;
+    const step = (to - timeWalker) / count;
-    const step = (to - timeWalker) / eventCount;
 
-    for (let i = 0; i < eventCount; i++) {
+    for (let i = 0; i < count; i++) {
       events.push({
-        annotation: options.annotation,
         time: timeWalker,
         text: 'This is the text, <a href="https://grafana.com">Grafana.com</a>',
         tags: ['text', 'server'],
       });
       timeWalker += step;
     }
-    return Promise.resolve(events);
+
+    return events;
+  }
+
+  annotationQuery(options: any) {
+    return Promise.resolve(this.buildFakeAnnotationEvents(options.range, 10));
   }
 
   getQueryDisplayText(query: TestDataQuery) {

public/app/plugins/panel/graph/module.ts

@@ -25,8 +25,8 @@ import { getLocationSrv } from '@grafana/runtime';
 import { getDataTimeRange } from './utils';
 import { changePanelPlugin } from 'app/features/dashboard/state/actions';
 import { dispatch } from 'app/store/store';
-
 import { ThresholdMapper } from 'app/features/alerting/state/ThresholdMapper';
+import { getAnnotationsFromData } from 'app/features/annotations/standardAnnotationSupport';
 
 export class GraphCtrl extends MetricsPanelCtrl {
   static template = template;
@@ -235,6 +235,10 @@ export class GraphCtrl extends MetricsPanelCtrl {
           (this.seriesList as any).alertState = this.alertState.state;
         }
 
+        if (this.panelData!.annotations?.length) {
+          this.annotations = getAnnotationsFromData(this.panelData!.annotations!);
+        }
+
         this.render(this.seriesList);
       },
       () => {

public/app/plugins/panel/graph/specs/graph_ctrl.test.ts

@@ -39,6 +39,7 @@ describe('GraphCtrl', () => {
     ctx.ctrl.events = {
       emit: () => {},
     };
+    ctx.ctrl.panelData = {};
     ctx.ctrl.annotationsSrv = {
       getAnnotations: () => Promise.resolve({}),
     };