Heatmap: use DataFrame rather than LegacyResponseData (#19026)

* merge master

* TimeSeries: datasources with labels should export tags (not labels) (#18977)

* merge master

* export prometheus tags

* Annotations: Add annotations support to Loki (#18949)

* Explore: Unify background color for fresh logs (#18973)

* Singlestat: render lines on the panel when sparklines are enabled (#18984)

* Image rendering: Add deprecation warning when PhantomJS is used for rendering images (#18933)

* Add deprecation warning

* Update pkg/services/rendering/rendering.go

Co-Authored-By: Marcus Efraimsson <marcus.efraimsson@gmail.com>

* Units: Adding T,P,E,Z,and Y bytes (#18706)

* Adding T and P for bytes

Luckily, all the hard work was done before; just added in these prefixes for our production environment.

* Future-proofing with other values (why not?)

* Yottaflops?

* Cutting back down to Peta sizes, except for hashes

* Refactor: move ScopedVars to grafana/data (#18992)

* Refactor: Move sql_engine to sub package of tsdb (#18991)

this way importing the tsdb package does not come with xorm dependencies

* use DataFrame in heatmaps

* actually use the setting :)

* remove unused timeSrv

* merge with master / useDataFrames

* fix test function

* merge master

* fix datasource type on snapshot

* reuse DataFrame calcs from graph panel

* update comments

public/app/plugins/panel/heatmap/heatmap_ctrl.ts

@@ -13,7 +13,10 @@ import {
   sortSeriesByLabel,
 } from './heatmap_data_converter';
 import { auto } from 'angular';
-import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
+import { LegacyResponseData } from '@grafana/ui';
+import { getProcessedDataFrames } from 'app/features/dashboard/state/runRequest';
+import { DataFrame } from '@grafana/data';
+import { DataProcessor } from '../graph/data_processor';
 
 const X_BUCKET_NUMBER_DEFAULT = 30;
 const Y_BUCKET_NUMBER_DEFAULT = 10;
@@ -114,16 +117,16 @@ export class HeatmapCtrl extends MetricsPanelCtrl {
   selectionActivated: boolean;
   unitFormats: any;
   data: any;
-  series: any[];
-  timeSrv: any;
+  series: TimeSeries[];
   dataWarning: any;
   decimals: number;
   scaledDecimals: number;
 
+  processor: DataProcessor; // Shared with graph panel
+
   /** @ngInject */
-  constructor($scope: any, $injector: auto.IInjectorService, timeSrv: TimeSrv) {
+  constructor($scope: any, $injector: auto.IInjectorService) {
     super($scope, $injector);
-    this.timeSrv = timeSrv;
     this.selectionActivated = false;
 
     _.defaultsDeep(this.panel, panelDefaults);
@@ -131,11 +134,18 @@ export class HeatmapCtrl extends MetricsPanelCtrl {
     this.colorModes = colorModes;
     this.colorSchemes = colorSchemes;
 
+    // Use DataFrames
+    this.useDataFrames = true;
+    this.processor = new DataProcessor({
+      xaxis: { mode: 'custom' }, // NOT: 'histogram' :)
+      aliasColors: {}, // avoids null reference
+    });
+
     // Bind grafana panel events
     this.events.on('render', this.onRender.bind(this));
-    this.events.on('data-received', this.onDataReceived.bind(this));
+    this.events.on('data-frames-received', this.onDataFramesReceived.bind(this));
     this.events.on('data-error', this.onDataError.bind(this));
-    this.events.on('data-snapshot-load', this.onDataReceived.bind(this));
+    this.events.on('data-snapshot-load', this.onSnapshotLoad.bind(this));
     this.events.on('init-edit-mode', this.onInitEditMode.bind(this));
 
     this.onCardColorChange = this.onCardColorChange.bind(this);
@@ -272,15 +282,25 @@ export class HeatmapCtrl extends MetricsPanelCtrl {
   }
 
   getPanelDataSourceType() {
-    if (this.datasource.meta && this.datasource.meta.id) {
+    if (this.datasource && this.datasource.meta && this.datasource.meta.id) {
       return this.datasource.meta.id;
     } else {
       return 'unknown';
     }
   }
 
-  onDataReceived(dataList: any) {
-    this.series = dataList.map(this.seriesHandler.bind(this));
+  // This should only be called from the snapshot callback
+  onSnapshotLoad(dataList: LegacyResponseData[]) {
+    this.onDataFramesReceived(getProcessedDataFrames(dataList));
+  }
+
+  // Directly support DataFrame
+  onDataFramesReceived(data: DataFrame[]) {
+    this.series = this.processor.getSeriesList({ dataList: data, range: this.range }).map(ts => {
+      ts.color = null; // remove whatever the processor set
+      ts.flotpairs = ts.getFlotPairs(this.panel.nullPointMode);
+      return ts;
+    });
 
     this.dataWarning = null;
     const datapointsCount = _.reduce(
@@ -321,31 +341,7 @@ export class HeatmapCtrl extends MetricsPanelCtrl {
     this.render();
   }
 
-  seriesHandler(dataFrame: any) {
-    if (dataFrame.datapoints === undefined) {
-      throw new Error('Heatmap error: data should be a time series');
-    }
-
-    const series = new TimeSeries({
-      datapoints: dataFrame.datapoints,
-      alias: dataFrame.target,
-    });
-
-    series.flotpairs = series.getFlotPairs(this.panel.nullPointMode);
-
-    const datapoints = dataFrame.datapoints || [];
-    if (datapoints && datapoints.length > 0) {
-      const last = datapoints[datapoints.length - 1][1];
-      const from = this.range.from;
-      if (last - from.valueOf() < -10000) {
-        series.isOutsideRange = true;
-      }
-    }
-
-    return series;
-  }
-
-  parseSeries(series: any[]) {
+  parseSeries(series: TimeSeries[]) {
     const min = _.min(_.map(series, s => s.stats.min));
     const minLog = _.min(_.map(series, s => s.stats.logmin));
     const max = _.max(_.map(series, s => s.stats.max));
@@ -357,7 +353,7 @@ export class HeatmapCtrl extends MetricsPanelCtrl {
     };
   }
 
-  parseHistogramSeries(series: any[]) {
+  parseHistogramSeries(series: TimeSeries[]) {
     const bounds = _.map(series, s => Number(s.alias));
     const min = _.min(bounds);
     const minLog = _.min(bounds);

public/app/plugins/panel/heatmap/specs/heatmap_ctrl.test.ts

@@ -35,7 +35,7 @@ describe('HeatmapCtrl', () => {
       ];
 
       ctx.ctrl.range = { from: dateTime().valueOf(), to: dateTime().valueOf() };
-      ctx.ctrl.onDataReceived(data);
+      ctx.ctrl.onSnapshotLoad(data);
     });
 
     it('should set datapointsOutside', () => {
@@ -60,7 +60,7 @@ describe('HeatmapCtrl', () => {
       ];
 
       ctx.ctrl.range = range;
-      ctx.ctrl.onDataReceived(data);
+      ctx.ctrl.onSnapshotLoad(data);
     });
 
     it('should set datapointsOutside', () => {
@@ -71,7 +71,7 @@ describe('HeatmapCtrl', () => {
   describe('datapointsCount given 2 series', () => {
     beforeEach(() => {
       const data: any = [{ target: 'test.cpu1', datapoints: [] }, { target: 'test.cpu2', datapoints: [] }];
-      ctx.ctrl.onDataReceived(data);
+      ctx.ctrl.onSnapshotLoad(data);
     });
 
     it('should set datapointsCount warning', () => {