cleanup and guess all columns

packages/grafana-ui/src/utils/processTableData.test.ts

@@ -1,4 +1,6 @@
-import { parseCSV, toTableData } from './processTableData';
+import { parseCSV, toTableData, guessColumnTypes, guessColumnTypeFromValue } from './processTableData';
+import { ColumnType } from '../types/data';
+import moment from 'moment';
 
 describe('processTableData', () => {
   describe('basic processing', () => {
@@ -20,23 +22,23 @@ describe('processTableData', () => {
 });
 
 describe('toTableData', () => {
-  it('converts timeseries to table skipping nulls', () => {
+  it('converts timeseries to table ', () => {
     const input1 = {
       target: 'Field Name',
       datapoints: [[100, 1], [200, 2]],
     };
+    let table = toTableData(input1);
+    expect(table.columns[0].text).toBe(input1.target);
+    expect(table.rows).toBe(input1.datapoints);
+
+    // Should fill a default name if target is empty
     const input2 = {
       // without target
       target: '',
       datapoints: [[100, 1], [200, 2]],
     };
-    const data = toTableData([null, input1, input2, null, null]);
+    table = toTableData(input2);
-    expect(data.length).toBe(2);
+    expect(table.columns[0].text).toEqual('Value');
-    expect(data[0].columns[0].text).toBe(input1.target);
-    expect(data[0].rows).toBe(input1.datapoints);
-
-    // Default name
-    expect(data[1].columns[0].text).toEqual('Value');
   });
 
   it('keeps tableData unchanged', () => {
@@ -44,15 +46,42 @@ describe('toTableData', () => {
       columns: [{ text: 'A' }, { text: 'B' }, { text: 'C' }],
       rows: [[100, 'A', 1], [200, 'B', 2], [300, 'C', 3]],
     };
-    const data = toTableData([null, input, null, null]);
+    const table = toTableData(input);
-    expect(data.length).toBe(1);
+    expect(table).toBe(input);
-    expect(data[0]).toBe(input);
+  });
+
+  it('Guess Colum Types from value', () => {
+    expect(guessColumnTypeFromValue(1)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue(1.234)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue(3.125e7)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue('1')).toBe(ColumnType.string);
+    expect(guessColumnTypeFromValue('1.234')).toBe(ColumnType.string);
+    expect(guessColumnTypeFromValue('3.125e7')).toBe(ColumnType.string);
+    expect(guessColumnTypeFromValue(true)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue(false)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue(new Date())).toBe(ColumnType.time);
+    expect(guessColumnTypeFromValue(moment())).toBe(ColumnType.time);
   });
 
-  it('supports null values OK', () => {
+  it('Guess Colum Types from strings', () => {
-    expect(toTableData([null, null, null, null])).toEqual([]);
+    expect(guessColumnTypeFromValue('1', true)).toBe(ColumnType.number);
-    expect(toTableData(undefined)).toEqual([]);
+    expect(guessColumnTypeFromValue('1.234', true)).toBe(ColumnType.number);
-    expect(toTableData((null as unknown) as any[])).toEqual([]);
+    expect(guessColumnTypeFromValue('3.125e7', true)).toBe(ColumnType.number);
-    expect(toTableData([])).toEqual([]);
+    expect(guessColumnTypeFromValue('True', true)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue('FALSE', true)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue('true', true)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue('xxxx', true)).toBe(ColumnType.string);
+  });
+
+  it('Guess Colum Types from table', () => {
+    const table = {
+      columns: [{ text: 'A (number)' }, { text: 'B (strings)' }, { text: 'C (nulls)' }, { text: 'Time' }],
+      rows: [[123, null, null, '2000'], [null, '123', null, 'XXX']],
+    };
+    const norm = guessColumnTypes(table);
+    expect(norm.columns[0].type).toBe(ColumnType.number);
+    expect(norm.columns[1].type).toBe(ColumnType.string);
+    expect(norm.columns[2].type).toBeUndefined();
+    expect(norm.columns[3].type).toBe(ColumnType.time); // based on name
   });
 });

packages/grafana-ui/src/utils/processTableData.ts

@@ -1,6 +1,8 @@
 // Libraries
 import isNumber from 'lodash/isNumber';
 import isString from 'lodash/isString';
+import isBoolean from 'lodash/isBoolean';
+import moment from 'moment';
 
 import Papa, { ParseError, ParseMeta } from 'papaparse';
 
@@ -159,22 +161,58 @@ export const getFirstTimeColumn = (table: TableData): number => {
   return -1;
 };
 
+// PapaParse Dynamic Typing regex:
+// https://github.com/mholt/PapaParse/blob/master/papaparse.js#L998
+const NUMBER = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
+
 /**
- * @returns a table Returns a copy of the table with the best guess for each column type
+ * Given a value this will guess the best column type
+ *
+ * TODO: better Date/Time support!  Look for standard date strings?
  */
-export const guessColumnTypes = (table: TableData): TableData => {
+export function guessColumnTypeFromValue(v: any, parseString?: boolean): ColumnType {
-  let changed = false;
+  if (isNumber(v)) {
-  const columns = table.columns.map((column, index) => {
+    return ColumnType.number;
-    if (!column.type) {
+  }
+
+  if (isString(v)) {
+    if (parseString) {
+      const c0 = v[0].toLowerCase();
+      if (c0 === 't' || c0 === 'f') {
+        if (v === 'true' || v === 'TRUE' || v === 'True' || v === 'false' || v === 'FALSE' || v === 'False') {
+          return ColumnType.boolean;
+        }
+      }
+
+      if (NUMBER.test(v)) {
+        return ColumnType.number;
+      }
+    }
+    return ColumnType.string;
+  }
+
+  if (isBoolean(v)) {
+    return ColumnType.boolean;
+  }
+
+  if (v instanceof Date || v instanceof moment) {
+    return ColumnType.time;
+  }
+
+  return ColumnType.other;
+}
+
+/**
+ * Looks at the data to guess the column type.  This ignores any existing setting
+ */
+function guessColumnTypeFromTable(table: TableData, index: number, parseString?: boolean): ColumnType | undefined {
+  const column = table.columns[index];
+
   // 1. Use the column name to guess
   if (column.text) {
     const name = column.text.toLowerCase();
     if (name === 'date' || name === 'time') {
-          changed = true;
+      return ColumnType.time;
-          return {
-            ...column,
-            type: ColumnType.time,
-          };
     }
   }
 
@@ -182,54 +220,53 @@ export const guessColumnTypes = (table: TableData): TableData => {
   for (let i = 0; i < table.rows.length; i++) {
     const v = table.rows[i][index];
     if (v !== null) {
-          let type: ColumnType | undefined;
+      return guessColumnTypeFromValue(v, parseString);
-          if (isNumber(v)) {
-            type = ColumnType.number;
-          } else if (isString(v)) {
-            type = ColumnType.string;
-          }
-          if (type) {
-            changed = true;
-            return {
-              ...column,
-              type,
-            };
-          }
-          break;
-        }
     }
   }
-    return column;
+
-  });
+  // Could not find anything
-  if (changed) {
+  return undefined;
+}
+
+/**
+ * @returns a table Returns a copy of the table with the best guess for each column type
+ * If the table already has column types defined, they will be used
+ */
+export const guessColumnTypes = (table: TableData): TableData => {
+  for (let i = 0; i < table.columns.length; i++) {
+    if (!table.columns[i].type) {
+      // Somethign is missing a type return a modified copy
       return {
         ...table,
-      columns,
+        columns: table.columns.map((column, index) => {
+          if (column.type) {
+            return column;
+          }
+          // Replace it with a calculated version
+          return {
+            ...column,
+            type: guessColumnTypeFromTable(table, index),
           };
+        }),
+      };
+    }
   }
+  // No changes necessary
   return table;
 };
 
 export const isTableData = (data: any): data is TableData => data && data.hasOwnProperty('columns');
 
-export const toTableData = (results?: any[]): TableData[] => {
+export const toTableData = (data: any): TableData => {
-  if (!results) {
-    return [];
-  }
-
-  return results
-    .filter(d => !!d)
-    .map(data => {
   if (data.hasOwnProperty('columns')) {
     return data as TableData;
   }
   if (data.hasOwnProperty('datapoints')) {
     return convertTimeSeriesToTableData(data);
   }
-      // TODO, try to convert JSON to table?
+  // TODO, try to convert JSON/Array to table?
   console.warn('Can not convert', data);
   throw new Error('Unsupported data format');
-    });
 };
 
 export function sortTableData(data: TableData, sortIndex?: number, reverse = false): TableData {

public/app/features/dashboard/dashgrid/DataPanel.test.tsx

@@ -0,0 +1,60 @@
+// Library
+import React from 'react';
+
+import { DataPanel, getProcessedTableData } from './DataPanel';
+
+describe('DataPanel', () => {
+  let dataPanel: DataPanel;
+
+  beforeEach(() => {
+    dataPanel = new DataPanel({
+      queries: [],
+      panelId: 1,
+      widthPixels: 100,
+      refreshCounter: 1,
+      datasource: 'xxx',
+      children: r => {
+        return <div>hello</div>;
+      },
+      onError: (message, error) => {},
+    });
+  });
+
+  it('starts with unloaded state', () => {
+    expect(dataPanel.state.isFirstLoad).toBe(true);
+  });
+
+  it('converts timeseries to table skipping nulls', () => {
+    const input1 = {
+      target: 'Field Name',
+      datapoints: [[100, 1], [200, 2]],
+    };
+    const input2 = {
+      // without target
+      target: '',
+      datapoints: [[100, 1], [200, 2]],
+    };
+    const data = getProcessedTableData([null, input1, input2, null, null]);
+    expect(data.length).toBe(2);
+    expect(data[0].columns[0].text).toBe(input1.target);
+    expect(data[0].rows).toBe(input1.datapoints);
+
+    // Default name
+    expect(data[1].columns[0].text).toEqual('Value');
+
+    // Every colun should have a name and a type
+    for (const table of data) {
+      for (const column of table.columns) {
+        expect(column.text).toBeDefined();
+        expect(column.type).toBeDefined();
+      }
+    }
+  });
+
+  it('supports null values from query OK', () => {
+    expect(getProcessedTableData([null, null, null, null])).toEqual([]);
+    expect(getProcessedTableData(undefined)).toEqual([]);
+    expect(getProcessedTableData((null as unknown) as any[])).toEqual([]);
+    expect(getProcessedTableData([])).toEqual([]);
+  });
+});

public/app/features/dashboard/dashgrid/DataPanel.tsx

@@ -15,6 +15,7 @@ import {
   TimeRange,
   ScopedVars,
   toTableData,
+  guessColumnTypes,
 } from '@grafana/ui';
 
 interface RenderProps {
@@ -46,6 +47,25 @@ export interface State {
   data?: TableData[];
 }
 
+/**
+ * All panels will be passed tables that have our best guess at colum type set
+ *
+ * This is also used by PanelChrome for snapshot support
+ */
+export function getProcessedTableData(results?: any[]): TableData[] {
+  if (!results) {
+    return [];
+  }
+
+  const tables: TableData[] = [];
+  for (const r of results) {
+    if (r) {
+      tables.push(guessColumnTypes(toTableData(r)));
+    }
+  }
+  return tables;
+}
+
 export class DataPanel extends Component<Props, State> {
   static defaultProps = {
     isVisible: true,
@@ -147,7 +167,7 @@ export class DataPanel extends Component<Props, State> {
       this.setState({
         loading: LoadingState.Done,
         response: resp,
-        data: toTableData(resp.data),
+        data: getProcessedTableData(resp.data),
         isFirstLoad: false,
       });
     } catch (err) {

public/app/features/dashboard/dashgrid/PanelChrome.tsx

@@ -19,11 +19,13 @@ import config from 'app/core/config';
 // Types
 import { DashboardModel, PanelModel } from '../state';
 import { PanelPlugin } from 'app/types';
-import { DataQueryResponse, TimeRange, LoadingState, TableData, DataQueryError, toTableData } from '@grafana/ui';
+import { DataQueryResponse, TimeRange, LoadingState, TableData, DataQueryError } from '@grafana/ui';
 import { ScopedVars } from '@grafana/ui';
 
 import templateSrv from 'app/features/templating/template_srv';
 
+import { getProcessedTableData } from './DataPanel';
+
 const DEFAULT_PLUGIN_ERROR = 'Error in plugin';
 
 export interface Props {
@@ -139,7 +141,7 @@ export class PanelChrome extends PureComponent<Props, State> {
   }
 
   get getDataForPanel() {
-    return this.hasPanelSnapshot ? toTableData(this.props.panel.snapshotData) : null;
+    return this.hasPanelSnapshot ? getProcessedTableData(this.props.panel.snapshotData) : null;
   }
 
   renderPanelPlugin(loading: LoadingState, data: TableData[], width: number, height: number): JSX.Element {

public/app/plugins/panel/graph2/GraphPanel.tsx

@@ -9,7 +9,6 @@ import {
   colors,
   TimeSeriesVMs,
   ColumnType,
-  guessColumnTypes,
   getFirstTimeColumn,
   processTimeSeries,
 } from '@grafana/ui';
@@ -23,8 +22,7 @@ export class GraphPanel extends PureComponent<Props> {
     const { showLines, showBars, showPoints } = this.props.options;
 
     const vmSeries: TimeSeriesVMs = [];
-    for (let t = 0; t < data.length; t++) {
+    for (const table of data) {
-      const table = guessColumnTypes(data[t]);
       const timeColumn = getFirstTimeColumn(table);
       if (timeColumn >= 0) {
         for (let i = 0; i < table.columns.length; i++) {

public/app/plugins/panel/singlestat2/SingleStatPanel.tsx

@@ -4,7 +4,7 @@ import React, { PureComponent, CSSProperties } from 'react';
 // Types
 import { SingleStatOptions, SingleStatBaseOptions } from './types';
 
-import { DisplayValue, PanelProps, processTimeSeries, NullValueMode, guessColumnTypes, ColumnType } from '@grafana/ui';
+import { DisplayValue, PanelProps, processTimeSeries, NullValueMode, ColumnType } from '@grafana/ui';
 import { config } from 'app/core/config';
 import { getDisplayProcessor } from '@grafana/ui';
 import { ProcessedValuesRepeater } from './ProcessedValuesRepeater';
@@ -25,8 +25,7 @@ export const getSingleStatValues = (props: PanelProps<SingleStatBaseOptions>): D
   });
 
   const values: DisplayValue[] = [];
-  for (let t = 0; t < data.length; t++) {
+  for (const table of data) {
-    const table = guessColumnTypes(data[t]);
     for (let i = 0; i < table.columns.length; i++) {
       const column = table.columns[i];