diff --git a/packages/grafana-ui/src/utils/processTableData.test.ts b/packages/grafana-ui/src/utils/processTableData.test.ts index ba9edcacebd..0201e9e1447 100644 --- a/packages/grafana-ui/src/utils/processTableData.test.ts +++ b/packages/grafana-ui/src/utils/processTableData.test.ts @@ -1,4 +1,6 @@ -import { parseCSV, toTableData } from './processTableData'; +import { parseCSV, toTableData, guessColumnTypes, guessColumnTypeFromValue } from './processTableData'; +import { ColumnType } from '../types/data'; +import moment from 'moment'; describe('processTableData', () => { describe('basic processing', () => { @@ -20,23 +22,23 @@ describe('processTableData', () => { }); describe('toTableData', () => { - it('converts timeseries to table skipping nulls', () => { + it('converts timeseries to table ', () => { const input1 = { target: 'Field Name', datapoints: [[100, 1], [200, 2]], }; + let table = toTableData(input1); + expect(table.columns[0].text).toBe(input1.target); + expect(table.rows).toBe(input1.datapoints); + + // Should fill a default name if target is empty const input2 = { // without target target: '', datapoints: [[100, 1], [200, 2]], }; - const data = toTableData([null, input1, input2, null, null]); - expect(data.length).toBe(2); - expect(data[0].columns[0].text).toBe(input1.target); - expect(data[0].rows).toBe(input1.datapoints); - - // Default name - expect(data[1].columns[0].text).toEqual('Value'); + table = toTableData(input2); + expect(table.columns[0].text).toEqual('Value'); }); it('keeps tableData unchanged', () => { @@ -44,15 +46,42 @@ describe('toTableData', () => { columns: [{ text: 'A' }, { text: 'B' }, { text: 'C' }], rows: [[100, 'A', 1], [200, 'B', 2], [300, 'C', 3]], }; - const data = toTableData([null, input, null, null]); - expect(data.length).toBe(1); - expect(data[0]).toBe(input); + const table = toTableData(input); + expect(table).toBe(input); + }); + + it('Guess Colum Types from value', () => { + expect(guessColumnTypeFromValue(1)).toBe(ColumnType.number); + expect(guessColumnTypeFromValue(1.234)).toBe(ColumnType.number); + expect(guessColumnTypeFromValue(3.125e7)).toBe(ColumnType.number); + expect(guessColumnTypeFromValue('1')).toBe(ColumnType.string); + expect(guessColumnTypeFromValue('1.234')).toBe(ColumnType.string); + expect(guessColumnTypeFromValue('3.125e7')).toBe(ColumnType.string); + expect(guessColumnTypeFromValue(true)).toBe(ColumnType.boolean); + expect(guessColumnTypeFromValue(false)).toBe(ColumnType.boolean); + expect(guessColumnTypeFromValue(new Date())).toBe(ColumnType.time); + expect(guessColumnTypeFromValue(moment())).toBe(ColumnType.time); }); - it('supports null values OK', () => { - expect(toTableData([null, null, null, null])).toEqual([]); - expect(toTableData(undefined)).toEqual([]); - expect(toTableData((null as unknown) as any[])).toEqual([]); - expect(toTableData([])).toEqual([]); + it('Guess Colum Types from strings', () => { + expect(guessColumnTypeFromValue('1', true)).toBe(ColumnType.number); + expect(guessColumnTypeFromValue('1.234', true)).toBe(ColumnType.number); + expect(guessColumnTypeFromValue('3.125e7', true)).toBe(ColumnType.number); + expect(guessColumnTypeFromValue('True', true)).toBe(ColumnType.boolean); + expect(guessColumnTypeFromValue('FALSE', true)).toBe(ColumnType.boolean); + expect(guessColumnTypeFromValue('true', true)).toBe(ColumnType.boolean); + expect(guessColumnTypeFromValue('xxxx', true)).toBe(ColumnType.string); + }); + + it('Guess Colum Types from table', () => { + const table = { + columns: [{ text: 'A (number)' }, { text: 'B (strings)' }, { text: 'C (nulls)' }, { text: 'Time' }], + rows: [[123, null, null, '2000'], [null, '123', null, 'XXX']], + }; + const norm = guessColumnTypes(table); + expect(norm.columns[0].type).toBe(ColumnType.number); + expect(norm.columns[1].type).toBe(ColumnType.string); + expect(norm.columns[2].type).toBeUndefined(); + expect(norm.columns[3].type).toBe(ColumnType.time); // based on name }); }); diff --git a/packages/grafana-ui/src/utils/processTableData.ts b/packages/grafana-ui/src/utils/processTableData.ts index 1ce514e7474..cc5c4ce7164 100644 --- a/packages/grafana-ui/src/utils/processTableData.ts +++ b/packages/grafana-ui/src/utils/processTableData.ts @@ -1,6 +1,8 @@ // Libraries import isNumber from 'lodash/isNumber'; import isString from 'lodash/isString'; +import isBoolean from 'lodash/isBoolean'; +import moment from 'moment'; import Papa, { ParseError, ParseMeta } from 'papaparse'; @@ -159,77 +161,112 @@ export const getFirstTimeColumn = (table: TableData): number => { return -1; }; +// PapaParse Dynamic Typing regex: +// https://github.com/mholt/PapaParse/blob/master/papaparse.js#L998 +const NUMBER = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i; + +/** + * Given a value this will guess the best column type + * + * TODO: better Date/Time support! Look for standard date strings? + */ +export function guessColumnTypeFromValue(v: any, parseString?: boolean): ColumnType { + if (isNumber(v)) { + return ColumnType.number; + } + + if (isString(v)) { + if (parseString) { + const c0 = v[0].toLowerCase(); + if (c0 === 't' || c0 === 'f') { + if (v === 'true' || v === 'TRUE' || v === 'True' || v === 'false' || v === 'FALSE' || v === 'False') { + return ColumnType.boolean; + } + } + + if (NUMBER.test(v)) { + return ColumnType.number; + } + } + return ColumnType.string; + } + + if (isBoolean(v)) { + return ColumnType.boolean; + } + + if (v instanceof Date || v instanceof moment) { + return ColumnType.time; + } + + return ColumnType.other; +} + +/** + * Looks at the data to guess the column type. This ignores any existing setting + */ +function guessColumnTypeFromTable(table: TableData, index: number, parseString?: boolean): ColumnType | undefined { + const column = table.columns[index]; + + // 1. Use the column name to guess + if (column.text) { + const name = column.text.toLowerCase(); + if (name === 'date' || name === 'time') { + return ColumnType.time; + } + } + + // 2. Check the first non-null value + for (let i = 0; i < table.rows.length; i++) { + const v = table.rows[i][index]; + if (v !== null) { + return guessColumnTypeFromValue(v, parseString); + } + } + + // Could not find anything + return undefined; +} + /** * @returns a table Returns a copy of the table with the best guess for each column type + * If the table already has column types defined, they will be used */ export const guessColumnTypes = (table: TableData): TableData => { - let changed = false; - const columns = table.columns.map((column, index) => { - if (!column.type) { - // 1. Use the column name to guess - if (column.text) { - const name = column.text.toLowerCase(); - if (name === 'date' || name === 'time') { - changed = true; + for (let i = 0; i < table.columns.length; i++) { + if (!table.columns[i].type) { + // Somethign is missing a type return a modified copy + return { + ...table, + columns: table.columns.map((column, index) => { + if (column.type) { + return column; + } + // Replace it with a calculated version return { ...column, - type: ColumnType.time, + type: guessColumnTypeFromTable(table, index), }; - } - } - - // 2. Check the first non-null value - for (let i = 0; i < table.rows.length; i++) { - const v = table.rows[i][index]; - if (v !== null) { - let type: ColumnType | undefined; - if (isNumber(v)) { - type = ColumnType.number; - } else if (isString(v)) { - type = ColumnType.string; - } - if (type) { - changed = true; - return { - ...column, - type, - }; - } - break; - } - } + }), + }; } - return column; - }); - if (changed) { - return { - ...table, - columns, - }; } + // No changes necessary return table; }; export const isTableData = (data: any): data is TableData => data && data.hasOwnProperty('columns'); -export const toTableData = (results?: any[]): TableData[] => { - if (!results) { - return []; +export const toTableData = (data: any): TableData => { + if (data.hasOwnProperty('columns')) { + return data as TableData; } - - return results - .filter(d => !!d) - .map(data => { - if (data.hasOwnProperty('columns')) { - return data as TableData; - } - if (data.hasOwnProperty('datapoints')) { - return convertTimeSeriesToTableData(data); - } - // TODO, try to convert JSON to table? - console.warn('Can not convert', data); - throw new Error('Unsupported data format'); - }); + if (data.hasOwnProperty('datapoints')) { + return convertTimeSeriesToTableData(data); + } + // TODO, try to convert JSON/Array to table? + console.warn('Can not convert', data); + throw new Error('Unsupported data format'); }; export function sortTableData(data: TableData, sortIndex?: number, reverse = false): TableData { diff --git a/public/app/features/dashboard/dashgrid/DataPanel.test.tsx b/public/app/features/dashboard/dashgrid/DataPanel.test.tsx new file mode 100644 index 00000000000..c2723b242e4 --- /dev/null +++ b/public/app/features/dashboard/dashgrid/DataPanel.test.tsx @@ -0,0 +1,60 @@ +// Library +import React from 'react'; + +import { DataPanel, getProcessedTableData } from './DataPanel'; + +describe('DataPanel', () => { + let dataPanel: DataPanel; + + beforeEach(() => { + dataPanel = new DataPanel({ + queries: [], + panelId: 1, + widthPixels: 100, + refreshCounter: 1, + datasource: 'xxx', + children: r => { + return
hello
; + }, + onError: (message, error) => {}, + }); + }); + + it('starts with unloaded state', () => { + expect(dataPanel.state.isFirstLoad).toBe(true); + }); + + it('converts timeseries to table skipping nulls', () => { + const input1 = { + target: 'Field Name', + datapoints: [[100, 1], [200, 2]], + }; + const input2 = { + // without target + target: '', + datapoints: [[100, 1], [200, 2]], + }; + const data = getProcessedTableData([null, input1, input2, null, null]); + expect(data.length).toBe(2); + expect(data[0].columns[0].text).toBe(input1.target); + expect(data[0].rows).toBe(input1.datapoints); + + // Default name + expect(data[1].columns[0].text).toEqual('Value'); + + // Every colun should have a name and a type + for (const table of data) { + for (const column of table.columns) { + expect(column.text).toBeDefined(); + expect(column.type).toBeDefined(); + } + } + }); + + it('supports null values from query OK', () => { + expect(getProcessedTableData([null, null, null, null])).toEqual([]); + expect(getProcessedTableData(undefined)).toEqual([]); + expect(getProcessedTableData((null as unknown) as any[])).toEqual([]); + expect(getProcessedTableData([])).toEqual([]); + }); +}); diff --git a/public/app/features/dashboard/dashgrid/DataPanel.tsx b/public/app/features/dashboard/dashgrid/DataPanel.tsx index 6941832c273..761fc599c16 100644 --- a/public/app/features/dashboard/dashgrid/DataPanel.tsx +++ b/public/app/features/dashboard/dashgrid/DataPanel.tsx @@ -15,6 +15,7 @@ import { TimeRange, ScopedVars, toTableData, + guessColumnTypes, } from '@grafana/ui'; interface RenderProps { @@ -46,6 +47,25 @@ export interface State { data?: TableData[]; } +/** + * All panels will be passed tables that have our best guess at colum type set + * + * This is also used by PanelChrome for snapshot support + */ +export function getProcessedTableData(results?: any[]): TableData[] { + if (!results) { + return []; + } + + const tables: TableData[] = []; + for (const r of results) { + if (r) { + tables.push(guessColumnTypes(toTableData(r))); + } + } + return tables; +} + export class DataPanel extends Component { static defaultProps = { isVisible: true, @@ -147,7 +167,7 @@ export class DataPanel extends Component { this.setState({ loading: LoadingState.Done, response: resp, - data: toTableData(resp.data), + data: getProcessedTableData(resp.data), isFirstLoad: false, }); } catch (err) { diff --git a/public/app/features/dashboard/dashgrid/PanelChrome.tsx b/public/app/features/dashboard/dashgrid/PanelChrome.tsx index 9a2dec5acd8..4872b93dc2b 100644 --- a/public/app/features/dashboard/dashgrid/PanelChrome.tsx +++ b/public/app/features/dashboard/dashgrid/PanelChrome.tsx @@ -19,11 +19,13 @@ import config from 'app/core/config'; // Types import { DashboardModel, PanelModel } from '../state'; import { PanelPlugin } from 'app/types'; -import { DataQueryResponse, TimeRange, LoadingState, TableData, DataQueryError, toTableData } from '@grafana/ui'; +import { DataQueryResponse, TimeRange, LoadingState, TableData, DataQueryError } from '@grafana/ui'; import { ScopedVars } from '@grafana/ui'; import templateSrv from 'app/features/templating/template_srv'; +import { getProcessedTableData } from './DataPanel'; + const DEFAULT_PLUGIN_ERROR = 'Error in plugin'; export interface Props { @@ -139,7 +141,7 @@ export class PanelChrome extends PureComponent { } get getDataForPanel() { - return this.hasPanelSnapshot ? toTableData(this.props.panel.snapshotData) : null; + return this.hasPanelSnapshot ? getProcessedTableData(this.props.panel.snapshotData) : null; } renderPanelPlugin(loading: LoadingState, data: TableData[], width: number, height: number): JSX.Element { diff --git a/public/app/plugins/panel/graph2/GraphPanel.tsx b/public/app/plugins/panel/graph2/GraphPanel.tsx index 1bd51cd97a7..99893631eb7 100644 --- a/public/app/plugins/panel/graph2/GraphPanel.tsx +++ b/public/app/plugins/panel/graph2/GraphPanel.tsx @@ -9,7 +9,6 @@ import { colors, TimeSeriesVMs, ColumnType, - guessColumnTypes, getFirstTimeColumn, processTimeSeries, } from '@grafana/ui'; @@ -23,8 +22,7 @@ export class GraphPanel extends PureComponent { const { showLines, showBars, showPoints } = this.props.options; const vmSeries: TimeSeriesVMs = []; - for (let t = 0; t < data.length; t++) { - const table = guessColumnTypes(data[t]); + for (const table of data) { const timeColumn = getFirstTimeColumn(table); if (timeColumn >= 0) { for (let i = 0; i < table.columns.length; i++) { diff --git a/public/app/plugins/panel/singlestat2/SingleStatPanel.tsx b/public/app/plugins/panel/singlestat2/SingleStatPanel.tsx index 2c151b09525..b8c222fadce 100644 --- a/public/app/plugins/panel/singlestat2/SingleStatPanel.tsx +++ b/public/app/plugins/panel/singlestat2/SingleStatPanel.tsx @@ -4,7 +4,7 @@ import React, { PureComponent, CSSProperties } from 'react'; // Types import { SingleStatOptions, SingleStatBaseOptions } from './types'; -import { DisplayValue, PanelProps, processTimeSeries, NullValueMode, guessColumnTypes, ColumnType } from '@grafana/ui'; +import { DisplayValue, PanelProps, processTimeSeries, NullValueMode, ColumnType } from '@grafana/ui'; import { config } from 'app/core/config'; import { getDisplayProcessor } from '@grafana/ui'; import { ProcessedValuesRepeater } from './ProcessedValuesRepeater'; @@ -25,8 +25,7 @@ export const getSingleStatValues = (props: PanelProps): D }); const values: DisplayValue[] = []; - for (let t = 0; t < data.length; t++) { - const table = guessColumnTypes(data[t]); + for (const table of data) { for (let i = 0; i < table.columns.length; i++) { const column = table.columns[i];