From bba477050978e1bb9eb5f8d749bd18d4bd1223d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=A4ggmark?= Date: Fri, 4 Sep 2020 11:21:24 +0200 Subject: [PATCH] PanelInspector: Adds a Raw display mode but defaults to Formatted display mode (#27306) * PanelInspector: Fields with overrides are formatted correct in CSV * Refactor: adds raw format * Refactor: changes switch to Formatted values * Tests: adds tests for applyRawFieldOverrides and getRawDisplayProcessor * Test: change to utc timeZone * Refactor: changes after PR comments --- .../src/field/displayProcessor.test.ts | 26 ++- .../src/field/displayProcessor.ts | 9 +- .../src/field/fieldOverrides.test.ts | 209 +++++++++++++++++- .../grafana-data/src/field/fieldOverrides.ts | 61 +++-- packages/grafana-data/src/field/index.ts | 2 +- packages/grafana-data/src/utils/csv.test.ts | 31 ++- packages/grafana-data/src/utils/csv.ts | 26 +-- .../src/components/Table/DefaultCell.tsx | 49 ++-- .../components/Inspector/InspectDataTab.tsx | 47 +--- .../components/Inspector/PanelInspector.tsx | 2 +- 10 files changed, 353 insertions(+), 109 deletions(-) diff --git a/packages/grafana-data/src/field/displayProcessor.test.ts b/packages/grafana-data/src/field/displayProcessor.test.ts index bf8db95340b..24ccf9d26f1 100644 --- a/packages/grafana-data/src/field/displayProcessor.test.ts +++ b/packages/grafana-data/src/field/displayProcessor.test.ts @@ -1,4 +1,4 @@ -import { getDisplayProcessor } from './displayProcessor'; +import { getDisplayProcessor, getRawDisplayProcessor } from './displayProcessor'; import { DisplayProcessor, DisplayValue } from '../types/displayValue'; import { MappingType, ValueMapping } from '../types/valueMapping'; import { Field, FieldConfig, FieldType, GrafanaTheme, Threshold, ThresholdsMode } from '../types'; @@ -326,3 +326,27 @@ describe('Date display options', () => { }); }); }); + +describe('getRawDisplayProcessor', () => { + const processor = getRawDisplayProcessor(); + const date = new Date('2020-01-01T00:00:00.000Z'); + const timestamp = date.valueOf(); + + it.each` + value | expected + ${0} | ${'0'} + ${13.37} | ${'13.37'} + ${true} | ${'true'} + ${false} | ${'false'} + ${date} | ${`${date}`} + ${timestamp} | ${'1577836800000'} + ${'a string'} | ${'a string'} + ${null} | ${'null'} + ${undefined} | ${'undefined'} + ${{ value: 0, label: 'a label' }} | ${'[object Object]'} + `('when called with value:{$value}', ({ value, expected }) => { + const result = processor(value); + + expect(result).toEqual({ text: expected, numeric: null }); + }); +}); diff --git a/packages/grafana-data/src/field/displayProcessor.ts b/packages/grafana-data/src/field/displayProcessor.ts index 83004c151fe..ecfcb703696 100644 --- a/packages/grafana-data/src/field/displayProcessor.ts +++ b/packages/grafana-data/src/field/displayProcessor.ts @@ -4,7 +4,7 @@ import _ from 'lodash'; // Types import { Field, FieldType } from '../types/dataFrame'; import { GrafanaTheme } from '../types/theme'; -import { DisplayProcessor, DisplayValue, DecimalCount, DecimalInfo } from '../types/displayValue'; +import { DecimalCount, DecimalInfo, DisplayProcessor, DisplayValue } from '../types/displayValue'; import { getValueFormat } from '../valueFormats/valueFormats'; import { getMappedValue } from '../utils/valueMappings'; import { dateTime } from '../datetime'; @@ -166,3 +166,10 @@ export function getDecimalsForValue(value: number, decimalOverride?: DecimalCoun return { decimals, scaledDecimals }; } + +export function getRawDisplayProcessor(): DisplayProcessor { + return (value: any) => ({ + text: `${value}`, + numeric: (null as unknown) as number, + }); +} diff --git a/packages/grafana-data/src/field/fieldOverrides.test.ts b/packages/grafana-data/src/field/fieldOverrides.test.ts index f478083ea2b..a9806f651d2 100644 --- a/packages/grafana-data/src/field/fieldOverrides.test.ts +++ b/packages/grafana-data/src/field/fieldOverrides.test.ts @@ -1,27 +1,32 @@ import { + applyFieldOverrides, + applyRawFieldOverrides, FieldOverrideEnv, findNumericFieldMinMax, - setFieldConfigDefaults, - setDynamicConfigValue, - applyFieldOverrides, getLinksSupplier, + setDynamicConfigValue, + setFieldConfigDefaults, } from './fieldOverrides'; import { MutableDataFrame, toDataFrame } from '../dataframe'; import { + DataFrame, + Field, + FieldColorMode, FieldConfig, FieldConfigPropertyItem, - GrafanaTheme, - FieldType, - DataFrame, FieldConfigSource, + FieldType, + GrafanaTheme, InterpolateFunction, + ThresholdsMode, } from '../types'; -import { Registry } from '../utils'; +import { locationUtil, Registry } from '../utils'; import { mockStandardProperties } from '../utils/tests/mockStandardProperties'; import { FieldMatcherID } from '../transformations'; import { FieldConfigOptionsRegistry } from './FieldConfigOptionsRegistry'; import { getFieldDisplayName } from './fieldState'; -import { locationUtil } from '../utils'; +import { ArrayVector } from '../vector'; +import { getDisplayProcessor } from './displayProcessor'; const property1: any = { id: 'custom.property1', // Match field properties @@ -543,3 +548,191 @@ describe('getLinksSupplier', () => { ); }); }); + +describe('applyRawFieldOverrides', () => { + const getNumberFieldConfig = () => ({ + custom: {}, + thresholds: { + mode: ThresholdsMode.Absolute, + steps: [ + { + color: 'green', + value: (null as unknown) as number, + }, + { + color: 'red', + value: 80, + }, + ], + }, + mappings: [], + color: { + mode: FieldColorMode.Thresholds, + }, + min: 0, + max: 1599124316808, + }); + + const getEmptyConfig = () => ({ + custom: {}, + mappings: [], + }); + + const getDisplayValue = (frames: DataFrame[], frameIndex: number, fieldIndex: number) => { + const field = frames[frameIndex].fields[fieldIndex]; + const value = field.values.get(0); + return field.display!(value); + }; + + const expectRawDataDisplayValue = (frames: DataFrame[], frameIndex: number) => { + expect(getDisplayValue(frames, frameIndex, 0)).toEqual({ text: '1599045551050', numeric: null }); + expect(getDisplayValue(frames, frameIndex, 1)).toEqual({ text: '3.14159265359', numeric: null }); + expect(getDisplayValue(frames, frameIndex, 2)).toEqual({ text: '0', numeric: null }); + expect(getDisplayValue(frames, frameIndex, 3)).toEqual({ text: '0', numeric: null }); + expect(getDisplayValue(frames, frameIndex, 4)).toEqual({ text: 'A - string', numeric: null }); + expect(getDisplayValue(frames, frameIndex, 5)).toEqual({ text: '1599045551050', numeric: null }); + }; + + const expectFormattedDataDisplayValue = (frames: DataFrame[], frameIndex: number) => { + expect(getDisplayValue(frames, frameIndex, 0)).toEqual({ + color: '#F2495C', + numeric: 1599045551050, + prefix: undefined, + suffix: undefined, + text: '1599045551050', + threshold: { + color: 'red', + value: 80, + }, + }); + + expect(getDisplayValue(frames, frameIndex, 1)).toEqual({ + color: '#73BF69', + numeric: 3.14159265359, + prefix: undefined, + suffix: undefined, + text: '3.142', + threshold: { + color: 'green', + value: null, + }, + }); + + expect(getDisplayValue(frames, frameIndex, 2)).toEqual({ + color: '#73BF69', + numeric: 0, + prefix: undefined, + suffix: undefined, + text: '0', + threshold: { + color: 'green', + value: null, + }, + }); + + expect(getDisplayValue(frames, frameIndex, 3)).toEqual({ + numeric: 0, + prefix: undefined, + suffix: undefined, + text: '0', + }); + + expect(getDisplayValue(frames, frameIndex, 4)).toEqual({ + numeric: NaN, + prefix: undefined, + suffix: undefined, + text: 'A - string', + }); + + expect(getDisplayValue(frames, frameIndex, 5)).toEqual({ + numeric: 1599045551050, + prefix: undefined, + suffix: undefined, + text: '2020-09-02 11:19:11', + }); + }; + + describe('when called', () => { + it('then all fields should have their display processor replaced with the raw display processor', () => { + const numberAsEpoc: Field = { + name: 'numberAsEpoc', + type: FieldType.number, + values: new ArrayVector([1599045551050]), + config: getNumberFieldConfig(), + }; + + const numberWithDecimals: Field = { + name: 'numberWithDecimals', + type: FieldType.number, + values: new ArrayVector([3.14159265359]), + config: { + ...getNumberFieldConfig(), + decimals: 3, + }, + }; + + const numberAsBoolean: Field = { + name: 'numberAsBoolean', + type: FieldType.number, + values: new ArrayVector([0]), + config: getNumberFieldConfig(), + }; + + const boolean: Field = { + name: 'boolean', + type: FieldType.boolean, + values: new ArrayVector([0]), + config: getEmptyConfig(), + }; + + const string: Field = { + name: 'string', + type: FieldType.boolean, + values: new ArrayVector(['A - string']), + config: getEmptyConfig(), + }; + + const datetime: Field = { + name: 'datetime', + type: FieldType.time, + values: new ArrayVector([1599045551050]), + config: { + unit: 'dateTimeAsIso', + }, + }; + + const dataFrameA: DataFrame = toDataFrame({ + fields: [numberAsEpoc, numberWithDecimals, numberAsBoolean, boolean, string, datetime], + }); + + dataFrameA.fields[0].display = getDisplayProcessor({ field: dataFrameA.fields[0] }); + dataFrameA.fields[1].display = getDisplayProcessor({ field: dataFrameA.fields[1] }); + dataFrameA.fields[2].display = getDisplayProcessor({ field: dataFrameA.fields[2] }); + dataFrameA.fields[3].display = getDisplayProcessor({ field: dataFrameA.fields[3] }); + dataFrameA.fields[4].display = getDisplayProcessor({ field: dataFrameA.fields[4] }); + dataFrameA.fields[5].display = getDisplayProcessor({ field: dataFrameA.fields[5], timeZone: 'utc' }); + + const dataFrameB: DataFrame = toDataFrame({ + fields: [numberAsEpoc, numberWithDecimals, numberAsBoolean, boolean, string, datetime], + }); + + dataFrameB.fields[0].display = getDisplayProcessor({ field: dataFrameB.fields[0] }); + dataFrameB.fields[1].display = getDisplayProcessor({ field: dataFrameB.fields[1] }); + dataFrameB.fields[2].display = getDisplayProcessor({ field: dataFrameB.fields[2] }); + dataFrameB.fields[3].display = getDisplayProcessor({ field: dataFrameB.fields[3] }); + dataFrameB.fields[4].display = getDisplayProcessor({ field: dataFrameB.fields[4] }); + dataFrameB.fields[5].display = getDisplayProcessor({ field: dataFrameB.fields[5], timeZone: 'utc' }); + + const data = [dataFrameA, dataFrameB]; + const rawData = applyRawFieldOverrides(data); + + // expect raw data is correct + expectRawDataDisplayValue(rawData, 0); + expectRawDataDisplayValue(rawData, 1); + + // expect the original data is still the same + expectFormattedDataDisplayValue(data, 0); + expectFormattedDataDisplayValue(data, 1); + }); + }); +}); diff --git a/packages/grafana-data/src/field/fieldOverrides.ts b/packages/grafana-data/src/field/fieldOverrides.ts index 99ee72711cf..6d0e66bb162 100644 --- a/packages/grafana-data/src/field/fieldOverrides.ts +++ b/packages/grafana-data/src/field/fieldOverrides.ts @@ -1,37 +1,37 @@ import { - DynamicConfigValue, - FieldConfig, + ApplyFieldOverrideOptions, + ColorScheme, DataFrame, + DataLink, + DataSourceInstanceSettings, + DynamicConfigValue, Field, - FieldType, FieldColorMode, - ColorScheme, - FieldOverrideContext, - ScopedVars, - ApplyFieldOverrideOptions, + FieldConfig, FieldConfigPropertyItem, - LinkModel, - InterpolateFunction, - ValueLinkConfig, + FieldOverrideContext, + FieldType, GrafanaTheme, + InterpolateFunction, + LinkModel, + ScopedVars, TimeZone, - DataLink, - DataSourceInstanceSettings, + ValueLinkConfig, } from '../types'; -import { fieldMatchers, ReducerID, reduceField } from '../transformations'; +import { fieldMatchers, reduceField, ReducerID } from '../transformations'; import { FieldMatcher } from '../types/transformations'; import isNumber from 'lodash/isNumber'; import set from 'lodash/set'; import unset from 'lodash/unset'; import get from 'lodash/get'; -import { getDisplayProcessor } from './displayProcessor'; +import { getDisplayProcessor, getRawDisplayProcessor } from './displayProcessor'; import { guessFieldTypeForField } from '../dataframe'; import { standardFieldConfigEditorRegistry } from './standardFieldConfigEditorRegistry'; import { FieldConfigOptionsRegistry } from './FieldConfigOptionsRegistry'; import { DataLinkBuiltInVars, locationUtil } from '../utils'; import { formattedValueToString } from '../valueFormats'; import { getFieldDisplayValuesProxy } from './getFieldDisplayValuesProxy'; -import { getFrameDisplayName, getFieldDisplayName } from './fieldState'; +import { getFieldDisplayName, getFrameDisplayName } from './fieldState'; import { getTimeField } from '../dataframe/processDataFrame'; import { mapInternalLinkToExplore } from '../utils/dataLinks'; import { getTemplateProxyForField } from './templateProxies'; @@ -427,3 +427,34 @@ export const getLinksSupplier = ( } }); }; + +/** + * Return a copy of the DataFrame with raw data + */ +export function applyRawFieldOverrides(data: DataFrame[]): DataFrame[] { + if (!data || data.length === 0) { + return []; + } + + const newData = [...data]; + const processor = getRawDisplayProcessor(); + + for (let frameIndex = 0; frameIndex < newData.length; frameIndex++) { + const newFrame = { ...newData[frameIndex] }; + const newFields = [...newFrame.fields]; + + for (let fieldIndex = 0; fieldIndex < newFields.length; fieldIndex++) { + newFields[fieldIndex] = { + ...newFields[fieldIndex], + display: processor, + }; + } + + newData[frameIndex] = { + ...newFrame, + fields: newFields, + }; + } + + return newData; +} diff --git a/packages/grafana-data/src/field/index.ts b/packages/grafana-data/src/field/index.ts index 33af1f5cfbb..e471b07e139 100644 --- a/packages/grafana-data/src/field/index.ts +++ b/packages/grafana-data/src/field/index.ts @@ -5,6 +5,6 @@ export * from './standardFieldConfigEditorRegistry'; export * from './overrides/processors'; export { FieldConfigOptionsRegistry } from './FieldConfigOptionsRegistry'; -export { applyFieldOverrides, validateFieldConfig } from './fieldOverrides'; +export { applyFieldOverrides, validateFieldConfig, applyRawFieldOverrides } from './fieldOverrides'; export { getFieldDisplayValuesProxy } from './getFieldDisplayValuesProxy'; export { getFieldDisplayName, getFrameDisplayName } from './fieldState'; diff --git a/packages/grafana-data/src/utils/csv.test.ts b/packages/grafana-data/src/utils/csv.test.ts index bb869848cf2..eb29df0a671 100644 --- a/packages/grafana-data/src/utils/csv.test.ts +++ b/packages/grafana-data/src/utils/csv.test.ts @@ -1,10 +1,10 @@ -import { readCSV, toCSV, CSVHeaderStyle } from './csv'; -import { getDataFrameRow } from '../dataframe/processDataFrame'; +import { CSVHeaderStyle, readCSV, toCSV } from './csv'; +import { getDataFrameRow, toDataFrameDTO } from '../dataframe/processDataFrame'; // Test with local CSV files import fs from 'fs'; -import { toDataFrameDTO } from '../dataframe/processDataFrame'; import { MutableDataFrame } from '../dataframe'; +import { getDisplayProcessor } from '../field'; describe('read csv', () => { it('should get X and y', () => { @@ -115,4 +115,29 @@ describe('DataFrame to CSV', () => { " `); }); + + it('should use field display processor if exists', () => { + const dataFrame = new MutableDataFrame({ + fields: [ + { name: 'Time', values: [1589455688623] }, + { + name: 'Value', + values: [1589455688623], + config: { + unit: 'dateTimeAsIso', + }, + }, + ], + }); + + dataFrame.fields[1].display = getDisplayProcessor({ field: dataFrame.fields[1], timeZone: 'utc' }); + + const csv = toCSV([dataFrame]); + expect(csv).toMatchInlineSnapshot(` + "\\"Time\\",\\"Value\\" + 1589455688623,2020-05-14 11:28:08 + + " + `); + }); }); diff --git a/packages/grafana-data/src/utils/csv.ts b/packages/grafana-data/src/utils/csv.ts index 0fd717d1392..783dca09e7a 100644 --- a/packages/grafana-data/src/utils/csv.ts +++ b/packages/grafana-data/src/utils/csv.ts @@ -1,13 +1,13 @@ // Libraries -import Papa, { ParseResult, ParseConfig, Parser } from 'papaparse'; +import Papa, { ParseConfig, Parser, ParseResult } from 'papaparse'; import defaults from 'lodash/defaults'; -import isNumber from 'lodash/isNumber'; // Types -import { DataFrame, Field, FieldType, FieldConfig } from '../types'; +import { DataFrame, Field, FieldConfig, FieldType } from '../types'; import { guessFieldTypeFromValue } from '../dataframe/processDataFrame'; import { MutableDataFrame } from '../dataframe/MutableDataFrame'; import { getFieldDisplayName } from '../field'; +import { formattedValueToString } from '../valueFormats'; export enum CSVHeaderStyle { full, @@ -205,21 +205,11 @@ function writeValue(value: any, config: CSVConfig): string { } function makeFieldWriter(field: Field, config: CSVConfig): FieldWriter { - if (field.type) { - if (field.type === FieldType.boolean) { - return (value: any) => { - return value ? 'true' : 'false'; - }; - } - - if (field.type === FieldType.number) { - return (value: any) => { - if (isNumber(value)) { - return value.toString(); - } - return writeValue(value, config); - }; - } + if (field.display) { + return (value: any) => { + const displayValue = field.display!(value); + return writeValue(formattedValueToString(displayValue), config); + }; } return (value: any) => writeValue(value, config); diff --git a/packages/grafana-ui/src/components/Table/DefaultCell.tsx b/packages/grafana-ui/src/components/Table/DefaultCell.tsx index 41ba509b7ea..17b119346e2 100644 --- a/packages/grafana-ui/src/components/Table/DefaultCell.tsx +++ b/packages/grafana-ui/src/components/Table/DefaultCell.tsx @@ -1,7 +1,8 @@ import React, { FC } from 'react'; -import { TableCellProps } from './types'; import { formattedValueToString, LinkModel } from '@grafana/data'; +import { TableCellProps } from './types'; + export const DefaultCell: FC = props => { const { field, cell, tableStyles, row } = props; let link: LinkModel | undefined; @@ -13,33 +14,33 @@ export const DefaultCell: FC = props => { valueRowIndex: row.index, })[0]; } - const value = field.display ? formattedValueToString(displayValue) : displayValue; + const value = field.display ? formattedValueToString(displayValue) : `${displayValue}`; + + if (!link) { + return
{value}
; + } return (
- {link ? ( - { - // Allow opening in new tab - if (!(event.ctrlKey || event.metaKey || event.shiftKey) && link!.onClick) { - event.preventDefault(); - link!.onClick(event); - } + { + // Allow opening in new tab + if (!(event.ctrlKey || event.metaKey || event.shiftKey) && link!.onClick) { + event.preventDefault(); + link!.onClick(event); } - : undefined - } - target={link.target} - title={link.title} - className={tableStyles.tableCellLink} - > - {value} - - ) : ( - value - )} + } + : undefined + } + target={link.target} + title={link.title} + className={tableStyles.tableCellLink} + > + {value} +
); }; diff --git a/public/app/features/dashboard/components/Inspector/InspectDataTab.tsx b/public/app/features/dashboard/components/Inspector/InspectDataTab.tsx index d210c30b3ec..eed27f4f838 100644 --- a/public/app/features/dashboard/components/Inspector/InspectDataTab.tsx +++ b/public/app/features/dashboard/components/Inspector/InspectDataTab.tsx @@ -1,6 +1,7 @@ import React, { PureComponent } from 'react'; import { applyFieldOverrides, + applyRawFieldOverrides, DataFrame, DataTransformerID, dateTimeFormat, @@ -8,13 +9,8 @@ import { SelectableValue, toCSV, transformDataFrame, - getTimeField, - FieldType, - FormattedVector, - DisplayProcessor, - getDisplayProcessor, } from '@grafana/data'; -import { Button, Field, Icon, Switch, Select, Table, VerticalGroup, Container, HorizontalGroup } from '@grafana/ui'; +import { Button, Container, Field, HorizontalGroup, Icon, Select, Switch, Table, VerticalGroup } from '@grafana/ui'; import { selectors } from '@grafana/e2e-selectors'; import AutoSizer from 'react-virtualized-auto-sizer'; @@ -60,33 +56,6 @@ export class InspectDataTab extends PureComponent { const { panel } = this.props; const { transformId } = this.state; - // Replace the time field with a formatted time - const { timeIndex, timeField } = getTimeField(dataFrame); - - if (timeField) { - // Use the configured date or standard time display - let processor: DisplayProcessor | undefined = timeField.display; - if (!processor) { - processor = getDisplayProcessor({ - field: timeField, - }); - } - - const formattedDateField = { - ...timeField, - type: FieldType.string, - values: new FormattedVector(timeField.values, processor), - }; - - const fields = [...dataFrame.fields]; - fields[timeIndex!] = formattedDateField; - - dataFrame = { - ...dataFrame, - fields, - }; - } - const dataFrameCsv = toCSV([dataFrame]); const blob = new Blob([String.fromCharCode(0xfeff), dataFrameCsv], { @@ -146,12 +115,16 @@ export class InspectDataTab extends PureComponent { }); } + if (!options.withFieldConfig) { + return applyRawFieldOverrides(data); + } + // We need to apply field config even though it was already applied in the PanelQueryRunner. // That's because transformers create new fields and data frames, so i.e. display processor is no longer there return applyFieldOverrides({ data, theme: config.theme, - fieldConfig: options.withFieldConfig ? this.props.panel.fieldConfig : { defaults: {}, overrides: [] }, + fieldConfig: this.props.panel.fieldConfig, replaceVariables: (value: string) => { return value; }, @@ -185,7 +158,7 @@ export class InspectDataTab extends PureComponent { } if (options.withFieldConfig) { - activeString += 'field configuration'; + activeString += 'formatted data'; } } @@ -261,8 +234,8 @@ export class InspectDataTab extends PureComponent { )} {showFieldConfigsOption && ( = ({ panel, dashboard, defaultT const dispatch = useDispatch(); const [dataOptions, setDataOptions] = useState({ withTransforms: false, - withFieldConfig: false, + withFieldConfig: true, }); const { data, isLoading, error } = usePanelLatestData(panel, dataOptions); const metaDs = useDatasourceMetadata(data);