mirror of https://github.com/grafana/grafana
Transforms: Labels to fields rewrite that uses merge transform inside it (#27125)
* testing things * POC: Simplify labels to fields by using merge transform inside it * removed old code * Fixed test ts issues * Added valueLabel field option * Updated merge transform tests to not expect a sort * fixed type * refactoring to minimize nestingpull/27187/head
parent
d5687cce11
commit
ddabf4ade1
@ -1,232 +1,104 @@ |
||||
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; |
||||
import { LabelsToFieldsOptions, labelsToFieldsTransformer } from './labelsToFields'; |
||||
import { DataTransformerConfig, Field, FieldType } from '../../types'; |
||||
import { DataTransformerConfig, FieldType, FieldDTO } from '../../types'; |
||||
import { DataTransformerID } from './ids'; |
||||
import { toDataFrame } from '../../dataframe'; |
||||
import { toDataFrame, toDataFrameDTO } from '../../dataframe'; |
||||
import { transformDataFrame } from '../transformDataFrame'; |
||||
import { ArrayVector } from '../../vector'; |
||||
|
||||
describe('Labels as Columns', () => { |
||||
beforeAll(() => { |
||||
mockTransformationsRegistry([labelsToFieldsTransformer]); |
||||
}); |
||||
|
||||
it('data frame with 1 value and 1 label', () => { |
||||
it('data frame with two labels', () => { |
||||
const cfg: DataTransformerConfig<LabelsToFieldsOptions> = { |
||||
id: DataTransformerID.labelsToFields, |
||||
options: {}, |
||||
}; |
||||
|
||||
const oneValueOneLabelA = toDataFrame({ |
||||
name: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [1000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside' } }, |
||||
], |
||||
}); |
||||
|
||||
const oneValueOneLabelB = toDataFrame({ |
||||
name: 'B', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [2000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside' } }, |
||||
], |
||||
}); |
||||
|
||||
const result = transformDataFrame([cfg], [oneValueOneLabelA, oneValueOneLabelB]); |
||||
const expected: Field[] = [ |
||||
{ name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} }, |
||||
{ name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} }, |
||||
{ name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} }, |
||||
]; |
||||
|
||||
expect(result[0].fields).toEqual(expected); |
||||
}); |
||||
|
||||
it('data frame with 2 values and 1 label', () => { |
||||
const cfg: DataTransformerConfig<LabelsToFieldsOptions> = { |
||||
id: DataTransformerID.labelsToFields, |
||||
options: {}, |
||||
}; |
||||
|
||||
const twoValuesOneLabelA = toDataFrame({ |
||||
name: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [1000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside' } }, |
||||
{ name: 'humidity', type: FieldType.number, values: [10000], labels: { location: 'inside' } }, |
||||
], |
||||
}); |
||||
|
||||
const twoValuesOneLabelB = toDataFrame({ |
||||
name: 'B', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [2000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside' } }, |
||||
{ name: 'humidity', type: FieldType.number, values: [11000], labels: { location: 'outside' } }, |
||||
], |
||||
}); |
||||
|
||||
const result = transformDataFrame([cfg], [twoValuesOneLabelA, twoValuesOneLabelB]); |
||||
const expected: Field[] = [ |
||||
{ name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} }, |
||||
{ name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} }, |
||||
{ name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} }, |
||||
{ name: 'humidity', type: FieldType.number, values: new ArrayVector([10000, 11000]), config: {} }, |
||||
]; |
||||
|
||||
expect(result[0].fields).toEqual(expected); |
||||
}); |
||||
|
||||
it('data frame with 1 value and 2 labels', () => { |
||||
const cfg: DataTransformerConfig<LabelsToFieldsOptions> = { |
||||
id: DataTransformerID.labelsToFields, |
||||
options: {}, |
||||
}; |
||||
|
||||
const oneValueTwoLabelsA = toDataFrame({ |
||||
const source = toDataFrame({ |
||||
name: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [1000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside', area: 'living room' } }, |
||||
], |
||||
}); |
||||
|
||||
const oneValueTwoLabelsB = toDataFrame({ |
||||
name: 'B', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [2000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside', area: 'backyard' } }, |
||||
{ name: 'time', type: FieldType.time, values: [1000, 2000] }, |
||||
{ name: 'Value', type: FieldType.number, values: [1, 2], labels: { location: 'inside', feelsLike: 'ok' } }, |
||||
], |
||||
}); |
||||
|
||||
const result = transformDataFrame([cfg], [oneValueTwoLabelsA, oneValueTwoLabelsB]); |
||||
const expected: Field[] = [ |
||||
{ name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} }, |
||||
{ name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} }, |
||||
{ name: 'area', type: FieldType.string, values: new ArrayVector(['living room', 'backyard']), config: {} }, |
||||
{ name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} }, |
||||
const result = toDataFrameDTO(transformDataFrame([cfg], [source])[0]); |
||||
const expected: FieldDTO[] = [ |
||||
{ name: 'time', type: FieldType.time, values: [1000, 2000], config: {} }, |
||||
{ |
||||
name: 'location', |
||||
type: FieldType.string, |
||||
values: ['inside', 'inside'], |
||||
config: {}, |
||||
}, |
||||
{ name: 'feelsLike', type: FieldType.string, values: ['ok', 'ok'], config: {} }, |
||||
{ name: 'Value', type: FieldType.number, values: [1, 2], config: {} }, |
||||
]; |
||||
|
||||
expect(result[0].fields).toEqual(expected); |
||||
expect(result.fields).toEqual(expected); |
||||
}); |
||||
|
||||
it('data frame with 2 values and 2 labels', () => { |
||||
it('data frame with two labels and valueLabel option', () => { |
||||
const cfg: DataTransformerConfig<LabelsToFieldsOptions> = { |
||||
id: DataTransformerID.labelsToFields, |
||||
options: {}, |
||||
options: { valueLabel: 'name' }, |
||||
}; |
||||
|
||||
const twoValuesTwoLabelsA = toDataFrame({ |
||||
const source = toDataFrame({ |
||||
name: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [1000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside', area: 'living room' } }, |
||||
{ |
||||
name: 'humidity', |
||||
type: FieldType.number, |
||||
values: [10000], |
||||
labels: { location: 'inside', area: 'living room' }, |
||||
}, |
||||
], |
||||
}); |
||||
|
||||
const twoValuesTwoLabelsB = toDataFrame({ |
||||
name: 'B', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [2000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside', area: 'backyard' } }, |
||||
{ |
||||
name: 'humidity', |
||||
type: FieldType.number, |
||||
values: [11000], |
||||
labels: { location: 'outside', area: 'backyard' }, |
||||
}, |
||||
{ name: 'time', type: FieldType.time, values: [1000, 2000] }, |
||||
{ name: 'Value', type: FieldType.number, values: [1, 2], labels: { location: 'inside', name: 'Request' } }, |
||||
], |
||||
}); |
||||
|
||||
const result = transformDataFrame([cfg], [twoValuesTwoLabelsA, twoValuesTwoLabelsB]); |
||||
const expected: Field[] = [ |
||||
{ name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} }, |
||||
{ name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} }, |
||||
{ name: 'area', type: FieldType.string, values: new ArrayVector(['living room', 'backyard']), config: {} }, |
||||
{ name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} }, |
||||
{ name: 'humidity', type: FieldType.number, values: new ArrayVector([10000, 11000]), config: {} }, |
||||
const result = toDataFrameDTO(transformDataFrame([cfg], [source])[0]); |
||||
const expected: FieldDTO[] = [ |
||||
{ name: 'time', type: FieldType.time, values: [1000, 2000], config: {} }, |
||||
{ |
||||
name: 'location', |
||||
type: FieldType.string, |
||||
values: ['inside', 'inside'], |
||||
config: {}, |
||||
}, |
||||
{ name: 'Request', type: FieldType.number, values: [1, 2], config: {} }, |
||||
]; |
||||
|
||||
expect(result[0].fields).toEqual(expected); |
||||
expect(result.fields).toEqual(expected); |
||||
}); |
||||
|
||||
it('data frames with different labels', () => { |
||||
it('two data frames with 1 value and 1 label', () => { |
||||
const cfg: DataTransformerConfig<LabelsToFieldsOptions> = { |
||||
id: DataTransformerID.labelsToFields, |
||||
options: {}, |
||||
}; |
||||
|
||||
const oneValueDifferentLabelsA = toDataFrame({ |
||||
const oneValueOneLabelA = toDataFrame({ |
||||
name: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [1000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside', feelsLike: 'ok' } }, |
||||
{ name: 'temp', type: FieldType.number, values: [1], labels: { location: 'inside' } }, |
||||
], |
||||
}); |
||||
|
||||
const oneValueDifferentLabelsB = toDataFrame({ |
||||
const oneValueOneLabelB = toDataFrame({ |
||||
name: 'B', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [2000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside', sky: 'cloudy' } }, |
||||
], |
||||
}); |
||||
|
||||
const result = transformDataFrame([cfg], [oneValueDifferentLabelsA, oneValueDifferentLabelsB]); |
||||
const expected: Field[] = [ |
||||
{ name: 'time', type: FieldType.time, values: new ArrayVector([1000, 2000]), config: {} }, |
||||
{ name: 'location', type: FieldType.string, values: new ArrayVector(['inside', 'outside']), config: {} }, |
||||
{ name: 'feelsLike', type: FieldType.string, values: new ArrayVector(['ok', null]), config: {} }, |
||||
{ name: 'sky', type: FieldType.string, values: new ArrayVector([null, 'cloudy']), config: {} }, |
||||
{ name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1]), config: {} }, |
||||
]; |
||||
|
||||
expect(result[0].fields).toEqual(expected); |
||||
}); |
||||
|
||||
it('data frames with same timestamp and different labels', () => { |
||||
const cfg: DataTransformerConfig<LabelsToFieldsOptions> = { |
||||
id: DataTransformerID.labelsToFields, |
||||
options: {}, |
||||
}; |
||||
|
||||
const oneValueDifferentLabelsA = toDataFrame({ |
||||
name: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [1000, 2000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [1, 2], labels: { location: 'inside', feelsLike: 'ok' } }, |
||||
{ name: 'temp', type: FieldType.number, values: [-1], labels: { location: 'outside' } }, |
||||
], |
||||
}); |
||||
|
||||
const oneValueDifferentLabelsB = toDataFrame({ |
||||
name: 'B', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [1000, 2000] }, |
||||
{ name: 'temp', type: FieldType.number, values: [-1, -2], labels: { location: 'outside', sky: 'cloudy' } }, |
||||
], |
||||
}); |
||||
const result = toDataFrameDTO(transformDataFrame([cfg], [oneValueOneLabelA, oneValueOneLabelB])[0]); |
||||
|
||||
const result = transformDataFrame([cfg], [oneValueDifferentLabelsA, oneValueDifferentLabelsB]); |
||||
const expected: Field[] = [ |
||||
{ name: 'time', type: FieldType.time, values: new ArrayVector([1000, 1000, 2000, 2000]), config: {} }, |
||||
{ |
||||
name: 'location', |
||||
type: FieldType.string, |
||||
values: new ArrayVector(['inside', 'outside', 'inside', 'outside']), |
||||
config: {}, |
||||
}, |
||||
{ name: 'feelsLike', type: FieldType.string, values: new ArrayVector(['ok', null, 'ok', null]), config: {} }, |
||||
{ name: 'sky', type: FieldType.string, values: new ArrayVector([null, 'cloudy', null, 'cloudy']), config: {} }, |
||||
{ name: 'temp', type: FieldType.number, values: new ArrayVector([1, -1, 2, -2]), config: {} }, |
||||
const expected: FieldDTO[] = [ |
||||
{ name: 'time', type: FieldType.time, values: [1000, 2000], config: {} }, |
||||
{ name: 'location', type: FieldType.string, values: ['inside', 'outside'], config: {} }, |
||||
{ name: 'temp', type: FieldType.number, values: [1, -1], config: {} }, |
||||
]; |
||||
|
||||
expect(result[0].fields).toEqual(expected); |
||||
expect(result.fields).toEqual(expected); |
||||
}); |
||||
}); |
||||
|
||||
@ -1,173 +1,68 @@ |
||||
import { DataFrame, DataTransformerInfo, FieldType, Field } from '../../types'; |
||||
import { DataTransformerID } from './ids'; |
||||
import { ArrayVector } from '../../vector'; |
||||
import { filterFieldsTransformer } from './filter'; |
||||
import { FieldMatcherID } from '..'; |
||||
import { MutableField } from '../../dataframe'; |
||||
import { mergeTransformer } from './merge'; |
||||
|
||||
export interface LabelsToFieldsOptions {} |
||||
type MapItem = { type: FieldType; values: Record<string, any>; isValue: boolean }; |
||||
type SeriesMapItem = Record<string, MapItem>; |
||||
type Map = Record<string, SeriesMapItem>; |
||||
export interface LabelsToFieldsOptions { |
||||
/* |
||||
* If set this will use this label's value as the value field name. |
||||
*/ |
||||
valueLabel?: string; |
||||
} |
||||
|
||||
export const labelsToFieldsTransformer: DataTransformerInfo<LabelsToFieldsOptions> = { |
||||
id: DataTransformerID.labelsToFields, |
||||
name: 'Labels to fields', |
||||
description: 'Groups series by time and return labels as columns', |
||||
description: 'Extract time series labels to fields (columns)', |
||||
defaultOptions: {}, |
||||
transformer: options => (data: DataFrame[]) => { |
||||
const framesWithTimeField = filterFieldsTransformer.transformer({ include: { id: FieldMatcherID.time } })(data); |
||||
if (!framesWithTimeField.length || !framesWithTimeField[0].fields.length) { |
||||
return data; |
||||
} |
||||
|
||||
// get frames with only value fields
|
||||
const framesWithoutTimeField = getFramesWithOnlyValueFields(data); |
||||
if (!framesWithoutTimeField.length || !framesWithoutTimeField[0].fields.length) { |
||||
return data; |
||||
} |
||||
|
||||
const columnsMap = createColumnsMap(framesWithTimeField, framesWithoutTimeField); |
||||
const fields = createFields(columnsMap); |
||||
const values: Record<string, any[]> = {}; |
||||
|
||||
const timeColumnItem = columnsMap[fields[0].name]; |
||||
const seriesIndexStrings = Object.keys(timeColumnItem); |
||||
for (const seriesIndexString of seriesIndexStrings) { |
||||
const seriesItem = timeColumnItem[seriesIndexString]; |
||||
const timeValueStrings = Object.keys(seriesItem.values); |
||||
const result: DataFrame[] = []; |
||||
|
||||
for (const timeValueString of timeValueStrings) { |
||||
if (!values[timeValueString]) { |
||||
values[timeValueString] = []; |
||||
} |
||||
let row = new Array(fields.length); |
||||
for (let index = 0; index < fields.length; index++) { |
||||
const field = fields[index]; |
||||
const valueItem = columnsMap[field.name][seriesIndexString]; |
||||
const value = valueItem ? valueItem.values[timeValueString] ?? null : null; |
||||
row[index] = value; |
||||
} |
||||
values[timeValueString].push(row); |
||||
} |
||||
} |
||||
for (const frame of data) { |
||||
const newFields: Field[] = []; |
||||
|
||||
const timestamps = Object.values(values); |
||||
for (const timestamp of timestamps) { |
||||
for (const row of timestamp) { |
||||
for (let fieldIndex = 0; fieldIndex < fields.length; fieldIndex++) { |
||||
fields[fieldIndex].values.add(row[fieldIndex]); |
||||
for (const field of frame.fields) { |
||||
if (!field.labels) { |
||||
newFields.push(field); |
||||
continue; |
||||
} |
||||
} |
||||
} |
||||
|
||||
return [ |
||||
{ |
||||
fields, |
||||
length: fields[0].values.length, |
||||
}, |
||||
]; |
||||
}, |
||||
}; |
||||
|
||||
function getFramesWithOnlyValueFields(data: DataFrame[]): DataFrame[] { |
||||
const processed: DataFrame[] = []; |
||||
|
||||
for (const series of data) { |
||||
const fields: Field[] = []; |
||||
|
||||
for (let i = 0; i < series.fields.length; i++) { |
||||
const field = series.fields[i]; |
||||
|
||||
if (field.type !== FieldType.number) { |
||||
continue; |
||||
} |
||||
|
||||
fields.push(field); |
||||
} |
||||
|
||||
if (!fields.length) { |
||||
continue; |
||||
} |
||||
let name = field.name; |
||||
|
||||
const copy = { |
||||
...series, // all the other properties
|
||||
fields, // but a different set of fields
|
||||
}; |
||||
|
||||
processed.push(copy); |
||||
} |
||||
|
||||
return processed; |
||||
} |
||||
|
||||
function addOrAppendMapItem(args: { map: Map; series: number; column: string; type: FieldType; isValue?: boolean }) { |
||||
const { map, column, type, series, isValue = false } = args; |
||||
// we're using the fact that the series (number) will automatically become a string prop on the object
|
||||
const seriesMapItem: SeriesMapItem = { [series]: { type, values: {}, isValue } }; |
||||
if (!map[column]) { |
||||
map[column] = seriesMapItem; |
||||
} |
||||
|
||||
if (!map[column][series]) { |
||||
map[column] = { ...map[column], ...seriesMapItem }; |
||||
} |
||||
} |
||||
|
||||
// this is a naive implementation that does the job, not optimized for performance or speed
|
||||
function createColumnsMap(framesWithTimeField: DataFrame[], framesWithoutTimeField: DataFrame[]) { |
||||
const map: Map = {}; |
||||
|
||||
for (let frameIndex = 0; frameIndex < framesWithTimeField.length; frameIndex++) { |
||||
const timeFrame = framesWithTimeField[frameIndex]; |
||||
const otherFrame = framesWithoutTimeField[frameIndex]; |
||||
const timeField = timeFrame.fields[0]; |
||||
|
||||
addOrAppendMapItem({ map, column: timeField.name, series: frameIndex, type: timeField.type }); |
||||
|
||||
for (let valueIndex = 0; valueIndex < timeFrame.length; valueIndex++) { |
||||
const timeFieldValue = timeField.values.get(valueIndex); |
||||
map[timeField.name][frameIndex].values[timeFieldValue] = timeFieldValue; |
||||
|
||||
for (const field of otherFrame.fields) { |
||||
if (field.labels) { |
||||
const labels = Object.keys(field.labels); |
||||
for (const label of labels) { |
||||
addOrAppendMapItem({ map, column: label, series: frameIndex, type: FieldType.string }); |
||||
|
||||
map[label][frameIndex].values[timeFieldValue] = field.labels[label]; |
||||
for (const labelName of Object.keys(field.labels)) { |
||||
// if we should use this label as the value field name store it and skip adding this as a seperate field
|
||||
if (options.valueLabel === labelName) { |
||||
name = field.labels[labelName]; |
||||
continue; |
||||
} |
||||
} |
||||
|
||||
const otherFieldValue = field.values.get(valueIndex); |
||||
addOrAppendMapItem({ map, column: field.name, series: frameIndex, type: field.type, isValue: true }); |
||||
const values = new Array(frame.length).fill(field.labels[labelName]); |
||||
newFields.push({ |
||||
name: labelName, |
||||
type: FieldType.string, |
||||
values: new ArrayVector(values), |
||||
config: {}, |
||||
}); |
||||
} |
||||
|
||||
map[field.name][frameIndex].values[timeFieldValue] = otherFieldValue; |
||||
// add the value field but clear out any labels or displayName
|
||||
newFields.push({ |
||||
...field, |
||||
name, |
||||
config: { |
||||
...field.config, |
||||
displayName: undefined, |
||||
}, |
||||
labels: undefined, |
||||
}); |
||||
} |
||||
} |
||||
} |
||||
|
||||
return map; |
||||
} |
||||
|
||||
function createFields(columnsMap: Map): MutableField[] { |
||||
const columns = Object.keys(columnsMap); |
||||
const fields: MutableField[] = []; |
||||
const valueColumns: string[] = []; |
||||
|
||||
for (const column of columns) { |
||||
const columnItem = Object.values<MapItem>(columnsMap[column])[0]; |
||||
if (columnItem.isValue) { |
||||
valueColumns.push(column); |
||||
continue; |
||||
result.push({ |
||||
fields: newFields, |
||||
length: frame.length, |
||||
}); |
||||
} |
||||
fields.push({ type: columnItem.type, values: new ArrayVector(), name: column, config: {} }); |
||||
} |
||||
|
||||
for (const column of valueColumns) { |
||||
const columnItem = Object.values<MapItem>(columnsMap[column])[0]; |
||||
fields.push({ type: columnItem.type, values: new ArrayVector(), name: column, config: {} }); |
||||
} |
||||
|
||||
return fields; |
||||
} |
||||
return mergeTransformer.transformer({})(result); |
||||
}, |
||||
}; |
||||
|
||||
Loading…
Reference in new issue