From a3c1cd836e830b93e75ebecc8552a5970452b57d Mon Sep 17 00:00:00 2001 From: Alex Karacaoglu <31391932+AlexKaracaoglu@users.noreply.github.com> Date: Thu, 18 Aug 2022 13:22:45 -0400 Subject: [PATCH] Transformations: Add support for an inner join transformation (#53865) --- .../transformation-functions.md | 29 + packages/grafana-data/package.json | 1 + .../grafana-data/src/transformations/index.ts | 3 +- .../transformers/joinDataFrames.test.ts | 164 ++-- .../transformers/joinDataFrames.ts | 40 +- .../transformers/seriesToColumns.test.ts | 763 +++++++++++++++--- .../transformers/seriesToColumns.ts | 11 +- yarn.lock | 8 + 8 files changed, 814 insertions(+), 205 deletions(-) diff --git a/docs/sources/panels/transform-data/transformation-functions.md b/docs/sources/panels/transform-data/transformation-functions.md index 0da8bbca552..f8068c1179b 100644 --- a/docs/sources/panels/transform-data/transformation-functions.md +++ b/docs/sources/panels/transform-data/transformation-functions.md @@ -568,3 +568,32 @@ Here is the result after adding a Limit transformation with a value of '3': | 2020-07-07 11:34:20 | Temperature | 25 | | 2020-07-07 11:34:20 | Humidity | 22 | | 2020-07-07 10:32:20 | Humidity | 29 | + +## Join by field (Inner join) + +Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one single result and drop rows where a successful join isn't able to occur - performing an inner join. + +In the example below, we have two queries returning table data. It is visualized as two separate tables before applying the inner join transformation. + +Query A: + +| Time | Job | Uptime | +| ------------------- | ------- | --------- | +| 2020-07-07 11:34:20 | node | 25260122 | +| 2020-07-07 11:24:20 | postgre | 123001233 | +| 2020-07-07 11:14:20 | postgre | 345001233 | + +Query B: + +| Time | Server | Errors | +| ------------------- | -------- | ------ | +| 2020-07-07 11:34:20 | server 1 | 15 | +| 2020-07-07 11:24:20 | server 2 | 5 | +| 2020-07-07 11:04:20 | server 3 | 10 | + +Result after applying the inner join transformation: + +| Time | Job | Uptime | Server | Errors | +| ------------------- | ------- | --------- | -------- | ------ | +| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 | +| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 | diff --git a/packages/grafana-data/package.json b/packages/grafana-data/package.json index 27f35045861..c767d6b6aee 100644 --- a/packages/grafana-data/package.json +++ b/packages/grafana-data/package.json @@ -39,6 +39,7 @@ "d3-interpolate": "1.4.0", "date-fns": "2.29.1", "eventemitter3": "4.0.7", + "fast_array_intersect": "1.1.0", "history": "4.10.1", "lodash": "4.17.21", "marked": "4.0.18", diff --git a/packages/grafana-data/src/transformations/index.ts b/packages/grafana-data/src/transformations/index.ts index 0a159d2f241..f689933c0bd 100644 --- a/packages/grafana-data/src/transformations/index.ts +++ b/packages/grafana-data/src/transformations/index.ts @@ -15,6 +15,7 @@ export { ByNamesMatcherMode, } from './matchers/nameMatcher'; export type { RenameByRegexTransformerOptions } from './transformers/renameByRegex'; -export { outerJoinDataFrames } from './transformers/joinDataFrames'; +/** @deprecated -- will be removed in future versions */ +export { joinDataFrames as outerJoinDataFrames } from './transformers/joinDataFrames'; export * from './transformers/histogram'; export { ensureTimeField } from './transformers/convertFieldType'; diff --git a/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts b/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts index 4fb552b0f26..c97cfe27d42 100644 --- a/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts +++ b/packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts @@ -4,21 +4,21 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformatio import { ArrayVector } from '../../vector'; import { calculateFieldTransformer } from './calculateField'; -import { isLikelyAscendingVector, outerJoinDataFrames } from './joinDataFrames'; +import { isLikelyAscendingVector, joinDataFrames } from './joinDataFrames'; +import { JoinMode } from './seriesToColumns'; describe('align frames', () => { beforeAll(() => { mockTransformationsRegistry([calculateFieldTransformer]); }); - it('by first time field', () => { + describe('by first time field', () => { const series1 = toDataFrame({ fields: [ { name: 'TheTime', type: FieldType.time, values: [1000, 2000] }, { name: 'A', type: FieldType.number, values: [1, 100] }, ], }); - const series2 = toDataFrame({ fields: [ { name: '_time', type: FieldType.time, values: [1000, 1500, 2000] }, @@ -28,56 +28,106 @@ describe('align frames', () => { ], }); - const out = outerJoinDataFrames({ frames: [series1, series2] })!; - expect( - out.fields.map((f) => ({ - name: f.name, - values: f.values.toArray(), - })) - ).toMatchInlineSnapshot(` - Array [ - Object { - "name": "TheTime", - "values": Array [ - 1000, - 1500, - 2000, - ], - }, - Object { - "name": "A", - "values": Array [ - 1, - undefined, - 100, - ], - }, - Object { - "name": "A", - "values": Array [ - 2, - 20, - 200, - ], - }, - Object { - "name": "B", - "values": Array [ - 3, - 30, - 300, - ], - }, - Object { - "name": "C", - "values": Array [ - "first", - "second", - "third", - ], - }, - ] - `); + it('should perform an outer join', () => { + const out = joinDataFrames({ frames: [series1, series2] })!; + expect( + out.fields.map((f) => ({ + name: f.name, + values: f.values.toArray(), + })) + ).toMatchInlineSnapshot(` + Array [ + Object { + "name": "TheTime", + "values": Array [ + 1000, + 1500, + 2000, + ], + }, + Object { + "name": "A", + "values": Array [ + 1, + undefined, + 100, + ], + }, + Object { + "name": "A", + "values": Array [ + 2, + 20, + 200, + ], + }, + Object { + "name": "B", + "values": Array [ + 3, + 30, + 300, + ], + }, + Object { + "name": "C", + "values": Array [ + "first", + "second", + "third", + ], + }, + ] + `); + }); + + it('should perform an inner join', () => { + const out = joinDataFrames({ frames: [series1, series2], mode: JoinMode.inner })!; + expect( + out.fields.map((f) => ({ + name: f.name, + values: f.values.toArray(), + })) + ).toMatchInlineSnapshot(` + Array [ + Object { + "name": "TheTime", + "values": Array [ + 1000, + 2000, + ], + }, + Object { + "name": "A", + "values": Array [ + 1, + 100, + ], + }, + Object { + "name": "A", + "values": Array [ + 2, + 200, + ], + }, + Object { + "name": "B", + "values": Array [ + 3, + 300, + ], + }, + Object { + "name": "C", + "values": Array [ + "first", + "third", + ], + }, + ] + `); + }); }); it('unsorted input keep indexes', () => { @@ -96,7 +146,7 @@ describe('align frames', () => { ], }); - let out = outerJoinDataFrames({ frames: [series1, series3], keepOriginIndices: true })!; + let out = joinDataFrames({ frames: [series1, series3], keepOriginIndices: true })!; expect( out.fields.map((f) => ({ name: f.name, @@ -151,7 +201,7 @@ describe('align frames', () => { `); // Fast path still adds origin indecies - out = outerJoinDataFrames({ frames: [series1], keepOriginIndices: true })!; + out = joinDataFrames({ frames: [series1], keepOriginIndices: true })!; expect( out.fields.map((f) => ({ name: f.name, @@ -189,7 +239,7 @@ describe('align frames', () => { ], }); - const out = outerJoinDataFrames({ frames: [series1], keepOriginIndices: true })!; + const out = joinDataFrames({ frames: [series1], keepOriginIndices: true })!; expect( out.fields.map((f) => ({ name: f.name, @@ -236,7 +286,7 @@ describe('align frames', () => { ], }); - const out = outerJoinDataFrames({ frames: [series1, series3] })!; + const out = joinDataFrames({ frames: [series1, series3] })!; expect( out.fields.map((f) => ({ name: f.name, diff --git a/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts b/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts index 63cda277a14..ee994921111 100644 --- a/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts +++ b/packages/grafana-data/src/transformations/transformers/joinDataFrames.ts @@ -1,9 +1,13 @@ +import intersect from 'fast_array_intersect'; + import { getTimeField, sortDataFrame } from '../../dataframe'; import { DataFrame, Field, FieldMatcher, FieldType, Vector } from '../../types'; import { ArrayVector } from '../../vector'; import { fieldMatchers } from '../matchers'; import { FieldMatcherID } from '../matchers/ids'; +import { JoinMode } from './seriesToColumns'; + export function pickBestJoinField(data: DataFrame[]): FieldMatcher { const { timeField } = getTimeField(data[0]); if (timeField) { @@ -52,6 +56,11 @@ export interface JoinOptions { * @internal -- used when we need to keep a reference to the original frame/field index */ keepOriginIndices?: boolean; + + /** + * @internal -- Optionally specify a join mode (outer or inner) + */ + mode?: JoinMode; } function getJoinMatcher(options: JoinOptions): FieldMatcher { @@ -77,7 +86,7 @@ export function maybeSortFrame(frame: DataFrame, fieldIdx: number) { * This will return a single frame joined by the first matching field. When a join field is not specified, * the default will use the first time field */ -export function outerJoinDataFrames(options: JoinOptions): DataFrame | undefined { +export function joinDataFrames(options: JoinOptions): DataFrame | undefined { if (!options.frames?.length) { return; } @@ -211,7 +220,7 @@ export function outerJoinDataFrames(options: JoinOptions): DataFrame | undefined allData.push(a); } - const joined = join(allData, nullModes); + const joined = join(allData, nullModes, options.mode); return { // ...options.data[0], // keep name, meta? @@ -272,16 +281,23 @@ function nullExpand(yVals: Array, nullIdxs: number[], alignedLen: } // nullModes is a tables-matched array indicating how to treat nulls in each series -export function join(tables: AlignedData[], nullModes?: number[][]) { - const xVals = new Set(); - - for (let ti = 0; ti < tables.length; ti++) { - let t = tables[ti]; - let xs = t[0]; - let len = xs.length; - - for (let i = 0; i < len; i++) { - xVals.add(xs[i]); +export function join(tables: AlignedData[], nullModes?: number[][], mode: JoinMode = JoinMode.outer) { + let xVals: Set; + + if (mode === JoinMode.inner) { + // @ts-ignore + xVals = new Set(intersect(tables.map((t) => t[0]))); + } else { + xVals = new Set(); + + for (let ti = 0; ti < tables.length; ti++) { + let t = tables[ti]; + let xs = t[0]; + let len = xs.length; + + for (let i = 0; i < len; i++) { + xVals.add(xs[i]); + } } } diff --git a/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts b/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts index 1d73aa88521..9f04b9daef4 100644 --- a/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts +++ b/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts @@ -5,44 +5,45 @@ import { ArrayVector } from '../../vector'; import { transformDataFrame } from '../transformDataFrame'; import { DataTransformerID } from './ids'; -import { SeriesToColumnsOptions, seriesToColumnsTransformer } from './seriesToColumns'; +import { JoinMode, SeriesToColumnsOptions, seriesToColumnsTransformer } from './seriesToColumns'; describe('SeriesToColumns Transformer', () => { beforeAll(() => { mockTransformationsRegistry([seriesToColumnsTransformer]); }); - const everySecondSeries = toDataFrame({ - name: 'even', - fields: [ - { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] }, - { name: 'temperature', type: FieldType.number, values: [10.3, 10.4, 10.5, 10.6] }, - { name: 'humidity', type: FieldType.number, values: [10000.3, 10000.4, 10000.5, 10000.6] }, - ], - }); + describe('outer join', () => { + const everySecondSeries = toDataFrame({ + name: 'even', + fields: [ + { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] }, + { name: 'temperature', type: FieldType.number, values: [10.3, 10.4, 10.5, 10.6] }, + { name: 'humidity', type: FieldType.number, values: [10000.3, 10000.4, 10000.5, 10000.6] }, + ], + }); - const everyOtherSecondSeries = toDataFrame({ - name: 'odd', - fields: [ - { name: 'time', type: FieldType.time, values: [1000, 3000, 5000, 7000] }, - { name: 'temperature', type: FieldType.number, values: [11.1, 11.3, 11.5, 11.7] }, - { name: 'humidity', type: FieldType.number, values: [11000.1, 11000.3, 11000.5, 11000.7] }, - ], - }); + const everyOtherSecondSeries = toDataFrame({ + name: 'odd', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 3000, 5000, 7000] }, + { name: 'temperature', type: FieldType.number, values: [11.1, 11.3, 11.5, 11.7] }, + { name: 'humidity', type: FieldType.number, values: [11000.1, 11000.3, 11000.5, 11000.7] }, + ], + }); - it('joins by time field', async () => { - const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, - options: { - byField: 'time', - }, - }; + it('joins by time field', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + }, + }; - await expect(transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])).toEmitValuesWith( - (received) => { - const data = received[0]; - const filtered = data[0]; - expect(filtered.fields).toMatchInlineSnapshot(` + await expect(transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])).toEmitValuesWith( + (received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` Array [ Object { "config": Object {}, @@ -128,23 +129,23 @@ describe('SeriesToColumns Transformer', () => { }, ] `); - } - ); - }); + } + ); + }); - it('joins by temperature field', async () => { - const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, - options: { - byField: 'temperature', - }, - }; + it('joins by temperature field', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'temperature', + }, + }; - await expect(transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])).toEmitValuesWith( - (received) => { - const data = received[0]; - const filtered = data[0]; - expect(filtered.fields).toMatchInlineSnapshot(` + await expect(transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])).toEmitValuesWith( + (received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` Array [ Object { "config": Object {}, @@ -244,27 +245,27 @@ describe('SeriesToColumns Transformer', () => { }, ] `); - } - ); - }); + } + ); + }); - it('joins by time field in reverse order', async () => { - const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, - options: { - byField: 'time', - }, - }; + it('joins by time field in reverse order', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + }, + }; - everySecondSeries.fields[0].values = new ArrayVector(everySecondSeries.fields[0].values.toArray().reverse()); - everySecondSeries.fields[1].values = new ArrayVector(everySecondSeries.fields[1].values.toArray().reverse()); - everySecondSeries.fields[2].values = new ArrayVector(everySecondSeries.fields[2].values.toArray().reverse()); + everySecondSeries.fields[0].values = new ArrayVector(everySecondSeries.fields[0].values.toArray().reverse()); + everySecondSeries.fields[1].values = new ArrayVector(everySecondSeries.fields[1].values.toArray().reverse()); + everySecondSeries.fields[2].values = new ArrayVector(everySecondSeries.fields[2].values.toArray().reverse()); - await expect(transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])).toEmitValuesWith( - (received) => { - const data = received[0]; - const filtered = data[0]; - expect(filtered.fields).toMatchInlineSnapshot(` + await expect(transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])).toEmitValuesWith( + (received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` Array [ Object { "config": Object {}, @@ -352,40 +353,533 @@ describe('SeriesToColumns Transformer', () => { }, ] `); - } - ); + } + ); + }); + + describe('Field names', () => { + const seriesWithSameFieldAndDataFrameName = toDataFrame({ + name: 'temperature', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, + { name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, + { name: 'temperature', type: FieldType.number, values: [2, 4, 6, 8] }, + ], + }); + + it('when dataframe and field share the same name then use the field name', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + }, + }; + + await expect(transformDataFrame([cfg], [seriesWithSameFieldAndDataFrameName, seriesB])).toEmitValuesWith( + (received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` + Array [ + Object { + "config": Object {}, + "name": "time", + "state": Object {}, + "type": "time", + "values": Array [ + 1000, + 2000, + 3000, + 4000, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "temperature", + }, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 1, + 3, + 5, + 7, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "B", + }, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 2, + 4, + 6, + 8, + ], + }, + ] + `); + } + ); + }); + }); + + it('joins if fields are missing', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + }, + }; + + const frame1 = toDataFrame({ + name: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1, 2, 3] }, + { name: 'temperature', type: FieldType.number, values: [10, 11, 12] }, + ], + }); + + const frame2 = toDataFrame({ + name: 'B', + fields: [], + }); + + const frame3 = toDataFrame({ + name: 'C', + fields: [ + { name: 'time', type: FieldType.time, values: [1, 2, 3] }, + { name: 'temperature', type: FieldType.number, values: [20, 22, 24] }, + ], + }); + + await expect(transformDataFrame([cfg], [frame1, frame2, frame3])).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` + Array [ + Object { + "config": Object {}, + "name": "time", + "state": Object {}, + "type": "time", + "values": Array [ + 1, + 2, + 3, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "A", + }, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 10, + 11, + 12, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "C", + }, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 20, + 22, + 24, + ], + }, + ] + `); + }); + }); + + it('handles duplicate field name', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + }, + }; + + const frame1 = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1] }, + { name: 'temperature', type: FieldType.number, values: [10] }, + ], + }); + + const frame2 = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1] }, + { name: 'temperature', type: FieldType.number, values: [20] }, + ], + }); + + await expect(transformDataFrame([cfg], [frame1, frame2])).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` + Array [ + Object { + "config": Object {}, + "name": "time", + "state": Object {}, + "type": "time", + "values": Array [ + 1, + ], + }, + Object { + "config": Object {}, + "labels": Object {}, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 10, + ], + }, + Object { + "config": Object {}, + "labels": Object {}, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 20, + ], + }, + ] + `); + }); + }); }); - describe('Field names', () => { - const seriesWithSameFieldAndDataFrameName = toDataFrame({ - name: 'temperature', + describe('inner join', () => { + const seriesA = toDataFrame({ + name: 'A', fields: [ - { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, - { name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] }, + { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] }, + { name: 'temperature', type: FieldType.number, values: [10.3, 10.4, 10.5, 10.6] }, + { name: 'humidity', type: FieldType.number, values: [10000.3, 10000.4, 10000.5, 10000.6] }, ], }); const seriesB = toDataFrame({ name: 'B', fields: [ - { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, - { name: 'temperature', type: FieldType.number, values: [2, 4, 6, 8] }, + { name: 'time', type: FieldType.time, values: [1000, 3000, 5000, 7000] }, + { name: 'temperature', type: FieldType.number, values: [11.1, 10.3, 10.5, 11.7] }, + { name: 'humidity', type: FieldType.number, values: [11000.1, 10000.3, 10000.5, 11000.7] }, ], }); - it('when dataframe and field share the same name then use the field name', async () => { + it('inner joins by time field', async () => { const cfg: DataTransformerConfig = { id: DataTransformerID.seriesToColumns, options: { byField: 'time', + mode: JoinMode.inner, }, }; - await expect(transformDataFrame([cfg], [seriesWithSameFieldAndDataFrameName, seriesB])).toEmitValuesWith( - (received) => { - const data = received[0]; - const filtered = data[0]; - expect(filtered.fields).toMatchInlineSnapshot(` + await expect(transformDataFrame([cfg], [seriesA, seriesB])).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` + Array [ + Object { + "config": Object {}, + "name": "time", + "state": Object {}, + "type": "time", + "values": Array [ + 3000, + 5000, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "A", + }, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 10.3, + 10.5, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "A", + }, + "name": "humidity", + "state": Object {}, + "type": "number", + "values": Array [ + 10000.3, + 10000.5, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "B", + }, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 10.3, + 10.5, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "B", + }, + "name": "humidity", + "state": Object {}, + "type": "number", + "values": Array [ + 10000.3, + 10000.5, + ], + }, + ] + `); + }); + }); + + it('inner joins by temperature field', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'temperature', + mode: JoinMode.inner, + }, + }; + + await expect(transformDataFrame([cfg], [seriesA, seriesB])).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` + Array [ + Object { + "config": Object {}, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 10.3, + 10.5, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "A", + }, + "name": "time", + "state": Object { + "multipleFrames": true, + }, + "type": "time", + "values": Array [ + 3000, + 5000, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "A", + }, + "name": "humidity", + "state": Object {}, + "type": "number", + "values": Array [ + 10000.3, + 10000.5, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "B", + }, + "name": "time", + "state": Object { + "multipleFrames": true, + }, + "type": "time", + "values": Array [ + 3000, + 5000, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "B", + }, + "name": "humidity", + "state": Object {}, + "type": "number", + "values": Array [ + 10000.3, + 10000.5, + ], + }, + ] + `); + }); + }); + + it('inner joins by time field in reverse order', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + mode: JoinMode.inner, + }, + }; + + seriesA.fields[0].values = new ArrayVector(seriesA.fields[0].values.toArray().reverse()); + seriesA.fields[1].values = new ArrayVector(seriesA.fields[1].values.toArray().reverse()); + seriesA.fields[2].values = new ArrayVector(seriesA.fields[2].values.toArray().reverse()); + + await expect(transformDataFrame([cfg], [seriesA, seriesB])).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` + Array [ + Object { + "config": Object {}, + "name": "time", + "state": Object { + "multipleFrames": true, + }, + "type": "time", + "values": Array [ + 3000, + 5000, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "A", + }, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 10.3, + 10.5, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "A", + }, + "name": "humidity", + "state": Object {}, + "type": "number", + "values": Array [ + 10000.3, + 10000.5, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "B", + }, + "name": "temperature", + "state": Object {}, + "type": "number", + "values": Array [ + 10.3, + 10.5, + ], + }, + Object { + "config": Object {}, + "labels": Object { + "name": "B", + }, + "name": "humidity", + "state": Object {}, + "type": "number", + "values": Array [ + 10000.3, + 10000.5, + ], + }, + ] + `); + }); + }); + + describe('Field names', () => { + const seriesWithSameFieldAndDataFrameName = toDataFrame({ + name: 'temperature', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, + { name: 'temperature', type: FieldType.number, values: [1, 3, 5, 7] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 2000, 3000, 4000] }, + { name: 'temperature', type: FieldType.number, values: [2, 4, 6, 8] }, + ], + }); + + it('when dataframe and field share the same name then use the field name', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + mode: JoinMode.inner, + }, + }; + + await expect(transformDataFrame([cfg], [seriesWithSameFieldAndDataFrameName, seriesB])).toEmitValuesWith( + (received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` Array [ Object { "config": Object {}, @@ -431,44 +925,45 @@ describe('SeriesToColumns Transformer', () => { }, ] `); - } - ); + } + ); + }); }); - }); - it('joins if fields are missing', async () => { - const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, - options: { - byField: 'time', - }, - }; + it('joins if fields are missing', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + mode: JoinMode.inner, + }, + }; - const frame1 = toDataFrame({ - name: 'A', - fields: [ - { name: 'time', type: FieldType.time, values: [1, 2, 3] }, - { name: 'temperature', type: FieldType.number, values: [10, 11, 12] }, - ], - }); + const frame1 = toDataFrame({ + name: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [1, 2, 3] }, + { name: 'temperature', type: FieldType.number, values: [10, 11, 12] }, + ], + }); - const frame2 = toDataFrame({ - name: 'B', - fields: [], - }); + const frame2 = toDataFrame({ + name: 'B', + fields: [], + }); - const frame3 = toDataFrame({ - name: 'C', - fields: [ - { name: 'time', type: FieldType.time, values: [1, 2, 3] }, - { name: 'temperature', type: FieldType.number, values: [20, 22, 24] }, - ], - }); + const frame3 = toDataFrame({ + name: 'C', + fields: [ + { name: 'time', type: FieldType.time, values: [1, 2, 3] }, + { name: 'temperature', type: FieldType.number, values: [20, 22, 24] }, + ], + }); - await expect(transformDataFrame([cfg], [frame1, frame2, frame3])).toEmitValuesWith((received) => { - const data = received[0]; - const filtered = data[0]; - expect(filtered.fields).toMatchInlineSnapshot(` + await expect(transformDataFrame([cfg], [frame1, frame2, frame3])).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` Array [ Object { "config": Object {}, @@ -511,35 +1006,36 @@ describe('SeriesToColumns Transformer', () => { }, ] `); + }); }); - }); - it('handles duplicate field name', async () => { - const cfg: DataTransformerConfig = { - id: DataTransformerID.seriesToColumns, - options: { - byField: 'time', - }, - }; + it('handles duplicate field name', async () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + mode: JoinMode.inner, + }, + }; - const frame1 = toDataFrame({ - fields: [ - { name: 'time', type: FieldType.time, values: [1] }, - { name: 'temperature', type: FieldType.number, values: [10] }, - ], - }); + const frame1 = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1] }, + { name: 'temperature', type: FieldType.number, values: [10] }, + ], + }); - const frame2 = toDataFrame({ - fields: [ - { name: 'time', type: FieldType.time, values: [1] }, - { name: 'temperature', type: FieldType.number, values: [20] }, - ], - }); + const frame2 = toDataFrame({ + fields: [ + { name: 'time', type: FieldType.time, values: [1] }, + { name: 'temperature', type: FieldType.number, values: [20] }, + ], + }); - await expect(transformDataFrame([cfg], [frame1, frame2])).toEmitValuesWith((received) => { - const data = received[0]; - const filtered = data[0]; - expect(filtered.fields).toMatchInlineSnapshot(` + await expect(transformDataFrame([cfg], [frame1, frame2])).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields).toMatchInlineSnapshot(` Array [ Object { "config": Object {}, @@ -572,6 +1068,7 @@ describe('SeriesToColumns Transformer', () => { }, ] `); + }); }); }); }); diff --git a/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts b/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts index 785a8be6fda..cf034c8d623 100644 --- a/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts +++ b/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts @@ -5,10 +5,16 @@ import { fieldMatchers } from '../matchers'; import { FieldMatcherID } from '../matchers/ids'; import { DataTransformerID } from './ids'; -import { outerJoinDataFrames } from './joinDataFrames'; +import { joinDataFrames } from './joinDataFrames'; + +export enum JoinMode { + outer = 'outer', + inner = 'inner', +} export interface SeriesToColumnsOptions { byField?: string; // empty will pick the field automatically + mode?: JoinMode; } export const seriesToColumnsTransformer: SynchronousDataTransformerInfo = { @@ -17,6 +23,7 @@ export const seriesToColumnsTransformer: SynchronousDataTransformerInfo (source) => source.pipe(map((data) => seriesToColumnsTransformer.transformer(options)(data))), @@ -28,7 +35,7 @@ export const seriesToColumnsTransformer: SynchronousDataTransformerInfo