Transformers: Support inner vs outer join (#53913)

pull/54116/head
Ryan McKinley 3 years ago committed by GitHub
parent 1766ea9fdf
commit 1d4e01f8ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 638
      devenv/dev-dashboards/transforms/join-by-field.json
  2. 358
      devenv/dev-dashboards/transforms/join-by-labels.json
  3. 101
      docs/sources/panels/transform-data/index.md
  4. 6
      packages/grafana-data/src/transformations/transformers.ts
  5. 4
      packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts
  6. 4
      packages/grafana-data/src/transformations/transformers/ensureColumns.ts
  7. 3
      packages/grafana-data/src/transformations/transformers/ids.ts
  8. 30
      packages/grafana-data/src/transformations/transformers/joinByField.test.ts
  9. 16
      packages/grafana-data/src/transformations/transformers/joinByField.ts
  10. 2
      packages/grafana-data/src/transformations/transformers/joinDataFrames.test.ts
  11. 4
      packages/grafana-data/src/transformations/transformers/joinDataFrames.ts
  12. 1
      packages/grafana-data/src/utils/tests/mockTransformationsRegistry.ts
  13. 2
      public/app/features/explore/utils/decorators.ts
  14. 4
      public/app/features/inspector/InspectDataOptions.tsx
  15. 10
      public/app/features/inspector/InspectDataTab.tsx
  16. 72
      public/app/features/transformers/editors/JoinByFieldTransformerEditor.tsx
  17. 49
      public/app/features/transformers/editors/SeriesToFieldsTransformerEditor.tsx
  18. 4
      public/app/features/transformers/standardTransformers.ts

@ -0,0 +1,638 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1351,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 9,
"panels": [],
"title": "Join by time",
"type": "row"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 1
},
"id": 11,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"refId": "A",
"scenarioId": "random_walk",
"seriesCount": 4
}
],
"title": "Timeseries data",
"type": "timeseries"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 1
},
"id": 13,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 11,
"refId": "A"
}
],
"title": "Same data (as a table)",
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 9
},
"id": 16,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 11,
"refId": "A"
}
],
"title": "OUTER join on time (default)",
"transformations": [
{
"id": "joinByField",
"options": {}
}
],
"type": "table"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 14
},
"id": 5,
"panels": [],
"title": "Join by string field",
"type": "row"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 15
},
"id": 2,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 0,
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"csvContent": "OrderID,CustomerID,Time\n100,A,10000\n101,B,20000\n102,C,30000",
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"refId": "Orders",
"scenarioId": "csv_content"
},
{
"csvContent": "CustomerID,Name,Country\nA,Customer A,USA\nB,Customer B,Germany\nC,Customer C,Spain\nD,Customer D,Canada",
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"hide": false,
"refId": "Customers",
"scenarioId": "csv_content"
}
],
"title": "Orders",
"transformations": [],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 15
},
"id": 3,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 1,
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "Customers",
"transformations": [],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "CustomerID"
},
"properties": [
{
"id": "custom.width",
"value": 101
}
]
},
{
"matcher": {
"id": "byName",
"options": "OrderID"
},
"properties": [
{
"id": "custom.width",
"value": 89
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 23
},
"id": 6,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 0,
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "OUTER join on CustomerID (keeps missing values)",
"transformations": [
{
"id": "joinByField",
"options": {
"byField": "CustomerID",
"mode": "outer"
}
}
],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "CustomerID"
},
"properties": [
{
"id": "custom.width",
"value": 101
}
]
},
{
"matcher": {
"id": "byName",
"options": "OrderID"
},
"properties": [
{
"id": "custom.width",
"value": 89
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 23
},
"id": 7,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 0,
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "INNER join on CustomerID ",
"transformations": [
{
"id": "joinByField",
"options": {
"byField": "CustomerID",
"mode": "inner"
}
}
],
"type": "table"
}
],
"schemaVersion": 37,
"style": "dark",
"tags": [
"gdev",
"transform"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Join by field",
"uid": "gw0K4rmVz",
"version": 6,
"weekStart": ""
}

@ -0,0 +1,358 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1342,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"maxDataPoints": 1,
"options": {
"colorMode": "none",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"labels": "site=A,measure=speed,state=CA",
"refId": "A",
"scenarioId": "random_walk"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"hide": false,
"labels": "site=B,measure=speed,state=OR",
"refId": "B",
"scenarioId": "random_walk"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"hide": false,
"labels": "site=B,measure=temp",
"refId": "C",
"scenarioId": "random_walk"
},
{
"datasource": {
"type": "testdata",
"uid": "PD8C576611E62080A"
},
"hide": false,
"labels": "site=A,measure=temp",
"refId": "D",
"scenarioId": "random_walk"
}
],
"title": "Labeled values",
"type": "stat"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 0
},
"id": 5,
"maxDataPoints": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 2,
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "Same values... in a table",
"transformations": [],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 10
},
"id": 4,
"maxDataPoints": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "Join by site",
"transformations": [
{
"id": "joinByLabels",
"options": {
"join": [
"site"
],
"value": "measure"
}
}
],
"type": "table"
},
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 10
},
"id": 6,
"maxDataPoints": 1,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "9.2.0-pre",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "-- Dashboard --"
},
"panelId": 2,
"refId": "A"
}
],
"title": "Join on all labels",
"transformations": [
{
"id": "joinByLabels",
"options": {
"value": "measure"
}
}
],
"type": "table"
}
],
"schemaVersion": 37,
"style": "dark",
"tags": [
"gdev",
"transform"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Join by labels",
"uid": "FVl-9CR4z",
"version": 10,
"weekStart": ""
}

@ -322,15 +322,77 @@ We would then get :
| server 2 | 88.6 | 90 | 2020-07-07 10:32:20 | Overload |
| server 3 | 59.6 | 62 | 2020-07-07 11:34:20 | OK |
This transformation allows you to extract some key information out of your time series and display them in a convenient way.
This transformation enables you to extract key information from your time series and display it in a convenient way.
### Join by field (outer join)
### Join by field
Use this transformation to join multiple time series from a result set by field.
Use this transformation to join multiple results into a single table. This is especially useful for converting multiple
time series results into a single wide table with a shared time field.
This transformation is especially useful if you want to combine queries so that you can calculate results from the fields.
#### Inner join
In the example below, I have a template query displaying time series data from multiple servers in a table visualization. I can only view the results of one query at a time.
An inner join merges data from multiple tables where all tables share the same value from the selected field. This type of join excludes
data where values do not match in every result.
Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one result, and drop rows where a successful join cannot occur.
In the following example, two queries return table data. It is visualized as two separate tables before applying the inner join transformation.
Query A:
| Time | Job | Uptime |
| ------------------- | ------- | --------- |
| 2020-07-07 11:34:20 | node | 25260122 |
| 2020-07-07 11:24:20 | postgre | 123001233 |
| 2020-07-07 11:14:20 | postgre | 345001233 |
Query B:
| Time | Server | Errors |
| ------------------- | -------- | ------ |
| 2020-07-07 11:34:20 | server 1 | 15 |
| 2020-07-07 11:24:20 | server 2 | 5 |
| 2020-07-07 11:04:20 | server 3 | 10 |
The result after applying the inner join transformation looks like the following:
| Time | Job | Uptime | Server | Errors |
| ------------------- | ------- | --------- | -------- | ------ |
| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 |
| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 |
#### Outer join
An outer join includes all data from an inner join and rows where values do not match in every input.
Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one result, and drop rows where a successful join cannot occur - performing an inner join.
In the following example, two queries return table data. It is visualized as two tables before applying the inner join transformation.
Query A:
| Time | Job | Uptime |
| ------------------- | ------- | --------- |
| 2020-07-07 11:34:20 | node | 25260122 |
| 2020-07-07 11:24:20 | postgre | 123001233 |
| 2020-07-07 11:14:20 | postgre | 345001233 |
Query B:
| Time | Server | Errors |
| ------------------- | -------- | ------ |
| 2020-07-07 11:34:20 | server 1 | 15 |
| 2020-07-07 11:24:20 | server 2 | 5 |
| 2020-07-07 11:04:20 | server 3 | 10 |
The result after applying the inner join transformation looks like the following:
| Time | Job | Uptime | Server | Errors |
| ------------------- | ------- | --------- | -------- | ------ |
| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 |
| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 |
In the following example, a template query displays time series data from multiple servers in a table visualization. The results of only one query can be viewed at a time.
{{< figure src="/static/img/docs/transformations/join-fields-before-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
@ -643,32 +705,3 @@ Here is the result after adding a Limit transformation with a value of '3':
| 2020-07-07 11:34:20 | Temperature | 25 |
| 2020-07-07 11:34:20 | Humidity | 22 |
| 2020-07-07 10:32:20 | Humidity | 29 |
### Join by field (Inner join)
Use this transformation to combine the results from multiple queries (combining on a passed join field or the first time column) into one single result and drop rows where a successful join isn't able to occur - performing an inner join.
In the example below, we have two queries returning table data. It is visualized as two separate tables before applying the inner join transformation.
Query A:
| Time | Job | Uptime |
| ------------------- | ------- | --------- |
| 2020-07-07 11:34:20 | node | 25260122 |
| 2020-07-07 11:24:20 | postgre | 123001233 |
| 2020-07-07 11:14:20 | postgre | 345001233 |
Query B:
| Time | Server | Errors |
| ------------------- | -------- | ------ |
| 2020-07-07 11:34:20 | server 1 | 15 |
| 2020-07-07 11:24:20 | server 2 | 5 |
| 2020-07-07 11:04:20 | server 3 | 10 |
Result after applying the inner join transformation:
| Time | Job | Uptime | Server | Errors |
| ------------------- | ------- | --------- | -------- | ------ |
| 2020-07-07 11:34:20 | node | 25260122 | server 1 | 15 |
| 2020-07-07 11:24:20 | postgre | 123001233 | server 2 | 5 |

@ -9,6 +9,7 @@ import { filterByValueTransformer } from './transformers/filterByValue';
import { groupByTransformer } from './transformers/groupBy';
import { groupingToMatrixTransformer } from './transformers/groupingToMatrix';
import { histogramTransformer } from './transformers/histogram';
import { joinByFieldTransformer } from './transformers/joinByField';
import { labelsToFieldsTransformer } from './transformers/labelsToFields';
import { limitTransformer } from './transformers/limit';
import { mergeTransformer } from './transformers/merge';
@ -18,7 +19,6 @@ import { organizeFieldsTransformer } from './transformers/organize';
import { reduceTransformer } from './transformers/reduce';
import { renameFieldsTransformer } from './transformers/rename';
import { renameByRegexTransformer } from './transformers/renameByRegex';
import { seriesToColumnsTransformer } from './transformers/seriesToColumns';
import { seriesToRowsTransformer } from './transformers/seriesToRows';
import { sortByTransformer } from './transformers/sortBy';
@ -34,7 +34,9 @@ export const standardTransformers = {
reduceTransformer,
concatenateTransformer,
calculateFieldTransformer,
seriesToColumnsTransformer,
joinByFieldTransformer,
/** @deprecated */
seriesToColumnsTransformer: joinByFieldTransformer,
seriesToRowsTransformer,
renameFieldsTransformer,
labelsToFieldsTransformer,

@ -5,7 +5,7 @@ import { transformDataFrame } from '../transformDataFrame';
import { ensureColumnsTransformer } from './ensureColumns';
import { DataTransformerID } from './ids';
import { seriesToColumnsTransformer } from './seriesToColumns';
import { joinByFieldTransformer } from './joinByField';
const seriesA = toDataFrame({
fields: [
@ -33,7 +33,7 @@ const seriesNoTime = toDataFrame({
describe('ensureColumns transformer', () => {
beforeAll(() => {
mockTransformationsRegistry([ensureColumnsTransformer, seriesToColumnsTransformer]);
mockTransformationsRegistry([ensureColumnsTransformer, joinByFieldTransformer]);
});
it('will transform to columns if time field exists and multiple frames', async () => {

@ -5,7 +5,7 @@ import { DataFrame } from '../../types/dataFrame';
import { SynchronousDataTransformerInfo } from '../../types/transformations';
import { DataTransformerID } from './ids';
import { seriesToColumnsTransformer } from './seriesToColumns';
import { joinByFieldTransformer } from './joinByField';
export const ensureColumnsTransformer: SynchronousDataTransformerInfo = {
id: DataTransformerID.ensureColumns,
@ -19,7 +19,7 @@ export const ensureColumnsTransformer: SynchronousDataTransformerInfo = {
const timeFieldName = findConsistentTimeFieldName(frames);
if (frames.length > 1 && timeFieldName) {
return seriesToColumnsTransformer.transformer({
return joinByFieldTransformer.transformer({
byField: timeFieldName,
})(frames);
}

@ -1,5 +1,4 @@
export enum DataTransformerID {
// join = 'join', // Pick a field and merge all series based on that field
append = 'append',
// rotate = 'rotate', // Columns to rows
reduce = 'reduce',
@ -7,6 +6,7 @@ export enum DataTransformerID {
organize = 'organize',
rename = 'rename',
calculateField = 'calculateField',
/** @deprecated use joinByField */
seriesToColumns = 'seriesToColumns',
seriesToRows = 'seriesToRows',
merge = 'merge',
@ -30,6 +30,7 @@ export enum DataTransformerID {
fieldLookup = 'fieldLookup',
heatmap = 'heatmap',
spatial = 'spatial',
joinByField = 'joinByField',
joinByLabels = 'joinByLabels',
extractFields = 'extractFields',
groupingToMatrix = 'groupingToMatrix',

@ -5,11 +5,11 @@ import { ArrayVector } from '../../vector';
import { transformDataFrame } from '../transformDataFrame';
import { DataTransformerID } from './ids';
import { JoinMode, SeriesToColumnsOptions, seriesToColumnsTransformer } from './seriesToColumns';
import { JoinMode, JoinByFieldOptions, joinByFieldTransformer } from './joinByField';
describe('SeriesToColumns Transformer', () => {
describe('JOIN Transformer', () => {
beforeAll(() => {
mockTransformationsRegistry([seriesToColumnsTransformer]);
mockTransformationsRegistry([joinByFieldTransformer]);
});
describe('outer join', () => {
@ -32,7 +32,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins by time field', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -134,7 +134,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins by temperature field', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'temperature',
@ -250,7 +250,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins by time field in reverse order', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -375,7 +375,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('when dataframe and field share the same name then use the field name', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -438,7 +438,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins if fields are missing', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -516,7 +516,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('handles duplicate field name', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -597,7 +597,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('inner joins by time field', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -678,7 +678,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('inner joins by temperature field', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'temperature',
@ -763,7 +763,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('inner joins by time field in reverse order', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -867,7 +867,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('when dataframe and field share the same name then use the field name', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -931,7 +931,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('joins if fields are missing', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',
@ -1010,7 +1010,7 @@ describe('SeriesToColumns Transformer', () => {
});
it('handles duplicate field name', async () => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
const cfg: DataTransformerConfig<JoinByFieldOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
byField: 'time',

@ -12,23 +12,25 @@ export enum JoinMode {
inner = 'inner',
}
export interface SeriesToColumnsOptions {
export interface JoinByFieldOptions {
byField?: string; // empty will pick the field automatically
mode?: JoinMode;
}
export const seriesToColumnsTransformer: SynchronousDataTransformerInfo<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
name: 'Series as columns', // Called 'Outer join' in the UI!
description: 'Groups series by field and returns values as columns',
export const joinByFieldTransformer: SynchronousDataTransformerInfo<JoinByFieldOptions> = {
id: DataTransformerID.joinByField,
aliasIds: [DataTransformerID.seriesToColumns],
name: 'Join by field',
description:
'Combine rows from two or more tables, based on a related field between them. This can be used to outer join multiple time series on the _time_ field to show many time series in one table.',
defaultOptions: {
byField: undefined, // DEFAULT_KEY_FIELD,
mode: JoinMode.outer,
},
operator: (options) => (source) => source.pipe(map((data) => seriesToColumnsTransformer.transformer(options)(data))),
operator: (options) => (source) => source.pipe(map((data) => joinByFieldTransformer.transformer(options)(data))),
transformer: (options: SeriesToColumnsOptions) => {
transformer: (options: JoinByFieldOptions) => {
let joinBy: FieldMatcher | undefined = undefined;
return (data: DataFrame[]) => {
if (data.length > 1) {

@ -4,8 +4,8 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformatio
import { ArrayVector } from '../../vector';
import { calculateFieldTransformer } from './calculateField';
import { JoinMode } from './joinByField';
import { isLikelyAscendingVector, joinDataFrames } from './joinDataFrames';
import { JoinMode } from './seriesToColumns';
describe('align frames', () => {
beforeAll(() => {

@ -6,7 +6,7 @@ import { ArrayVector } from '../../vector';
import { fieldMatchers } from '../matchers';
import { FieldMatcherID } from '../matchers/ids';
import { JoinMode } from './seriesToColumns';
import { JoinMode } from './joinByField';
export function pickBestJoinField(data: DataFrame[]): FieldMatcher {
const { timeField } = getTimeField(data[0]);
@ -34,7 +34,7 @@ export function pickBestJoinField(data: DataFrame[]): FieldMatcher {
}
/**
* @alpha
* @internal
*/
export interface JoinOptions {
/**

@ -6,6 +6,7 @@ export const mockTransformationsRegistry = (transformers: Array<DataTransformerI
return transformers.map((t) => {
return {
id: t.id,
aliasIds: t.aliasIds,
name: t.name,
transformation: t,
description: t.description,

@ -109,7 +109,7 @@ export const decorateWithTableResult = (data: ExplorePanelData): Observable<Expl
// non timeseries or some mix of data we are not trying to join on anything and just try to merge them in
// single table, which may not make sense in most cases, but it's up to the user to query something sensible.
const transformer = hasOnlyTimeseries
? of(data.tableFrames).pipe(standardTransformers.seriesToColumnsTransformer.operator({}))
? of(data.tableFrames).pipe(standardTransformers.joinByFieldTransformer.operator({}))
: of(data.tableFrames).pipe(standardTransformers.mergeTransformer.operator({}));
return transformer.pipe(

@ -45,7 +45,7 @@ export const InspectDataOptions: FC<Props> = ({
const showFieldConfigsOption = panel && !panel.plugin?.fieldConfigRegistry.isEmpty();
let dataSelect = dataFrames;
if (selectedDataFrame === DataTransformerID.seriesToColumns) {
if (selectedDataFrame === DataTransformerID.joinByField) {
dataSelect = data!;
}
@ -67,7 +67,7 @@ export const InspectDataOptions: FC<Props> = ({
const parts: string[] = [];
if (selectedDataFrame === DataTransformerID.seriesToColumns) {
if (selectedDataFrame === DataTransformerID.joinByField) {
parts.push(t({ id: 'dashboard.inspect-data.series-to-columns', message: 'Series joined by time' }));
} else if (data.length > 1) {
parts.push(getFrameDisplayName(data[selectedDataFrame as number]));

@ -44,7 +44,7 @@ interface Props {
}
interface State {
/** The string is seriesToColumns transformation. Otherwise it is a dataframe index */
/** The string is joinByField transformation. Otherwise it is a dataframe index */
selectedDataFrame: number | DataTransformerID;
transformId: DataTransformerID;
dataFrameIndex: number;
@ -197,7 +197,7 @@ export class InspectDataTab extends PureComponent<Props, State> {
onDataFrameChange = (item: SelectableValue<DataTransformerID | number>) => {
this.setState({
transformId:
item.value === DataTransformerID.seriesToColumns ? DataTransformerID.seriesToColumns : DataTransformerID.noop,
item.value === DataTransformerID.joinByField ? DataTransformerID.joinByField : DataTransformerID.noop,
dataFrameIndex: typeof item.value === 'number' ? item.value : 0,
selectedDataFrame: item.value!,
});
@ -349,14 +349,14 @@ export class InspectDataTab extends PureComponent<Props, State> {
function buildTransformationOptions() {
const transformations: Array<SelectableValue<DataTransformerID>> = [
{
value: DataTransformerID.seriesToColumns,
value: DataTransformerID.joinByField,
label: t({
id: 'dashboard.inspect-data.transformation',
message: 'Series joined by time',
}),
transformer: {
id: DataTransformerID.seriesToColumns,
options: { byField: 'Time' },
id: DataTransformerID.joinByField,
options: { byField: undefined }, // defaults to time field
},
},
];

@ -0,0 +1,72 @@
import React, { useCallback } from 'react';
import {
DataTransformerID,
SelectableValue,
standardTransformers,
TransformerRegistryItem,
TransformerUIProps,
} from '@grafana/data';
import { JoinByFieldOptions, JoinMode } from '@grafana/data/src/transformations/transformers/joinByField';
import { Select, InlineFieldRow, InlineField } from '@grafana/ui';
import { useAllFieldNamesFromDataFrames } from '../utils';
const modes = [
{ value: JoinMode.outer, label: 'OUTER', description: 'Keep all rows from any table with a value' },
{ value: JoinMode.inner, label: 'INNER', description: 'Drop rows that do not match a value in all tables' },
];
export function SeriesToFieldsTransformerEditor({ input, options, onChange }: TransformerUIProps<JoinByFieldOptions>) {
const fieldNames = useAllFieldNamesFromDataFrames(input).map((item: string) => ({ label: item, value: item }));
const onSelectField = useCallback(
(value: SelectableValue<string>) => {
onChange({
...options,
byField: value?.value,
});
},
[onChange, options]
);
const onSetMode = useCallback(
(value: SelectableValue<JoinMode>) => {
onChange({
...options,
mode: value?.value,
});
},
[onChange, options]
);
return (
<>
<InlineFieldRow>
<InlineField label="Mode" labelWidth={8} grow>
<Select options={modes} value={options.mode ?? JoinMode.outer} onChange={onSetMode} />
</InlineField>
</InlineFieldRow>
<InlineFieldRow>
<InlineField label="Field" labelWidth={8} grow>
<Select
options={fieldNames}
value={options.byField}
onChange={onSelectField}
placeholder="time"
isClearable
/>
</InlineField>
</InlineFieldRow>
</>
);
}
export const joinByFieldTransformerRegistryItem: TransformerRegistryItem<JoinByFieldOptions> = {
id: DataTransformerID.joinByField,
aliasIds: [DataTransformerID.seriesToColumns],
editor: SeriesToFieldsTransformerEditor,
transformation: standardTransformers.joinByFieldTransformer,
name: standardTransformers.joinByFieldTransformer.name,
description: standardTransformers.joinByFieldTransformer.description,
};

@ -1,49 +0,0 @@
import React, { useCallback } from 'react';
import {
DataTransformerID,
SelectableValue,
standardTransformers,
TransformerRegistryItem,
TransformerUIProps,
} from '@grafana/data';
import { SeriesToColumnsOptions } from '@grafana/data/src/transformations/transformers/seriesToColumns';
import { Select } from '@grafana/ui';
import { useAllFieldNamesFromDataFrames } from '../utils';
export const SeriesToFieldsTransformerEditor: React.FC<TransformerUIProps<SeriesToColumnsOptions>> = ({
input,
options,
onChange,
}) => {
const fieldNames = useAllFieldNamesFromDataFrames(input).map((item: string) => ({ label: item, value: item }));
const onSelectField = useCallback(
(value: SelectableValue<string>) => {
onChange({
...options,
byField: value?.value,
});
},
[onChange, options]
);
return (
<div className="gf-form-inline">
<div className="gf-form gf-form--grow">
<div className="gf-form-label width-8">Field name</div>
<Select options={fieldNames} value={options.byField} onChange={onSelectField} isClearable />
</div>
</div>
);
};
export const seriesToFieldsTransformerRegistryItem: TransformerRegistryItem<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
editor: SeriesToFieldsTransformerEditor,
transformation: standardTransformers.seriesToColumnsTransformer,
name: 'Outer join',
description:
'Joins many time series/tables by a field. This can be used to outer join multiple time series on the _time_ field to show many time series in one table.',
};

@ -11,13 +11,13 @@ import { filterFramesByRefIdTransformRegistryItem } from './editors/FilterByRefI
import { groupByTransformRegistryItem } from './editors/GroupByTransformerEditor';
import { groupingToMatrixTransformRegistryItem } from './editors/GroupingToMatrixTransformerEditor';
import { histogramTransformRegistryItem } from './editors/HistogramTransformerEditor';
import { joinByFieldTransformerRegistryItem } from './editors/JoinByFieldTransformerEditor';
import { labelsToFieldsTransformerRegistryItem } from './editors/LabelsToFieldsTransformerEditor';
import { limitTransformRegistryItem } from './editors/LimitTransformerEditor';
import { mergeTransformerRegistryItem } from './editors/MergeTransformerEditor';
import { organizeFieldsTransformRegistryItem } from './editors/OrganizeFieldsTransformerEditor';
import { reduceTransformRegistryItem } from './editors/ReduceTransformerEditor';
import { renameByRegexTransformRegistryItem } from './editors/RenameByRegexTransformer';
import { seriesToFieldsTransformerRegistryItem } from './editors/SeriesToFieldsTransformerEditor';
import { seriesToRowsTransformerRegistryItem } from './editors/SeriesToRowsTransformerEditor';
import { sortByTransformRegistryItem } from './editors/SortByTransformerEditor';
import { extractFieldsTransformRegistryItem } from './extractFields/ExtractFieldsTransformerEditor';
@ -35,7 +35,7 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
filterFramesByRefIdTransformRegistryItem,
filterByValueTransformRegistryItem,
organizeFieldsTransformRegistryItem,
seriesToFieldsTransformerRegistryItem,
joinByFieldTransformerRegistryItem,
seriesToRowsTransformerRegistryItem,
concatenateTransformRegistryItem,
calculateFieldTransformRegistryItem,

Loading…
Cancel
Save