[v11.5.x] TransformationFilter: Include transformation outputs in transformation filtering options (#99878)

TransformationFilter: Include transformation outputs in transformation filtering options (#98323)

* wip: include transformation output as filtering option

* add refId to joinByField transformation

* clean up

* add refId to transformations that create new data frames

* adjust duplicate query removal for filtering options

* refactor transformation input/output subscription effect

* adjust input data frame filtering logic to include transformations as input for debug view

* transformation filter can only filter on output of previous transformation

(cherry picked from commit a32eed1d13)

Co-authored-by: Sergej-Vlasov <37613182+Sergej-Vlasov@users.noreply.github.com>

packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts

@@ -42,7 +42,10 @@ describe('ensureColumns transformer', () => {
       options: {},
     };
 
-    const data = [seriesA, seriesBC];
+    const data = [
+      { refId: 'A', ...seriesA },
+      { refId: 'B', ...seriesBC },
+    ];
 
     await expect(transformDataFrame([cfg], data)).toEmitValuesWith((received) => {
       const filtered = received[0];
@@ -109,6 +112,7 @@ describe('ensureColumns transformer', () => {
             },
           ],
           "length": 2,
+          "refId": "joinByField-A-B",
         }
       `);
     });

packages/grafana-data/src/transformations/transformers/histogram.ts

@@ -592,5 +592,6 @@ export function histogramFieldsToFrame(info: HistogramFields, theme?: GrafanaThe
       type: DataFrameType.Histogram,
     },
     fields: [info.xMin, info.xMax, ...info.counts],
+    refId: `${DataTransformerID.histogram}`,
   };
 }

packages/grafana-data/src/transformations/transformers/joinByField.ts

@@ -42,6 +42,7 @@ export const joinByFieldTransformer: SynchronousDataTransformerInfo<JoinByFieldO
         }
         const joined = joinDataFrames({ frames: data, joinBy, mode: options.mode });
         if (joined) {
+          joined.refId = `${DataTransformerID.joinByField}-${data.map((frame) => frame.refId).join('-')}`;
           return [joined];
         }
       }

packages/grafana-data/src/transformations/transformers/merge.ts

@@ -43,7 +43,10 @@ export const mergeTransformer: DataTransformerInfo<MergeTransformerOptions> = {
         const fieldNames = new Set<string>();
         const fieldIndexByName: Record<string, Record<number, number>> = {};
         const fieldNamesForKey: string[] = [];
-        const dataFrame = new MutableDataFrame();
+        const dataFrame = new MutableDataFrame({
+          refId: `${DataTransformerID.merge}-${data.map((frame) => frame.refId).join('-')}`,
+          fields: [],
+        });
 
         for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
           const frame = data[frameIndex];

packages/grafana-data/src/transformations/transformers/reduce.ts

@@ -56,7 +56,9 @@ export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> =
 
         // Add a row for each series
         const res = reduceSeriesToRows(data, matcher, options.reducers, options.labelsToFields);
-        return res ? [res] : [];
+        return res
+          ? [{ ...res, refId: `${DataTransformerID.reduce}-${data.map((frame) => frame.refId).join('-')}` }]
+          : [];
       })
     ),
 };

packages/grafana-data/src/transformations/transformers/seriesToRows.ts

@@ -37,7 +37,10 @@ export const seriesToRowsTransformer: DataTransformerInfo<SeriesToRowsTransforme
 
         const timeFieldByIndex: Record<number, number> = {};
         const targetFields = new Set<string>();
-        const dataFrame = new MutableDataFrame();
+        const dataFrame = new MutableDataFrame({
+          refId: `${DataTransformerID.seriesToRows}-${data.map((frame) => frame.refId).join('-')}`,
+          fields: [],
+        });
         const metricField: Field = {
           name: TIME_SERIES_METRIC_FIELD_NAME,
           values: [],

packages/grafana-data/src/transformations/transformers/transpose.ts

@@ -80,6 +80,7 @@ function transposeDataFrame(options: TransposeTransformerOptions, data: DataFram
       ...frame,
       fields: newFields,
       length: Math.max(...newFields.map((field) => field.values.length)),
+      refId: `${DataTransformerID.transpose}-${frame.refId}`,
     };
   });
 }

public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx

@@ -1,26 +1,17 @@
 import { css } from '@emotion/css';
-import { createElement, useEffect, useMemo, useState } from 'react';
-import { mergeMap } from 'rxjs/operators';
+import { createElement, useMemo } from 'react';
 
-import {
-  DataFrame,
-  DataTransformerConfig,
-  GrafanaTheme2,
-  transformDataFrame,
-  TransformerRegistryItem,
-  getFrameMatchers,
-  DataTransformContext,
-} from '@grafana/data';
+import { DataFrame, DataTransformerConfig, GrafanaTheme2, TransformerRegistryItem } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
-import { getTemplateSrv } from '@grafana/runtime';
 import { Icon, JSONFormatter, useStyles2, Drawer } from '@grafana/ui';
 
 import { TransformationsEditorTransformation } from './types';
 
 interface TransformationEditorProps {
+  input: DataFrame[];
+  output: DataFrame[];
   debugMode?: boolean;
   index: number;
-  data: DataFrame[];
   uiConfig: TransformerRegistryItem;
   configs: TransformationsEditorTransformation[];
   onChange: (index: number, config: DataTransformerConfig) => void;
@@ -28,45 +19,18 @@ interface TransformationEditorProps {
 }
 
 export const TransformationEditor = ({
+  input,
+  output,
   debugMode,
   index,
-  data,
   uiConfig,
   configs,
   onChange,
   toggleShowDebug,
 }: TransformationEditorProps) => {
   const styles = useStyles2(getStyles);
-  const [input, setInput] = useState<DataFrame[]>([]);
-  const [output, setOutput] = useState<DataFrame[]>([]);
   const config = useMemo(() => configs[index], [configs, index]);
 
-  useEffect(() => {
-    const config = configs[index].transformation;
-    const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
-    const inputTransforms = configs.slice(0, index).map((t) => t.transformation);
-    const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation);
-
-    const ctx: DataTransformContext = {
-      interpolate: (v: string) => getTemplateSrv().replace(v),
-    };
-
-    const inputSubscription = transformDataFrame(inputTransforms, data, ctx).subscribe((v) => {
-      if (matcher) {
-        v = data.filter((v) => matcher(v));
-      }
-      setInput(v);
-    });
-    const outputSubscription = transformDataFrame(inputTransforms, data, ctx)
-      .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before, ctx)))
-      .subscribe(setOutput);
-
-    return function unsubscribe() {
-      inputSubscription.unsubscribe();
-      outputSubscription.unsubscribe();
-    };
-  }, [index, data, configs]);
-
   const editor = useMemo(
     () =>
       createElement(uiConfig.editor, {

public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx

@@ -1,40 +1,35 @@
 import { css } from '@emotion/css';
 import { useMemo } from 'react';
 
-import {
-  DataTransformerConfig,
-  GrafanaTheme2,
-  StandardEditorContext,
-  StandardEditorsRegistryItem,
-} from '@grafana/data';
+import { DataFrame, DataTransformerConfig, GrafanaTheme2 } from '@grafana/data';
 import { DataTopic } from '@grafana/schema';
 import { Field, Select, useStyles2 } from '@grafana/ui';
 import { FrameMultiSelectionEditor } from 'app/plugins/panel/geomap/editor/FrameSelectionEditor';
 
-import { TransformationData } from './TransformationsEditor';
-
 interface TransformationFilterProps {
+  /** data frames from the output of previous transformation */
+  data: DataFrame[];
   index: number;
   config: DataTransformerConfig;
-  data: TransformationData;
+  annotations?: DataFrame[];
   onChange: (index: number, config: DataTransformerConfig) => void;
 }
 
-export const TransformationFilter = ({ index, data, config, onChange }: TransformationFilterProps) => {
+export const TransformationFilter = ({ index, annotations, config, onChange, data }: TransformationFilterProps) => {
   const styles = useStyles2(getStyles);
 
   const opts = useMemo(() => {
     return {
       // eslint-disable-next-line
-      context: { data: data.series } as StandardEditorContext<unknown>,
-      showTopic: true || data.annotations?.length || config.topic?.length,
+      context: { data },
+      showTopic: true || annotations?.length || config.topic?.length,
       showFilter: config.topic !== DataTopic.Annotations,
       source: [
-        { value: DataTopic.Series, label: `Query results` },
+        { value: DataTopic.Series, label: `Query and Transformation results` },
         { value: DataTopic.Annotations, label: `Annotation data` },
       ],
     };
-  }, [data, config.topic]);
+  }, [data, annotations?.length, config.topic]);
 
   return (
     <div className={styles.wrapper}>
@@ -59,8 +54,6 @@ export const TransformationFilter = ({ index, data, config, onChange }: Transfor
             <FrameMultiSelectionEditor
               value={config.filter!}
               context={opts.context}
-              // eslint-disable-next-line
-              item={{} as StandardEditorsRegistryItem}
               onChange={(filter) => onChange(index, { ...config, filter })}
             />
           )}

public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx

@@ -1,10 +1,18 @@
-import { useCallback } from 'react';
-import * as React from 'react';
+import { useCallback, useEffect, useState } from 'react';
 import { useToggle } from 'react-use';
+import { mergeMap } from 'rxjs';
 
-import { DataTransformerConfig, TransformerRegistryItem, FrameMatcherID, DataTopic } from '@grafana/data';
+import {
+  DataTransformerConfig,
+  TransformerRegistryItem,
+  FrameMatcherID,
+  DataTransformContext,
+  getFrameMatchers,
+  transformDataFrame,
+  DataFrame,
+} from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
-import { reportInteraction } from '@grafana/runtime';
+import { getTemplateSrv, reportInteraction } from '@grafana/runtime';
 import { ConfirmModal } from '@grafana/ui';
 import {
   QueryOperationAction,
@@ -46,6 +54,10 @@ export const TransformationOperationRow = ({
   const topic = configs[index].transformation.topic;
   const showFilterEditor = configs[index].transformation.filter != null || topic != null;
   const showFilterToggle = showFilterEditor || data.series.length > 0 || (data.annotations?.length ?? 0) > 0;
+  const [input, setInput] = useState<DataFrame[]>([]);
+  const [output, setOutput] = useState<DataFrame[]>([]);
+  // output of previous transformation
+  const [prevOutput, setPrevOutput] = useState<DataFrame[]>([]);
 
   const onDisableToggle = useCallback(
     (index: number) => {
@@ -92,6 +104,48 @@ export const TransformationOperationRow = ({
     [configs, index]
   );
 
+  useEffect(() => {
+    const config = configs[index].transformation;
+    const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
+    // we need previous transformation index to get its outputs
+    //    to be used in this transforms inputs
+    const prevTransformIndex = index - 1;
+
+    let prevInputTransforms: Array<DataTransformerConfig<{}>> = [];
+    let prevOutputTransforms: Array<DataTransformerConfig<{}>> = [];
+
+    if (prevTransformIndex >= 0) {
+      prevInputTransforms = configs.slice(0, prevTransformIndex).map((t) => t.transformation);
+      prevOutputTransforms = configs.slice(prevTransformIndex, index).map((t) => t.transformation);
+    }
+
+    const inputTransforms = configs.slice(0, index).map((t) => t.transformation);
+    const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation);
+
+    const ctx: DataTransformContext = {
+      interpolate: (v: string) => getTemplateSrv().replace(v),
+    };
+
+    const inputSubscription = transformDataFrame(inputTransforms, data.series, ctx).subscribe((data) => {
+      if (matcher) {
+        data = data.filter((frame) => matcher(frame));
+      }
+      setInput(data);
+    });
+    const outputSubscription = transformDataFrame(inputTransforms, data.series, ctx)
+      .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before, ctx)))
+      .subscribe(setOutput);
+    const prevOutputSubscription = transformDataFrame(prevInputTransforms, data.series, ctx)
+      .pipe(mergeMap((before) => transformDataFrame(prevOutputTransforms, before, ctx)))
+      .subscribe(setPrevOutput);
+
+    return function unsubscribe() {
+      inputSubscription.unsubscribe();
+      outputSubscription.unsubscribe();
+      prevOutputSubscription.unsubscribe();
+    };
+  }, [index, data, configs]);
+
   const renderActions = () => {
     return (
       <>
@@ -162,13 +216,20 @@ export const TransformationOperationRow = ({
         }}
       >
         {showFilterEditor && (
-          <TransformationFilter index={index} config={configs[index].transformation} data={data} onChange={onChange} />
+          <TransformationFilter
+            data={prevOutput}
+            index={index}
+            config={configs[index].transformation}
+            annotations={data.annotations}
+            onChange={onChange}
+          />
         )}
 
         <TransformationEditor
+          input={input}
+          output={output}
           debugMode={showDebug}
           index={index}
-          data={topic === DataTopic.Annotations ? (data.annotations ?? []) : data.series}
           configs={configs}
           uiConfig={uiConfig}
           onChange={onChange}

public/app/features/transformers/joinByLabels/joinByLabels.ts

@@ -35,7 +35,7 @@ interface JoinValues {
 
 export function joinByLabels(options: JoinByLabelsTransformOptions, data: DataFrame[]): DataFrame {
   if (!options.value?.length) {
-    return getErrorFrame('No value labele configured');
+    return getErrorFrame('No value label configured');
   }
   const distinctLabels = getDistinctLabels(data);
   if (distinctLabels.size < 1) {
@@ -104,7 +104,11 @@ export function joinByLabels(options: JoinByLabelsTransformOptions, data: DataFr
     }
   }
 
-  const frame: DataFrame = { fields: [], length: nameValues[0].length };
+  const frame: DataFrame = {
+    fields: [],
+    length: nameValues[0].length,
+    refId: `${DataTransformerID.joinByLabels}-${data.map((frame) => frame.refId).join('-')}`,
+  };
   for (let i = 0; i < join.length; i++) {
     frame.fields.push({
       name: join[i],

public/app/features/transformers/rowsToFields/rowsToFields.test.ts

@@ -12,6 +12,7 @@ describe('Rows to fields', () => {
         { name: 'Miiin', type: FieldType.number, values: [3, 100] },
         { name: 'max', type: FieldType.string, values: [15, 200] },
       ],
+      refId: 'A',
     });
 
     const result = rowsToFields(
@@ -57,6 +58,7 @@ describe('Rows to fields', () => {
           },
         ],
         "length": 1,
+        "refId": "rowsToFields-A",
       }
     `);
   });

public/app/features/transformers/rowsToFields/rowsToFields.ts

@@ -64,6 +64,7 @@ export function rowsToFields(options: RowToFieldsTransformOptions, data: DataFra
   return {
     fields: outFields,
     length: 1,
+    refId: `${DataTransformerID.rowsToFields}-${data.refId}`,
   };
 }
 

public/app/plugins/panel/geomap/editor/FrameSelectionEditor.tsx

@@ -29,7 +29,9 @@ export const FrameSelectionEditor = ({ value, context, onChange }: Props) => {
   );
 };
 
-export const FrameMultiSelectionEditor = ({ value, context, onChange }: Props) => {
+type FrameMultiSelectionEditorProps = Omit<StandardEditorProps<MatcherConfig>, 'item'>;
+
+export const FrameMultiSelectionEditor = ({ value, context, onChange }: FrameMultiSelectionEditorProps) => {
   const onFilterChange = useCallback(
     (v: string[]) => {
       onChange(