Timeseries to table transformation: Improve time series detection (#77841)

* Improve time series detection

* Prettier

* Add test

* Update packages/grafana-data/src/dataframe/utils.ts

Co-authored-by: Jev Forsberg <46619047+baldm0mma@users.noreply.github.com>

* Ensure correct time field support and set maximum size

* Look at each field to see if they are time series

* Add further tests

* Prettier

---------

Co-authored-by: Jev Forsberg <46619047+baldm0mma@users.noreply.github.com>

packages/grafana-data/src/dataframe/index.ts

@@ -7,5 +7,5 @@ export * from './dimensions';
 export * from './ArrayDataFrame';
 export * from './DataFrameJSON';
 export * from './frameComparisons';
-export { anySeriesWithTimeField, isTimeSeriesFrame, isTimeSeriesFrames } from './utils';
+export { anySeriesWithTimeField, isTimeSeriesFrame, isTimeSeriesFrames, isTimeSeriesField } from './utils';
 export { StreamingDataFrame, StreamingFrameAction, type StreamingFrameOptions, closestIdx } from './StreamingDataFrame';

packages/grafana-data/src/dataframe/utils.ts

@@ -1,24 +1,70 @@
-import { DataFrame, FieldType } from '../types/dataFrame';
+import { DataFrame, Field, FieldType } from '../types/dataFrame';
 
 import { getTimeField } from './processDataFrame';
 
+const MAX_TIME_COMPARISONS = 100;
+
 export function isTimeSeriesFrame(frame: DataFrame) {
   // If we have less than two frames we can't have a timeseries
   if (frame.fields.length < 2) {
     return false;
   }
 
-  // In order to have a time series we need a time field
-  // and at least one number field
-  const timeField = frame.fields.find((field) => field.type === FieldType.time);
+  // Find a number field, as long as we have any number field this should work
   const numberField = frame.fields.find((field) => field.type === FieldType.number);
-  return timeField !== undefined && numberField !== undefined;
+
+  // There are certain query types in which we will
+  // get times but they will be the same or not be
+  // in increasing order. To have a time-series the
+  // times need to be ordered from past to present
+  let timeFieldFound = false;
+  for (const field of frame.fields) {
+    if (isTimeSeriesField(field)) {
+      timeFieldFound = true;
+      break;
+    }
+  }
+
+  return timeFieldFound && numberField !== undefined;
 }
 
 export function isTimeSeriesFrames(data: DataFrame[]) {
   return !data.find((frame) => !isTimeSeriesFrame(frame));
 }
 
+/**
+ * Determines if a field is a time field in ascending
+ * order within the sampling range specified by
+ * MAX_TIME_COMPARISONS
+ *
+ * @param field
+ * @returns boolean
+ */
+export function isTimeSeriesField(field: Field) {
+  if (field.type !== FieldType.time) {
+    return false;
+  }
+
+  let greatestTime: number | null = null;
+  let testWindow = field.values.length > MAX_TIME_COMPARISONS ? MAX_TIME_COMPARISONS : field.values.length;
+
+  // Test up to the test window number of values
+  for (let i = 0; i < testWindow; i++) {
+    const time = field.values[i];
+
+    // Check to see if the current time is greater than
+    // the last time. If we get to the end then we
+    // have a time series otherwise we return false
+    if (greatestTime === null || (time !== null && time > greatestTime)) {
+      greatestTime = time;
+    } else {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 /**
  * Indicates if there is any time field in the array of data frames
  * @param data

public/app/features/transformers/timeSeriesTable/TimeSeriesTableTransformEditor.tsx

@@ -9,6 +9,7 @@ import {
   SelectableValue,
   Field,
   FieldType,
+  isTimeSeriesField,
 } from '@grafana/data';
 import { InlineFieldRow, InlineField, StatsPicker, Select, InlineLabel } from '@grafana/ui';
 
@@ -70,7 +71,7 @@ export function TimeSeriesTableTransformEditor({
     for (const frame of input) {
       if (frame.refId === refId) {
         for (const field of frame.fields) {
-          if (field.type === 'time') {
+          if (isTimeSeriesField(field)) {
             timeFields[field.name] = field;
           }
         }

public/app/features/transformers/timeSeriesTable/timeSeriesTableTransformer.test.ts

@@ -117,31 +117,105 @@ describe('timeSeriesTableTransformer', () => {
 
     expect(results[0].fields[2].values[0].value).toEqual(null);
   });
-});
 
-it('Will transform multiple data series with the same label', () => {
-  const series = [
-    getTimeSeries('A', { instance: 'A', pod: 'B' }, [4, 2, 3]),
-    getTimeSeries('B', { instance: 'A', pod: 'B' }, [3, 4, 5]),
-    getTimeSeries('C', { instance: 'A', pod: 'B' }, [3, 4, 5]),
-  ];
-
-  const results = timeSeriesToTableTransform({}, series);
-
-  // Check series A
-  expect(results[0].fields).toHaveLength(3);
-  expect(results[0].fields[0].values[0]).toBe('A');
-  expect(results[0].fields[1].values[0]).toBe('B');
-
-  // Check series B
-  expect(results[1].fields).toHaveLength(3);
-  expect(results[1].fields[0].values[0]).toBe('A');
-  expect(results[1].fields[1].values[0]).toBe('B');
-
-  // Check series C
-  expect(results[2].fields).toHaveLength(3);
-  expect(results[2].fields[0].values[0]).toBe('A');
-  expect(results[2].fields[1].values[0]).toBe('B');
+  it('Will transform multiple data series with the same label', () => {
+    const series = [
+      getTimeSeries('A', { instance: 'A', pod: 'B' }, [4, 2, 3]),
+      getTimeSeries('B', { instance: 'A', pod: 'B' }, [3, 4, 5]),
+      getTimeSeries('C', { instance: 'A', pod: 'B' }, [3, 4, 5]),
+    ];
+
+    const results = timeSeriesToTableTransform({}, series);
+
+    // Check series A
+    expect(results[0].fields).toHaveLength(3);
+    expect(results[0].fields[0].values[0]).toBe('A');
+    expect(results[0].fields[1].values[0]).toBe('B');
+
+    // Check series B
+    expect(results[1].fields).toHaveLength(3);
+    expect(results[1].fields[0].values[0]).toBe('A');
+    expect(results[1].fields[1].values[0]).toBe('B');
+
+    // Check series C
+    expect(results[2].fields).toHaveLength(3);
+    expect(results[2].fields[0].values[0]).toBe('A');
+    expect(results[2].fields[1].values[0]).toBe('B');
+  });
+
+  it('will not transform frames with time fields that are non-timeseries', () => {
+    const series = [
+      getTimeSeries('A', { instance: 'A', pod: 'B' }, [4, 2, 3]),
+      getNonTimeSeries('B', { instance: 'A', pod: 'B' }, [3, 4, 5]),
+    ];
+
+    const results = timeSeriesToTableTransform({}, series);
+
+    // Expect the timeseries to be transformed
+    // Having a trend field will show this
+    expect(results[1].fields[2].name).toBe('Trend #A');
+
+    // We should expect the field length to remain at
+    // 2 with a time field and a value field
+    expect(results[0].fields.length).toBe(2);
+  });
+
+  it('will not transform series that have the same value for all times', () => {
+    const series = [getNonTimeSeries('A', { instance: 'A' }, [4, 2, 5], [1699476339, 1699476339, 1699476339])];
+
+    const results = timeSeriesToTableTransform({}, series);
+
+    expect(results[0].fields[0].values[0]).toBe(1699476339);
+    expect(results[0].fields[1].values[0]).toBe(4);
+  });
+
+  it('will transform a series with two time fields', () => {
+    const frame = toDataFrame({
+      refId: 'A',
+      fields: [
+        { name: 'Time', type: FieldType.time, values: [0, 50, 90] },
+        { name: 'UpdateTime', type: FieldType.time, values: [10, 100, 100] },
+        {
+          name: 'Value',
+          type: FieldType.number,
+          values: [2, 3, 4],
+        },
+      ],
+    });
+
+    const results = timeSeriesToTableTransform({}, [frame]);
+
+    // We should have a created trend field
+    // with the first time field used as a time
+    // and the values coming along with that
+    expect(results[0].fields[0].name).toBe('Trend #A');
+    expect(results[0].fields[0].values[0].fields[0].values[0]).toBe(0);
+    expect(results[0].fields[0].values[0].fields[1].values[0]).toBe(2);
+  });
+
+  it('will transform a series with two time fields and a time field configured', () => {
+    const frame = toDataFrame({
+      refId: 'A',
+      fields: [
+        { name: 'Time', type: FieldType.time, values: [0, 50, 90] },
+        { name: 'UpdateTime', type: FieldType.time, values: [10, 100, 100] },
+        {
+          name: 'Value',
+          type: FieldType.number,
+          values: [2, 3, 4],
+        },
+      ],
+    });
+
+    const results = timeSeriesToTableTransform({ A: { timeField: 'UpdateTime' } }, [frame]);
+
+    // We should have a created trend field
+    // with the "UpdateTime" time field used as a time
+    // and the values coming along with that
+    expect(results[0].fields[0].name).toBe('Trend #A');
+    expect(results[0].fields[0].values[0].fields[0].values[0]).toBe(10);
+    expect(results[0].fields[0].values[0].fields[1].values[0]).toBe(2);
+  });
 });
 
 function assertFieldsEqual(field1: Field, field2: Field) {
@@ -176,6 +250,27 @@ function getTimeSeries(refId: string, labels: Labels, values: number[] = [10]) {
   });
 }
 
+function getNonTimeSeries(refId: string, labels: Labels, values: number[], times?: number[]) {
+  if (times === undefined) {
+    times = [1699476339, 1699475339, 1699476300];
+  }
+
+  return toDataFrame({
+    refId,
+    fields: [
+      // These times are in non-ascending order
+      // and thus this isn't a timeseries
+      { name: 'Time', type: FieldType.time, values: times },
+      {
+        name: 'Value',
+        type: FieldType.number,
+        values,
+        labels,
+      },
+    ],
+  });
+}
+
 function getTable(refId: string, fields: string[]) {
   return toDataFrame({
     refId,

public/app/features/transformers/timeSeriesTable/timeSeriesTableTransformer.ts

@@ -12,6 +12,7 @@ import {
   ReducerID,
   reduceField,
   TransformationApplicabilityLevels,
+  isTimeSeriesField,
 } from '@grafana/data';
 
 /**
@@ -145,6 +146,16 @@ export function timeSeriesToTableTransform(options: TimeSeriesTableTransformerOp
     for (let i = 0; i < framesForRef.length; i++) {
       const frame = framesForRef[i];
 
+      // Retrieve the time field that's been configured
+      // If one isn't configured then use the first found
+      let timeField = null;
+      let timeFieldName = options[refId]?.timeField;
+      if (timeFieldName && timeFieldName.length > 0) {
+        timeField = frame.fields.find((field) => field.name === timeFieldName);
+      } else {
+        timeField = frame.fields.find((field) => isTimeSeriesField(field));
+      }
+
       // If it's not a time series frame we add
       // it unmodified to the result
       if (!isTimeSeriesFrame(frame)) {
@@ -152,15 +163,6 @@ export function timeSeriesToTableTransform(options: TimeSeriesTableTransformerOp
         continue;
       }
 
-      // Retrieve the time field that's been configured
-      // If one isn't configured then use the first found
-      let timeField = null;
-      if (options[refId]?.timeField !== undefined) {
-        timeField = frame.fields.find((field) => field.name === options[refId]?.timeField);
-      } else {
-        timeField = frame.fields.find((field) => field.type === FieldType.time);
-      }
-
       for (const field of frame.fields) {
         // Skip non-number based fields
         // i.e. we skip time, strings, etc.