DataFrame: expose an object array as a data frame (#23494)

packages/grafana-data/src/dataframe/ArrayDataFrame.test.ts

@@ -0,0 +1,95 @@
+import { ArrayDataFrame } from './ArrayDataFrame';
+import { toDataFrameDTO } from './processDataFrame';
+import { FieldType } from '../types';
+
+describe('Array DataFrame', () => {
+  const input = [
+    { name: 'first', value: 1, time: 123 },
+    { name: 'second', value: 2, time: 456, extra: 'here' },
+    { name: 'third', value: 3, time: 789 },
+  ];
+
+  const frame = new ArrayDataFrame(input);
+  frame.name = 'Hello';
+  frame.refId = 'Z';
+  frame.setFieldType('phantom', FieldType.string, v => '🦥');
+  const field = frame.fields.find(f => f.name == 'value');
+  field!.config.unit = 'kwh';
+
+  test('Should support functional methods', () => {
+    const expectedNames = input.map(row => row.name);
+
+    // Check map
+    expect(frame.map(row => row.name)).toEqual(expectedNames);
+
+    let names: string[] = [];
+    for (const row of frame) {
+      names.push(row.name);
+    }
+    expect(names).toEqual(expectedNames);
+
+    names = [];
+    frame.forEach(row => {
+      names.push(row.name);
+    });
+    expect(names).toEqual(expectedNames);
+  });
+
+  test('Should convert an array of objects to a dataframe', () => {
+    expect(toDataFrameDTO(frame)).toMatchInlineSnapshot(`
+      Object {
+        "fields": Array [
+          Object {
+            "config": Object {},
+            "labels": undefined,
+            "name": "name",
+            "type": "string",
+            "values": Array [
+              "first",
+              "second",
+              "third",
+            ],
+          },
+          Object {
+            "config": Object {
+              "unit": "kwh",
+            },
+            "labels": undefined,
+            "name": "value",
+            "type": "number",
+            "values": Array [
+              1,
+              2,
+              3,
+            ],
+          },
+          Object {
+            "config": Object {},
+            "labels": undefined,
+            "name": "time",
+            "type": "time",
+            "values": Array [
+              123,
+              456,
+              789,
+            ],
+          },
+          Object {
+            "config": Object {},
+            "labels": undefined,
+            "name": "phantom",
+            "type": "string",
+            "values": Array [
+              "🦥",
+              "🦥",
+              "🦥",
+            ],
+          },
+        ],
+        "meta": undefined,
+        "name": "Hello",
+        "refId": "Z",
+      }
+    `);
+  });
+});

packages/grafana-data/src/dataframe/ArrayDataFrame.ts

@@ -0,0 +1,119 @@
+import { Field, FieldType, DataFrame } from '../types/dataFrame';
+import { vectorToArray } from '../vector/vectorToArray';
+import { Vector, QueryResultMeta } from '../types';
+import { guessFieldTypeFromNameAndValue, toDataFrameDTO } from './processDataFrame';
+import { FunctionalVector } from '../vector/FunctionalVector';
+
+export type ValueConverter<T = any> = (val: any) => T;
+
+const NOOP: ValueConverter = v => v;
+
+class ArrayPropertyVector<T = any> implements Vector<T> {
+  converter = NOOP;
+
+  constructor(private source: any[], private prop: string) {}
+
+  get length(): number {
+    return this.source.length;
+  }
+
+  get(index: number): T {
+    return this.converter(this.source[index][this.prop]);
+  }
+
+  toArray(): T[] {
+    return vectorToArray(this);
+  }
+
+  toJSON(): T[] {
+    return vectorToArray(this);
+  }
+}
+
+/**
+ * The ArrayDataFrame takes an array of objects and presents it as a DataFrame
+ *
+ * @alpha
+ */
+export class ArrayDataFrame<T = any> extends FunctionalVector<T> implements DataFrame {
+  name?: string;
+  refId?: string;
+  meta?: QueryResultMeta;
+
+  private theFields: Field[] = [];
+
+  constructor(private source: T[], names?: string[]) {
+    super();
+
+    const first: any = source.length ? source[0] : {};
+    if (names) {
+      this.theFields = names.map(name => {
+        return {
+          name,
+          type: guessFieldTypeFromNameAndValue(name, first[name]),
+          config: {},
+          values: new ArrayPropertyVector(source, name),
+        };
+      });
+    } else {
+      this.setFieldsFromObject(first);
+    }
+  }
+
+  /**
+   * Add a field for each property in the object.  This will guess the type
+   */
+  setFieldsFromObject(obj: any) {
+    this.theFields = Object.keys(obj).map(name => {
+      return {
+        name,
+        type: guessFieldTypeFromNameAndValue(name, obj[name]),
+        config: {},
+        values: new ArrayPropertyVector(this.source, name),
+      };
+    });
+  }
+
+  /**
+   * Configure how the object property is passed to the data frame
+   */
+  setFieldType(name: string, type: FieldType, converter?: ValueConverter): Field {
+    let field = this.fields.find(f => f.name === name);
+    if (field) {
+      field.type = type;
+    } else {
+      field = {
+        name,
+        type,
+        config: {},
+        values: new ArrayPropertyVector(this.source, name),
+      };
+      this.fields.push(field);
+    }
+    (field.values as any).converter = converter ?? NOOP;
+    return field;
+  }
+
+  get fields(): Field[] {
+    return this.theFields;
+  }
+
+  // Defined for Vector interface
+  get length() {
+    return this.source.length;
+  }
+
+  /**
+   * Get an object with a property for each field in the DataFrame
+   */
+  get(idx: number): T {
+    return this.source[idx];
+  }
+
+  /**
+   * The simplified JSON values used in JSON.stringify()
+   */
+  toJSON() {
+    return toDataFrameDTO(this);
+  }
+}

packages/grafana-data/src/dataframe/DataFrameView.ts

@@ -1,6 +1,6 @@
-import { Vector } from '../types/vector';
 import { DataFrame } from '../types/dataFrame';
 import { DisplayProcessor } from '../types';
+import { FunctionalVector } from '../vector/FunctionalVector';
 
 /**
  * This abstraction will present the contents of a DataFrame as if
@@ -13,11 +13,12 @@ import { DisplayProcessor } from '../types';
  * @typeParam T - Type of object stored in the DataFrame.
  * @beta
  */
-export class DataFrameView<T = any> implements Vector<T> {
+export class DataFrameView<T = any> extends FunctionalVector<T> {
   private index = 0;
   private obj: T;
 
   constructor(private data: DataFrame) {
+    super();
     const obj = ({} as unknown) as T;
 
     for (let i = 0; i < data.fields.length; i++) {
@@ -91,24 +92,8 @@ export class DataFrameView<T = any> implements Vector<T> {
   }
 
   toArray(): T[] {
-    return new Array(this.data.length).fill(0).map((_, i) => ({ ...this.get(i) }));
+    return new Array(this.data.length)
-  }
+      .fill(0) // Needs to make a full copy
-
+      .map((_, i) => ({ ...this.get(i) }));
-  toJSON(): T[] {
-    return this.toArray();
-  }
-
-  forEachRow(iterator: (row: T) => void) {
-    for (let i = 0; i < this.data.length; i++) {
-      iterator(this.get(i));
-    }
-  }
-
-  map<V>(iterator: (item: T, index: number) => V) {
-    const acc: V[] = [];
-    for (let i = 0; i < this.data.length; i++) {
-      acc.push(iterator(this.get(i), i));
-    }
-    return acc;
   }
 }

packages/grafana-data/src/dataframe/MutableDataFrame.ts

@@ -6,7 +6,7 @@ import isString from 'lodash/isString';
 import { makeFieldParser } from '../utils/fieldParser';
 import { MutableVector, Vector } from '../types/vector';
 import { ArrayVector } from '../vector/ArrayVector';
-import { vectorToArray } from '../vector/vectorToArray';
+import { FunctionalVector } from '../vector/FunctionalVector';
 
 export type MutableField<T = any> = Field<T, MutableVector<T>>;
 
@@ -14,7 +14,7 @@ type MutableVectorCreator = (buffer?: any[]) => MutableVector;
 
 export const MISSING_VALUE: any = null;
 
-export class MutableDataFrame<T = any> implements DataFrame, MutableVector<T> {
+export class MutableDataFrame<T = any> extends FunctionalVector<T> implements DataFrame, MutableVector<T> {
   name?: string;
   refId?: string;
   meta?: QueryResultMeta;
@@ -26,6 +26,8 @@ export class MutableDataFrame<T = any> implements DataFrame, MutableVector<T> {
   private creator: MutableVectorCreator;
 
   constructor(source?: DataFrame | DataFrameDTO, creator?: MutableVectorCreator) {
+    super();
+
     // This creates the underlying storage buffers
     this.creator = creator
       ? creator
@@ -267,10 +269,6 @@ export class MutableDataFrame<T = any> implements DataFrame, MutableVector<T> {
     return v as T;
   }
 
-  toArray(): T[] {
-    return vectorToArray(this);
-  }
-
   /**
    * The simplified JSON values used in JSON.stringify()
    */

packages/grafana-data/src/dataframe/index.ts

@@ -5,3 +5,4 @@ export * from './MutableDataFrame';
 export * from './processDataFrame';
 export * from './dimensions';
 export * from './ArrowDataFrame';
+export * from './ArrayDataFrame';

packages/grafana-data/src/dataframe/processDataFrame.ts

@@ -159,6 +159,19 @@ function convertJSONDocumentDataToDataFrame(timeSeries: TimeSeries): DataFrame {
 // https://github.com/mholt/PapaParse/blob/master/papaparse.js#L998
 const NUMBER = /^\s*(-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?|NAN)\s*$/i;
 
+/**
+ * Given a name and value, this will pick a reasonable field type
+ */
+export function guessFieldTypeFromNameAndValue(name: string, v: any): FieldType {
+  if (name) {
+    name = name.toLowerCase();
+    if (name === 'date' || name === 'time') {
+      return FieldType.time;
+    }
+  }
+  return guessFieldTypeFromValue(v);
+}
+
 /**
  * Given a value this will guess the best column type
  *

packages/grafana-data/src/vector/ArrayVector.ts

@@ -1,9 +1,11 @@
 import { MutableVector } from '../types/vector';
+import { FunctionalVector } from './FunctionalVector';
 
-export class ArrayVector<T = any> implements MutableVector<T> {
+export class ArrayVector<T = any> extends FunctionalVector<T> implements MutableVector<T> {
   buffer: T[];
 
   constructor(buffer?: T[]) {
+    super();
     this.buffer = buffer ? buffer : [];
   }
 

packages/grafana-data/src/vector/CircularVector.ts

@@ -1,5 +1,6 @@
 import { MutableVector } from '../types/vector';
 import { vectorToArray } from './vectorToArray';
+import { FunctionalVector } from './FunctionalVector';
 
 interface CircularOptions<T> {
   buffer?: T[];
@@ -14,13 +15,15 @@ interface CircularOptions<T> {
  * This supports addting to the 'head' or 'tail' and will grow the buffer
  * to match a configured capacity.
  */
-export class CircularVector<T = any> implements MutableVector<T> {
+export class CircularVector<T = any> extends FunctionalVector implements MutableVector<T> {
   private buffer: T[];
   private index: number;
   private capacity: number;
   private tail: boolean;
 
   constructor(options: CircularOptions<T>) {
+    super();
+
     this.buffer = options.buffer || [];
     this.capacity = this.buffer.length;
     this.tail = 'head' !== options.append;

packages/grafana-data/src/vector/FunctionalVector.ts

@@ -0,0 +1,77 @@
+import { vectorToArray } from './vectorToArray';
+import { Vector } from '../types';
+
+export abstract class FunctionalVector<T = any> implements Vector<T>, Iterable<T> {
+  abstract get length(): number;
+
+  abstract get(index: number): T;
+
+  // Implement "iterator protocol"
+  *iterator() {
+    for (let i = 0; i < this.length; i++) {
+      yield this.get(i);
+    }
+  }
+
+  // Implement "iterable protocol"
+  [Symbol.iterator]() {
+    return this.iterator();
+  }
+
+  forEach(iterator: (row: T) => void) {
+    return vectorator(this).forEach(iterator);
+  }
+
+  map<V>(transform: (item: T, index: number) => V) {
+    return vectorator(this).map(transform);
+  }
+
+  filter<V>(predicate: (item: T) => V) {
+    return vectorator(this).filter(predicate);
+  }
+
+  toArray(): T[] {
+    return vectorToArray(this);
+  }
+
+  toJSON(): any {
+    return this.toArray();
+  }
+}
+
+/**
+ * Use functional programming with your vector
+ */
+export function vectorator<T>(vector: Vector<T>) {
+  return {
+    *[Symbol.iterator]() {
+      for (let i = 0; i < vector.length; i++) {
+        yield vector.get(i);
+      }
+    },
+
+    forEach(iterator: (row: T) => void) {
+      for (let i = 0; i < vector.length; i++) {
+        iterator(vector.get(i));
+      }
+    },
+
+    map<V>(transform: (item: T, index: number) => V) {
+      const result: V[] = [];
+      for (let i = 0; i < vector.length; i++) {
+        result.push(transform(vector.get(i), i));
+      }
+      return result;
+    },
+
+    filter<V>(predicate: (item: T) => V) {
+      const result: T[] = [];
+      for (const val of this) {
+        if (predicate(val)) {
+          result.push(val);
+        }
+      }
+      return result;
+    },
+  };
+}

packages/grafana-data/src/vector/index.ts

@@ -4,3 +4,5 @@ export * from './CircularVector';
 export * from './ConstantVector';
 export * from './ScaledVector';
 export * from './SortedVector';
+
+export { vectorator } from './FunctionalVector';

public/app/plugins/datasource/loki/datasource.ts

@@ -646,7 +646,7 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
       }
       const view = new DataFrameView<{ ts: string; line: string }>(frame);
 
-      view.forEachRow(row => {
+      view.forEach(row => {
         annotations.push({
           time: new Date(row.ts).valueOf(),
           text: row.line,

public/app/plugins/datasource/loki/result_transformer.ts

@@ -420,7 +420,7 @@ export const enhanceDataFrame = (dataFrame: DataFrame, config: LokiOptions | nul
   }, {} as Record<string, any>);
 
   const view = new DataFrameView(dataFrame);
-  view.forEachRow((row: { line: string }) => {
+  view.forEach((row: { line: string }) => {
     for (const field of derivedFields) {
       const logMatch = row.line.match(field.matcherRegex);
       fields[field.name].values.add(logMatch && logMatch[1]);