From 0ad2242fb89ee74e31430dd541871f02d5f39275 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Tue, 8 Oct 2019 09:33:33 +0200
Subject: [PATCH] Table: Proper handling of json data with dataframes (#19596)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When using Raw Document query with Elasticsearch there's a special
response from datasource that is used which includes a type field with
the value json. In the table panel there is a transformation for JSON
data which up until this fix didn't work at all due to the new data
structure we call data frames.

Co-Authored-By: Hugo Häggmark <hugo.haggmark@grafana.com>

Fixes #19531
---
 .../src/dataframe/processDataFrame.test.ts    | 63 +++++++++++++++++++
 .../src/dataframe/processDataFrame.ts         | 42 +++++++++++++
 2 files changed, 105 insertions(+)

diff --git a/packages/grafana-data/src/dataframe/processDataFrame.test.ts b/packages/grafana-data/src/dataframe/processDataFrame.test.ts
index b44a3bf8c72..47d3780b238 100644
--- a/packages/grafana-data/src/dataframe/processDataFrame.test.ts
+++ b/packages/grafana-data/src/dataframe/processDataFrame.test.ts
@@ -103,6 +103,36 @@ describe('toDataFrame', () => {
     expect(norm.fields[2].type).toBe(FieldType.other);
     expect(norm.fields[3].type).toBe(FieldType.time); // based on name
   });
+
+  it('converts JSON document data to series', () => {
+    const input1 = {
+      datapoints: [
+        {
+          _id: 'W5rvjW0BKe0cA-E1aHvr',
+          _type: '_doc',
+          _index: 'logs-2019.10.02',
+          '@message': 'Deployed website',
+          '@timestamp': [1570044340458],
+          tags: ['deploy', 'website-01'],
+          description: 'Torkel deployed website',
+          coordinates: { latitude: 12, longitude: 121, level: { depth: 3, coolnes: 'very' } },
+          long:
+            'asdsaa asdas dasdas dasdasdas asdaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa asdasdasdasdasdasdas asd',
+          'unescaped-content': 'breaking <br /> the <br /> row',
+        },
+      ],
+      filterable: true,
+      target: 'docs',
+      total: 206,
+      type: 'docs',
+    };
+    const dataFrame = toDataFrame(input1);
+    expect(dataFrame.fields[0].name).toBe(input1.target);
+
+    const v0 = dataFrame.fields[0].values;
+    expect(v0.length).toEqual(1);
+    expect(v0.get(0)).toEqual(input1.datapoints[0]);
+  });
 });
 
 describe('SerisData backwards compatibility', () => {
@@ -178,6 +208,39 @@ describe('SerisData backwards compatibility', () => {
     const names = table.columns.map(c => c.text);
     expect(names).toEqual(['T', 'N', 'S']);
   });
+
+  it('can convert TimeSeries to JSON document and back again', () => {
+    const timeseries = {
+      datapoints: [
+        {
+          _id: 'W5rvjW0BKe0cA-E1aHvr',
+          _type: '_doc',
+          _index: 'logs-2019.10.02',
+          '@message': 'Deployed website',
+          '@timestamp': [1570044340458],
+          tags: ['deploy', 'website-01'],
+          description: 'Torkel deployed website',
+          coordinates: { latitude: 12, longitude: 121, level: { depth: 3, coolnes: 'very' } },
+          long:
+            'asdsaa asdas dasdas dasdasdas asdaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa asdasdasdasdasdasdas asd',
+          'unescaped-content': 'breaking <br /> the <br /> row',
+        },
+      ],
+      filterable: true,
+      target: 'docs',
+      total: 206,
+      type: 'docs',
+    };
+    const series = toDataFrame(timeseries);
+    expect(isDataFrame(timeseries)).toBeFalsy();
+    expect(isDataFrame(series)).toBeTruthy();
+
+    const roundtrip = toLegacyResponseData(series) as any;
+    expect(isDataFrame(roundtrip)).toBeFalsy();
+    expect(roundtrip.type).toBe('docs');
+    expect(roundtrip.target).toBe('docs');
+    expect(roundtrip.filterable).toBeTruthy();
+  });
 });
 
 describe('sorted DataFrame', () => {
diff --git a/packages/grafana-data/src/dataframe/processDataFrame.ts b/packages/grafana-data/src/dataframe/processDataFrame.ts
index 04e61694d47..5123be79c17 100644
--- a/packages/grafana-data/src/dataframe/processDataFrame.ts
+++ b/packages/grafana-data/src/dataframe/processDataFrame.ts
@@ -127,6 +127,33 @@ function convertGraphSeriesToDataFrame(graphSeries: GraphSeriesXY): DataFrame {
   };
 }
 
+function convertJSONDocumentDataToDataFrame(timeSeries: TimeSeries): DataFrame {
+  const fields = [
+    {
+      name: timeSeries.target,
+      type: FieldType.other,
+      config: {
+        unit: timeSeries.unit,
+        filterable: (timeSeries as any).filterable,
+      },
+      values: new ArrayVector(),
+    },
+  ];
+
+  for (const point of timeSeries.datapoints) {
+    fields[0].values.buffer.push(point);
+  }
+
+  return {
+    name: timeSeries.target,
+    labels: timeSeries.tags,
+    refId: timeSeries.target,
+    meta: { json: true },
+    fields,
+    length: timeSeries.datapoints.length,
+  };
+}
+
 // PapaParse Dynamic Typing regex:
 // https://github.com/mholt/PapaParse/blob/master/papaparse.js#L998
 const NUMBER = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
@@ -241,6 +268,11 @@ export const toDataFrame = (data: any): DataFrame => {
     return new MutableDataFrame(data as DataFrameDTO);
   }
 
+  // Handle legacy docs/json type
+  if (data.hasOwnProperty('type') && data.type === 'docs') {
+    return convertJSONDocumentDataToDataFrame(data);
+  }
+
   if (data.hasOwnProperty('datapoints')) {
     return convertTimeSeriesToDataFrame(data);
   }
@@ -288,6 +320,16 @@ export const toLegacyResponseData = (frame: DataFrame): TimeSeries | TableData =
     }
   }
 
+  if (frame.meta && frame.meta.json) {
+    return {
+      alias: fields[0].name || frame.name,
+      target: fields[0].name || frame.name,
+      datapoints: fields[0].values.toArray(),
+      filterable: fields[0].config ? fields[0].config.filterable : undefined,
+      type: 'docs',
+    } as TimeSeries;
+  }
+
   return {
     columns: fields.map(f => {
       const { name, config } = f;