Elasticsearch: Fix passing of limit and datalinks to logs data frame (#68554)

* Elasticsearch: Fix passing of limit and datalinks to logs data frame

* Update public/app/core/logsModel.ts

Co-authored-by: François Massot <francois.massot@gmail.com>

---------

Co-authored-by: François Massot <francois.massot@gmail.com>

pkg/tsdb/elasticsearch/response_parser.go

@@ -160,7 +160,7 @@ func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields
 	frames := data.Frames{}
 	frame := data.NewFrame("", fields...)
 	setPreferredVisType(frame, data.VisTypeLogs)
-	setSearchWords(frame, searchWords)
+	setLogsCustomMeta(frame, searchWords, stringToIntWithDefaultValue(target.Metrics[0].Settings.Get("limit").MustString(), defaultSize))
 	frames = append(frames, frame)
 
 	queryRes.Frames = frames
@@ -1137,7 +1137,7 @@ func setPreferredVisType(frame *data.Frame, visType data.VisType) {
 	frame.Meta.PreferredVisualization = visType
 }
 
-func setSearchWords(frame *data.Frame, searchWords map[string]bool) {
+func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int) {
 	i := 0
 	searchWordsList := make([]string, len(searchWords))
 	for searchWord := range searchWords {
@@ -1156,6 +1156,7 @@ func setSearchWords(frame *data.Frame, searchWords map[string]bool) {
 
 	frame.Meta.Custom = map[string]interface{}{
 		"searchWords": searchWordsList,
+		"limit":       limit,
 	}
 }
 

pkg/tsdb/elasticsearch/response_parser_test.go

@@ -105,7 +105,7 @@ func TestProcessLogsResponse(t *testing.T) {
 			logsFrame := frames[0]
 
 			meta := logsFrame.Meta
-			require.Equal(t, map[string]interface{}{"searchWords": []string{"hello", "message"}}, meta.Custom)
+			require.Equal(t, map[string]interface{}{"searchWords": []string{"hello", "message"}, "limit": 500}, meta.Custom)
 			require.Equal(t, data.VisTypeLogs, string(meta.PreferredVisualization))
 
 			logsFieldMap := make(map[string]*data.Field)
@@ -430,6 +430,7 @@ func TestProcessLogsResponse(t *testing.T) {
 
 		require.Equal(t, map[string]interface{}{
 			"searchWords": []string{"hello", "message"},
+			"limit":       500,
 		}, customMeta)
 	})
 }

pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc

@@ -6,6 +6,7 @@
 //          0
 //      ],
 //      "custom": {
+//          "limit": 500,
 //          "searchWords": [
 //              "hello",
 //              "message"
@@ -40,6 +41,7 @@
             0
           ],
           "custom": {
+            "limit": 500,
             "searchWords": [
               "hello",
               "message"

public/app/core/logsModel.test.ts

@@ -354,6 +354,48 @@ describe('dataFrameToLogsModel', () => {
     });
   });
 
+  it('given one series with limit as custom meta property should return correct limit', () => {
+    const series: DataFrame[] = [
+      new MutableDataFrame({
+        fields: [
+          {
+            name: 'time',
+            type: FieldType.time,
+            values: ['2019-04-26T09:28:11.352440161Z', '2019-04-26T14:42:50.991981292Z'],
+          },
+          {
+            name: 'message',
+            type: FieldType.string,
+            values: [
+              't=2019-04-26T11:05:28+0200 lvl=info msg="Initializing DatasourceCacheService" logger=server',
+              't=2019-04-26T16:42:50+0200 lvl=eror msg="new token…t unhashed token=56d9fdc5c8b7400bd51b060eea8ca9d7',
+            ],
+            labels: {
+              filename: '/var/log/grafana/grafana.log',
+              job: 'grafana',
+            },
+          },
+          {
+            name: 'id',
+            type: FieldType.string,
+            values: ['foo', 'bar'],
+          },
+        ],
+        meta: {
+          custom: {
+            limit: 1000,
+          },
+        },
+      }),
+    ];
+    const logsModel = dataFrameToLogsModel(series, 1);
+    expect(logsModel.meta![1]).toMatchObject({
+      label: LIMIT_LABEL,
+      value: `1000 (2 returned)`,
+      kind: LogsMetaKind.String,
+    });
+  });
+
   it('given one series with labels-field should return expected logs model', () => {
     const series: DataFrame[] = [
       new MutableDataFrame({

public/app/core/logsModel.ts

@@ -474,10 +474,11 @@ export function logSeriesToLogsModel(logSeries: DataFrame[], queries: DataQuery[
       kind: LogsMetaKind.LabelsMap,
     });
   }
-
-  const limits = logSeries.filter((series) => series.meta && series.meta.limit);
+  // Data sources that set up searchWords on backend use meta.custom.limit.
+  // Data sources that set up searchWords through frontend can use meta.limit.
+  const limits = logSeries.filter((series) => series?.meta?.custom?.limit ?? series?.meta?.limit);
   const lastLimitPerRef = limits.reduce<Record<string, number>>((acc, elem) => {
-    acc[elem.refId ?? ''] = elem.meta?.limit ?? 0;
+    acc[elem.refId ?? ''] = elem.meta?.custom?.limit ?? elem.meta?.limit ?? 0;
 
     return acc;
   }, {});

public/app/plugins/datasource/elasticsearch/LegacyQueryRunner.ts

@@ -15,10 +15,10 @@ import {
 import { BackendSrvRequest, getBackendSrv, TemplateSrv } from '@grafana/runtime';
 
 import { ElasticResponse } from './ElasticResponse';
-import { ElasticDatasource, enhanceDataFrame } from './datasource';
+import { ElasticDatasource, enhanceDataFrameWithDataLinks } from './datasource';
 import { defaultBucketAgg, hasMetricOfType } from './queryDef';
 import { trackQuery } from './tracking';
-import { ElasticsearchQuery, Logs } from './types';
+import { DataLinkConfig, ElasticsearchQuery, Logs } from './types';
 
 export class LegacyQueryRunner {
   datasource: ElasticDatasource;
@@ -250,3 +250,17 @@ function transformHitsBasedOnDirection(response: any, direction: 'asc' | 'desc')
     ],
   };
 }
+
+/**
+ * Modifies dataFrame and adds dataLinks from the config.
+ * Exported for tests.
+ */
+export function enhanceDataFrame(dataFrame: DataFrame, dataLinks: DataLinkConfig[], limit?: number) {
+  if (limit) {
+    dataFrame.meta = {
+      ...dataFrame.meta,
+      limit,
+    };
+  }
+  enhanceDataFrameWithDataLinks(dataFrame, dataLinks);
+}

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -25,7 +25,8 @@ import { TemplateSrv } from 'app/features/templating/template_srv';
 
 import { createFetchResponse } from '../../../../test/helpers/createFetchResponse';
 
-import { ElasticDatasource, enhanceDataFrame } from './datasource';
+import { enhanceDataFrame } from './LegacyQueryRunner';
+import { ElasticDatasource } from './datasource';
 import { createElasticDatasource } from './mocks';
 import { Filters, ElasticsearchOptions, ElasticsearchQuery } from './types';
 

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -626,7 +626,15 @@ export class ElasticDatasource
     const { enableElasticsearchBackendQuerying } = config.featureToggles;
     if (enableElasticsearchBackendQuerying) {
       const start = new Date();
-      return super.query(request).pipe(tap((response) => trackQuery(response, request, start)));
+      return super.query(request).pipe(
+        tap((response) => trackQuery(response, request, start)),
+        map((response) => {
+          response.data.forEach((dataFrame) => {
+            enhanceDataFrameWithDataLinks(dataFrame, this.dataLinks);
+          });
+          return response;
+        })
+      );
     }
     return this.legacyQueryRunner.query(request);
   }
@@ -1034,18 +1042,7 @@ export class ElasticDatasource
   };
 }
 
-/**
- * Modifies dataframe and adds dataLinks from the config.
- * Exported for tests.
- */
-export function enhanceDataFrame(dataFrame: DataFrame, dataLinks: DataLinkConfig[], limit?: number) {
-  if (limit) {
-    dataFrame.meta = {
-      ...dataFrame.meta,
-      limit,
-    };
-  }
-
+export function enhanceDataFrameWithDataLinks(dataFrame: DataFrame, dataLinks: DataLinkConfig[]) {
   if (!dataLinks.length) {
     return;
   }