Elasticsearch: Use _field_caps instead of _mapping to get fields (#97607)

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -120,6 +120,7 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general-
 | `ssoSettingsLDAP`                 | Use the new SSO Settings API to configure LDAP                                                                                                                                               |
 | `useSessionStorageForRedirection` | Use session storage for handling the redirection after login                                                                                                                                 |
 | `reportingUseRawTimeRange`        | Uses the original report or dashboard time range instead of making an absolute transformation                                                                                                |
+| `elasticsearchCrossClusterSearch` | Enables cross cluster search in the Elasticsearch datasource                                                                                                                                 |
 
 ## Experimental feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -242,4 +242,5 @@ export interface FeatureToggles {
   azureMonitorEnableUserAuth?: boolean;
   alertingNotificationsStepMode?: boolean;
   feedbackButton?: boolean;
+  elasticsearchCrossClusterSearch?: boolean;
 }

pkg/services/featuremgmt/registry.go

@@ -1676,6 +1676,12 @@ var (
 			Owner:        grafanaOperatorExperienceSquad,
 			HideFromDocs: true,
 		},
+		{
+			Name:        "elasticsearchCrossClusterSearch",
+			Description: "Enables cross cluster search in the Elasticsearch datasource",
+			Stage:       FeatureStagePublicPreview,
+			Owner:       awsDatasourcesSquad,
+		},
 	}
 )
 

pkg/services/featuremgmt/toggles_gen.csv

@@ -223,3 +223,4 @@ alertingUIOptimizeReducer,GA,@grafana/alerting-squad,false,false,true
 azureMonitorEnableUserAuth,GA,@grafana/partner-datasources,false,false,false
 alertingNotificationsStepMode,experimental,@grafana/alerting-squad,false,false,true
 feedbackButton,experimental,@grafana/grafana-operator-experience-squad,false,false,false
+elasticsearchCrossClusterSearch,preview,@grafana/aws-datasources,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -902,4 +902,8 @@ const (
 	// FlagFeedbackButton
 	// Enables a button to send feedback from the Grafana UI
 	FlagFeedbackButton = "feedbackButton"
+
+	// FlagElasticsearchCrossClusterSearch
+	// Enables cross cluster search in the Elasticsearch datasource
+	FlagElasticsearchCrossClusterSearch = "elasticsearchCrossClusterSearch"
 )

pkg/services/featuremgmt/toggles_gen.json

@@ -1269,6 +1269,21 @@
         "frontend": true
       }
     },
+    {
+      "metadata": {
+        "name": "elasticsearchCrossClusterSearch",
+        "resourceVersion": "1733848475752",
+        "creationTimestamp": "2024-12-09T13:53:38Z",
+        "annotations": {
+          "grafana.app/updatedTimestamp": "2024-12-10 16:34:35.752111 +0000 UTC"
+        }
+      },
+      "spec": {
+        "description": "Enables cross cluster search in the Elasticsearch datasource",
+        "stage": "preview",
+        "codeowner": "@grafana/aws-datasources"
+      }
+    },
     {
       "metadata": {
         "name": "enableDatagridEditing",

pkg/tsdb/elasticsearch/elasticsearch.go

@@ -182,14 +182,21 @@ func (s *Service) getDSInfo(ctx context.Context, pluginCtx backend.PluginContext
 	return &instance, nil
 }
 
+func isFieldCaps(url string) bool {
+	return strings.HasSuffix(url, "/_field_caps") || url == "_field_caps"
+}
+
 func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
 	logger := s.logger.FromContext(ctx)
 	// allowed paths for resource calls:
 	// - empty string for fetching db version
 	// - /_mapping for fetching index mapping, e.g. requests going to `index/_mapping`
+	// - /_field_caps for fetching field capabilities, e.g. requests going to `index/_field_caps`
 	// - _msearch for executing getTerms queries
 	// - _mapping for fetching "root" index mappings
-	if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_msearch" && req.Path != "_mapping" {
+	// - _field_caps for fetching "root" field capabilities
+	if req.Path != "" && !isFieldCaps(req.Path) && req.Path != "_msearch" &&
+		!strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_mapping" {
 		logger.Error("Invalid resource path", "path", req.Path)
 		return fmt.Errorf("invalid resource URL: %s", req.Path)
 	}
@@ -266,6 +273,9 @@ func createElasticsearchURL(req *backend.CallResourceRequest, ds *es.DatasourceI
 	}
 
 	esUrl.Path = path.Join(esUrl.Path, req.Path)
+	if isFieldCaps(req.Path) {
+		esUrl.RawQuery = "fields=*"
+	}
 	esUrlString := esUrl.String()
 	// If the request path is empty and the URL does not end with a slash, add a slash to the URL.
 	// This ensures that for version checks executed to the root URL, the URL ends with a slash.

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -12,7 +12,7 @@ import {
   TimeRange,
   toUtc,
 } from '@grafana/data';
-import { FetchResponse, reportInteraction, getBackendSrv, setBackendSrv, BackendSrv } from '@grafana/runtime';
+import { FetchResponse, reportInteraction, getBackendSrv, setBackendSrv, BackendSrv, config } from '@grafana/runtime';
 
 import { ElasticDatasource } from './datasource';
 import { createElasticDatasource, createElasticQuery, mockResponseFrames } from './mocks';
@@ -1253,4 +1253,191 @@ describe('ElasticDatasource', () => {
       });
     });
   });
+
+  describe('getFieldsFieldCap', () => {
+    const originalFeatureToggleValue = config.featureToggles.elasticsearchCrossClusterSearch;
+
+    afterEach(() => {
+      config.featureToggles.elasticsearchCrossClusterSearch = originalFeatureToggleValue;
+    });
+    const getFieldsMockData = {
+      fields: {
+        '@timestamp_millis': {
+          date: {
+            type: 'date',
+            metadata_field: false,
+          },
+        },
+        classification_terms: {
+          keyword: {
+            type: 'keyword',
+            metadata_field: false,
+          },
+        },
+        ip_address: {
+          ip: {
+            type: 'ip',
+            metadata_field: false,
+          },
+        },
+        'justification_blob.criterion.keyword': {
+          keyword: {
+            type: 'keyword',
+            metadata_field: false,
+          },
+        },
+        'justification_blob.criterion': {
+          text: {
+            type: 'text',
+            metadata_field: false,
+          },
+        },
+        justification_blob: {
+          object: {
+            type: 'object',
+            metadata_field: false,
+          },
+        },
+        'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness': {
+          float: {
+            type: 'float',
+            metadata_field: false,
+          },
+        },
+        'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score': {
+          float: {
+            type: 'float',
+            metadata_field: false,
+          },
+        },
+        'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness': {
+          float: {
+            type: 'float',
+            metadata_field: false,
+          },
+        },
+        'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score': {
+          float: {
+            type: 'float',
+            metadata_field: false,
+          },
+        },
+        overall_vote_score: {
+          float: {
+            type: 'float',
+            metadata_field: false,
+          },
+        },
+        _index: {
+          _index: {
+            type: '_index',
+            metadata_field: true,
+          },
+        },
+      },
+      indices: ['[test-]YYYY.MM.DD'],
+    };
+
+    it('should not retry when ES is down', async () => {
+      config.featureToggles.elasticsearchCrossClusterSearch = true;
+      const twoDaysBefore = toUtc().subtract(2, 'day').format('YYYY.MM.DD');
+      const ds = createElasticDatasource({
+        jsonData: { interval: 'Daily' },
+      });
+
+      ds.getResource = jest.fn().mockImplementation((options) => {
+        if (options.url === `test-${twoDaysBefore}/_field_caps`) {
+          return of({
+            data: {},
+          });
+        }
+        return throwError({ status: 500 });
+      });
+
+      const timeRange = { from: 1, to: 2 } as unknown as TimeRange;
+      await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => {
+        expect(received.length).toBe(1);
+        expect(received[0]).toStrictEqual({ status: 500 });
+        expect(ds.getResource).toBeCalledTimes(1);
+      });
+    });
+
+    it('should not retry more than 7 indices', async () => {
+      config.featureToggles.elasticsearchCrossClusterSearch = true;
+      const ds = createElasticDatasource({
+        jsonData: { interval: 'Daily' },
+      });
+
+      ds.getResource = jest.fn().mockImplementation(() => {
+        return throwError({ status: 404 });
+      });
+
+      const timeRange = createTimeRange(dateTime().subtract(2, 'week'), dateTime());
+
+      await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => {
+        expect(received.length).toBe(1);
+        expect(received[0]).toStrictEqual('Could not find an available index for this time range.');
+        expect(ds.getResource).toBeCalledTimes(7);
+      });
+    });
+
+    it('should return nested fields', async () => {
+      config.featureToggles.elasticsearchCrossClusterSearch = true;
+      const ds = createElasticDatasource({
+        jsonData: { interval: 'Daily' },
+      });
+
+      ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData);
+
+      await expect(ds.getFields()).toEmitValuesWith((received) => {
+        expect(received.length).toBe(1);
+
+        const fieldObjects = received[0];
+        const fields = map(fieldObjects, 'text');
+        expect(fields).toEqual([
+          '@timestamp_millis',
+          'classification_terms',
+          'ip_address',
+          'justification_blob.criterion.keyword',
+          'justification_blob.criterion',
+          'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
+          'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
+          'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
+          'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
+          'overall_vote_score',
+        ]);
+      });
+    });
+    it('should return number fields', async () => {
+      config.featureToggles.elasticsearchCrossClusterSearch = true;
+      ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData);
+
+      await expect(ds.getFields(['number'])).toEmitValuesWith((received) => {
+        expect(received.length).toBe(1);
+
+        const fieldObjects = received[0];
+        const fields = map(fieldObjects, 'text');
+        expect(fields).toEqual([
+          'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
+          'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
+          'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
+          'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
+          'overall_vote_score',
+        ]);
+      });
+    });
+
+    it('should return date fields', async () => {
+      config.featureToggles.elasticsearchCrossClusterSearch = true;
+      ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData);
+
+      await expect(ds.getFields(['date'])).toEmitValuesWith((received) => {
+        expect(received.length).toBe(1);
+
+        const fieldObjects = received[0];
+        const fields = map(fieldObjects, 'text');
+        expect(fields).toEqual(['@timestamp_millis']);
+      });
+    });
+  });
 });

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -1,4 +1,4 @@
-import { cloneDeep, first as _first, isNumber, isObject, isString, map as _map, find } from 'lodash';
+import { cloneDeep, first as _first, isNumber, isString, map as _map, find, isObject } from 'lodash';
 import { from, generate, lastValueFrom, Observable, of } from 'rxjs';
 import { catchError, first, map, mergeMap, skipWhile, throwIfEmpty, tap } from 'rxjs/operators';
 import { SemVer } from 'semver';
@@ -46,6 +46,7 @@ import {
   BackendSrvRequest,
   TemplateSrv,
   getTemplateSrv,
+  config,
 } from '@grafana/runtime';
 
 import { IndexPattern, intervalMap } from './IndexPattern';
@@ -197,7 +198,7 @@ export class ElasticDatasource
       indexList = [this.indexPattern.getIndexForToday()];
     }
 
-    const url = '_mapping';
+    const url = config.featureToggles.elasticsearchCrossClusterSearch ? '_field_caps' : '_mapping';
 
     const indexUrlList = indexList.map((index) => {
       // make sure `index` does not end with a slash
@@ -743,6 +744,10 @@ export class ElasticDatasource
    * or fix the implementation.
    */
   getFields(type?: string[], range?: TimeRange): Observable<MetricFindValue[]> {
+    if (config.featureToggles.elasticsearchCrossClusterSearch) {
+      return this.getFieldsCrossCluster(type, range);
+    }
+
     const typeMap: Record<string, string> = {
       float: 'number',
       double: 'number',
@@ -823,6 +828,67 @@ export class ElasticDatasource
     );
   }
 
+  getFieldsCrossCluster(type?: string[], range?: TimeRange): Observable<MetricFindValue[]> {
+    const typeMap: Record<string, string> = {
+      float: 'number',
+      double: 'number',
+      integer: 'number',
+      long: 'number',
+      date: 'date',
+      date_nanos: 'date',
+      string: 'string',
+      text: 'string',
+      scaled_float: 'number',
+      nested: 'nested',
+      histogram: 'number',
+    };
+    return this.requestAllIndices(range).pipe(
+      map((result) => {
+        interface FieldInfo {
+          metadata_field: string;
+        }
+        const shouldAddField = (obj: Record<string, Record<string, FieldInfo>>) => {
+          // equal query type filter, or via type map translation
+          for (const objField in obj) {
+            if (objField === 'object') {
+              continue;
+            }
+            if (obj[objField].metadata_field) {
+              continue;
+            }
+
+            if (!type || type.length === 0) {
+              return true;
+            }
+
+            if (type.includes(objField) || type.includes(typeMap[objField])) {
+              return true;
+            }
+          }
+          return false;
+        };
+
+        const fields: Record<string, { text: string; type: string }> = {};
+
+        const fieldsData = result['fields'];
+        for (const fieldName in fieldsData) {
+          const fieldInfo = fieldsData[fieldName];
+          if (shouldAddField(fieldInfo)) {
+            fields[fieldName] = {
+              text: fieldName,
+              type: fieldInfo.type,
+            };
+          }
+        }
+
+        // transform to array
+        return _map(fields, (value) => {
+          return value;
+        });
+      })
+    );
+  }
+
   /**
    * Get values for a given field.
    * Used for example in getTagValues.

public/app/plugins/datasource/elasticsearch/hooks/useFields.ts

@@ -60,7 +60,7 @@ export const useFields = (type: AggregationType | string[]) => {
   let rawFields: MetricFindValue[];
 
   return async (q?: string) => {
-    // _mapping doesn't support filtering, we avoid sending a request everytime q changes
+    // TODO: use _field_caps to support filtering
     if (!rawFields) {
       rawFields = await lastValueFrom(datasource.getFields(filter, range));
     }