From b3a12f486eba69e20dd7ff3a3d4dd065ede7a99f Mon Sep 17 00:00:00 2001 From: Isabella Siu Date: Thu, 12 Dec 2024 17:20:04 -0500 Subject: [PATCH] Elasticsearch: Use _field_caps instead of _mapping to get fields (#97607) --- .../feature-toggles/index.md | 1 + .../src/types/featureToggles.gen.ts | 1 + pkg/services/featuremgmt/registry.go | 6 + pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/services/featuremgmt/toggles_gen.json | 15 ++ pkg/tsdb/elasticsearch/elasticsearch.go | 12 +- .../elasticsearch/datasource.test.ts | 189 +++++++++++++++++- .../datasource/elasticsearch/datasource.ts | 70 ++++++- .../elasticsearch/hooks/useFields.ts | 2 +- 10 files changed, 296 insertions(+), 5 deletions(-) diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 8ec9eedebd7..1d10f7f7530 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -120,6 +120,7 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general- | `ssoSettingsLDAP` | Use the new SSO Settings API to configure LDAP | | `useSessionStorageForRedirection` | Use session storage for handling the redirection after login | | `reportingUseRawTimeRange` | Uses the original report or dashboard time range instead of making an absolute transformation | +| `elasticsearchCrossClusterSearch` | Enables cross cluster search in the Elasticsearch datasource | ## Experimental feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 37c84438f1f..7241d05eaa7 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -242,4 +242,5 @@ export interface FeatureToggles { azureMonitorEnableUserAuth?: boolean; alertingNotificationsStepMode?: boolean; feedbackButton?: boolean; + elasticsearchCrossClusterSearch?: boolean; } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 7537e52b4d6..a59e0bdd43e 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -1676,6 +1676,12 @@ var ( Owner: grafanaOperatorExperienceSquad, HideFromDocs: true, }, + { + Name: "elasticsearchCrossClusterSearch", + Description: "Enables cross cluster search in the Elasticsearch datasource", + Stage: FeatureStagePublicPreview, + Owner: awsDatasourcesSquad, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 1fde233fece..b7b6d1c3662 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -223,3 +223,4 @@ alertingUIOptimizeReducer,GA,@grafana/alerting-squad,false,false,true azureMonitorEnableUserAuth,GA,@grafana/partner-datasources,false,false,false alertingNotificationsStepMode,experimental,@grafana/alerting-squad,false,false,true feedbackButton,experimental,@grafana/grafana-operator-experience-squad,false,false,false +elasticsearchCrossClusterSearch,preview,@grafana/aws-datasources,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 4ba518fe8e9..2fe1fb9c22a 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -902,4 +902,8 @@ const ( // FlagFeedbackButton // Enables a button to send feedback from the Grafana UI FlagFeedbackButton = "feedbackButton" + + // FlagElasticsearchCrossClusterSearch + // Enables cross cluster search in the Elasticsearch datasource + FlagElasticsearchCrossClusterSearch = "elasticsearchCrossClusterSearch" ) diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 8feb0ce7185..3a0d9740432 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -1269,6 +1269,21 @@ "frontend": true } }, + { + "metadata": { + "name": "elasticsearchCrossClusterSearch", + "resourceVersion": "1733848475752", + "creationTimestamp": "2024-12-09T13:53:38Z", + "annotations": { + "grafana.app/updatedTimestamp": "2024-12-10 16:34:35.752111 +0000 UTC" + } + }, + "spec": { + "description": "Enables cross cluster search in the Elasticsearch datasource", + "stage": "preview", + "codeowner": "@grafana/aws-datasources" + } + }, { "metadata": { "name": "enableDatagridEditing", diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index ddd8671a114..40073bf1740 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -182,14 +182,21 @@ func (s *Service) getDSInfo(ctx context.Context, pluginCtx backend.PluginContext return &instance, nil } +func isFieldCaps(url string) bool { + return strings.HasSuffix(url, "/_field_caps") || url == "_field_caps" +} + func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { logger := s.logger.FromContext(ctx) // allowed paths for resource calls: // - empty string for fetching db version // - /_mapping for fetching index mapping, e.g. requests going to `index/_mapping` + // - /_field_caps for fetching field capabilities, e.g. requests going to `index/_field_caps` // - _msearch for executing getTerms queries // - _mapping for fetching "root" index mappings - if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_msearch" && req.Path != "_mapping" { + // - _field_caps for fetching "root" field capabilities + if req.Path != "" && !isFieldCaps(req.Path) && req.Path != "_msearch" && + !strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_mapping" { logger.Error("Invalid resource path", "path", req.Path) return fmt.Errorf("invalid resource URL: %s", req.Path) } @@ -266,6 +273,9 @@ func createElasticsearchURL(req *backend.CallResourceRequest, ds *es.DatasourceI } esUrl.Path = path.Join(esUrl.Path, req.Path) + if isFieldCaps(req.Path) { + esUrl.RawQuery = "fields=*" + } esUrlString := esUrl.String() // If the request path is empty and the URL does not end with a slash, add a slash to the URL. // This ensures that for version checks executed to the root URL, the URL ends with a slash. diff --git a/public/app/plugins/datasource/elasticsearch/datasource.test.ts b/public/app/plugins/datasource/elasticsearch/datasource.test.ts index beddbe42c05..c653ac8a132 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.test.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.test.ts @@ -12,7 +12,7 @@ import { TimeRange, toUtc, } from '@grafana/data'; -import { FetchResponse, reportInteraction, getBackendSrv, setBackendSrv, BackendSrv } from '@grafana/runtime'; +import { FetchResponse, reportInteraction, getBackendSrv, setBackendSrv, BackendSrv, config } from '@grafana/runtime'; import { ElasticDatasource } from './datasource'; import { createElasticDatasource, createElasticQuery, mockResponseFrames } from './mocks'; @@ -1253,4 +1253,191 @@ describe('ElasticDatasource', () => { }); }); }); + + describe('getFieldsFieldCap', () => { + const originalFeatureToggleValue = config.featureToggles.elasticsearchCrossClusterSearch; + + afterEach(() => { + config.featureToggles.elasticsearchCrossClusterSearch = originalFeatureToggleValue; + }); + const getFieldsMockData = { + fields: { + '@timestamp_millis': { + date: { + type: 'date', + metadata_field: false, + }, + }, + classification_terms: { + keyword: { + type: 'keyword', + metadata_field: false, + }, + }, + ip_address: { + ip: { + type: 'ip', + metadata_field: false, + }, + }, + 'justification_blob.criterion.keyword': { + keyword: { + type: 'keyword', + metadata_field: false, + }, + }, + 'justification_blob.criterion': { + text: { + type: 'text', + metadata_field: false, + }, + }, + justification_blob: { + object: { + type: 'object', + metadata_field: false, + }, + }, + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness': { + float: { + type: 'float', + metadata_field: false, + }, + }, + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score': { + float: { + type: 'float', + metadata_field: false, + }, + }, + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness': { + float: { + type: 'float', + metadata_field: false, + }, + }, + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score': { + float: { + type: 'float', + metadata_field: false, + }, + }, + overall_vote_score: { + float: { + type: 'float', + metadata_field: false, + }, + }, + _index: { + _index: { + type: '_index', + metadata_field: true, + }, + }, + }, + indices: ['[test-]YYYY.MM.DD'], + }; + + it('should not retry when ES is down', async () => { + config.featureToggles.elasticsearchCrossClusterSearch = true; + const twoDaysBefore = toUtc().subtract(2, 'day').format('YYYY.MM.DD'); + const ds = createElasticDatasource({ + jsonData: { interval: 'Daily' }, + }); + + ds.getResource = jest.fn().mockImplementation((options) => { + if (options.url === `test-${twoDaysBefore}/_field_caps`) { + return of({ + data: {}, + }); + } + return throwError({ status: 500 }); + }); + + const timeRange = { from: 1, to: 2 } as unknown as TimeRange; + await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + expect(received[0]).toStrictEqual({ status: 500 }); + expect(ds.getResource).toBeCalledTimes(1); + }); + }); + + it('should not retry more than 7 indices', async () => { + config.featureToggles.elasticsearchCrossClusterSearch = true; + const ds = createElasticDatasource({ + jsonData: { interval: 'Daily' }, + }); + + ds.getResource = jest.fn().mockImplementation(() => { + return throwError({ status: 404 }); + }); + + const timeRange = createTimeRange(dateTime().subtract(2, 'week'), dateTime()); + + await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + expect(received[0]).toStrictEqual('Could not find an available index for this time range.'); + expect(ds.getResource).toBeCalledTimes(7); + }); + }); + + it('should return nested fields', async () => { + config.featureToggles.elasticsearchCrossClusterSearch = true; + const ds = createElasticDatasource({ + jsonData: { interval: 'Daily' }, + }); + + ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData); + + await expect(ds.getFields()).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + + const fieldObjects = received[0]; + const fields = map(fieldObjects, 'text'); + expect(fields).toEqual([ + '@timestamp_millis', + 'classification_terms', + 'ip_address', + 'justification_blob.criterion.keyword', + 'justification_blob.criterion', + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness', + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score', + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness', + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score', + 'overall_vote_score', + ]); + }); + }); + it('should return number fields', async () => { + config.featureToggles.elasticsearchCrossClusterSearch = true; + ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData); + + await expect(ds.getFields(['number'])).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + + const fieldObjects = received[0]; + const fields = map(fieldObjects, 'text'); + expect(fields).toEqual([ + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness', + 'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score', + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness', + 'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score', + 'overall_vote_score', + ]); + }); + }); + + it('should return date fields', async () => { + config.featureToggles.elasticsearchCrossClusterSearch = true; + ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData); + + await expect(ds.getFields(['date'])).toEmitValuesWith((received) => { + expect(received.length).toBe(1); + + const fieldObjects = received[0]; + const fields = map(fieldObjects, 'text'); + expect(fields).toEqual(['@timestamp_millis']); + }); + }); + }); }); diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index 584f970fef8..3cc574ab770 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -1,4 +1,4 @@ -import { cloneDeep, first as _first, isNumber, isObject, isString, map as _map, find } from 'lodash'; +import { cloneDeep, first as _first, isNumber, isString, map as _map, find, isObject } from 'lodash'; import { from, generate, lastValueFrom, Observable, of } from 'rxjs'; import { catchError, first, map, mergeMap, skipWhile, throwIfEmpty, tap } from 'rxjs/operators'; import { SemVer } from 'semver'; @@ -46,6 +46,7 @@ import { BackendSrvRequest, TemplateSrv, getTemplateSrv, + config, } from '@grafana/runtime'; import { IndexPattern, intervalMap } from './IndexPattern'; @@ -197,7 +198,7 @@ export class ElasticDatasource indexList = [this.indexPattern.getIndexForToday()]; } - const url = '_mapping'; + const url = config.featureToggles.elasticsearchCrossClusterSearch ? '_field_caps' : '_mapping'; const indexUrlList = indexList.map((index) => { // make sure `index` does not end with a slash @@ -743,6 +744,10 @@ export class ElasticDatasource * or fix the implementation. */ getFields(type?: string[], range?: TimeRange): Observable { + if (config.featureToggles.elasticsearchCrossClusterSearch) { + return this.getFieldsCrossCluster(type, range); + } + const typeMap: Record = { float: 'number', double: 'number', @@ -823,6 +828,67 @@ export class ElasticDatasource ); } + getFieldsCrossCluster(type?: string[], range?: TimeRange): Observable { + const typeMap: Record = { + float: 'number', + double: 'number', + integer: 'number', + long: 'number', + date: 'date', + date_nanos: 'date', + string: 'string', + text: 'string', + scaled_float: 'number', + nested: 'nested', + histogram: 'number', + }; + return this.requestAllIndices(range).pipe( + map((result) => { + interface FieldInfo { + metadata_field: string; + } + const shouldAddField = (obj: Record>) => { + // equal query type filter, or via type map translation + for (const objField in obj) { + if (objField === 'object') { + continue; + } + if (obj[objField].metadata_field) { + continue; + } + + if (!type || type.length === 0) { + return true; + } + + if (type.includes(objField) || type.includes(typeMap[objField])) { + return true; + } + } + return false; + }; + + const fields: Record = {}; + + const fieldsData = result['fields']; + for (const fieldName in fieldsData) { + const fieldInfo = fieldsData[fieldName]; + if (shouldAddField(fieldInfo)) { + fields[fieldName] = { + text: fieldName, + type: fieldInfo.type, + }; + } + } + + // transform to array + return _map(fields, (value) => { + return value; + }); + }) + ); + } + /** * Get values for a given field. * Used for example in getTagValues. diff --git a/public/app/plugins/datasource/elasticsearch/hooks/useFields.ts b/public/app/plugins/datasource/elasticsearch/hooks/useFields.ts index 8f36406a53d..df9d1271431 100644 --- a/public/app/plugins/datasource/elasticsearch/hooks/useFields.ts +++ b/public/app/plugins/datasource/elasticsearch/hooks/useFields.ts @@ -60,7 +60,7 @@ export const useFields = (type: AggregationType | string[]) => { let rawFields: MetricFindValue[]; return async (q?: string) => { - // _mapping doesn't support filtering, we avoid sending a request everytime q changes + // TODO: use _field_caps to support filtering if (!rawFields) { rawFields = await lastValueFrom(datasource.getFields(filter, range)); }