diff --git a/devenv/docker/blocks/elastic/docker-compose.yaml b/devenv/docker/blocks/elastic/docker-compose.yaml index 3ef9853f943..78c67982814 100644 --- a/devenv/docker/blocks/elastic/docker-compose.yaml +++ b/devenv/docker/blocks/elastic/docker-compose.yaml @@ -1,19 +1,22 @@ # You need to run 'sysctl -w vm.max_map_count=262144' on the host machine elasticsearch-latest: - image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2 + image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-beta1 command: elasticsearch environment: - "discovery.type=single-node" + - "xpack.license.self_generated.type=basic" + - "xpack.security.enabled=false" ports: - "14200:9200" - "14300:9300" - fake-elastic-latest-data: - image: grafana/fake-data-gen - links: - - elasticsearch-latest - environment: - FD_SERVER: elasticsearch-latest - FD_DATASOURCE: elasticsearch7 - FD_PORT: 9200 \ No newline at end of file + # TODO: uncomment when https://github.com/grafana/fake-data-gen/pull/20 is merged + # fake-elastic-latest-data: + # image: grafana/fake-data-gen + # links: + # - elasticsearch-latest + # environment: + # FD_SERVER: elasticsearch-latest + # FD_DATASOURCE: elasticsearch8 + # FD_PORT: 9200 diff --git a/devenv/docker/blocks/elastic/elasticsearch.yml b/devenv/docker/blocks/elastic/elasticsearch.yml deleted file mode 100644 index 0061758eb58..00000000000 --- a/devenv/docker/blocks/elastic/elasticsearch.yml +++ /dev/null @@ -1,3 +0,0 @@ -script.inline: on -script.indexed: on -xpack.license.self_generated.type: basic diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 79039e224c7..611763efc35 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -239,6 +239,7 @@ type HistogramAgg struct { type DateHistogramAgg struct { Field string `json:"field"` Interval string `json:"interval,omitempty"` + FixedInterval string `json:"fixed_interval,omitempty"` MinDocCount int `json:"min_doc_count"` Missing *string `json:"missing,omitempty"` ExtendedBounds *ExtendedBounds `json:"extended_bounds"` diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go index 958120d3faa..a35de074c88 100644 --- a/pkg/tsdb/elasticsearch/client/search_request.go +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -342,6 +342,11 @@ func (b *aggBuilderImpl) DateHistogram(key, field string, fn func(a *DateHistogr fn(innerAgg, builder) } + if b.version.Major() >= 8 { + innerAgg.FixedInterval = innerAgg.Interval + innerAgg.Interval = "" + } + b.aggDefs = append(b.aggDefs, aggDef) return b diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index 79c45f82422..67b6ce47fe0 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -1049,6 +1049,62 @@ func TestSettingsCasting(t *testing.T) { assert.Equal(t, 10, dateHistogramAgg.MinDocCount) }) + + t.Run("interval parameter", func(t *testing.T) { + t.Run("Uses interval with ES < 8.0.0", func(t *testing.T) { + c := newFakeClient("7.7.0") + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "type": "date_histogram", + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "1d" + } + } + ], + "metrics": [ + { "id": "1", "type": "average", "field": "@value" } + ] + }`, from, to, 15*time.Second) + assert.Nil(t, err) + sr := c.multisearchRequests[0].Requests[0] + + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg) + + assert.Zero(t, dateHistogramAgg.FixedInterval) + assert.NotZero(t, dateHistogramAgg.Interval) + }) + + t.Run("Uses fixed_interval with ES >= 8.0.0", func(t *testing.T) { + c := newFakeClient("8.0.0") + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { + "type": "date_histogram", + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "1d" + } + } + ], + "metrics": [ + { "id": "1", "type": "average", "field": "@value" } + ] + }`, from, to, 15*time.Second) + assert.Nil(t, err) + sr := c.multisearchRequests[0].Requests[0] + + dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg) + + assert.NotZero(t, dateHistogramAgg.FixedInterval) + assert.Zero(t, dateHistogramAgg.Interval) + }) + }) }) t.Run("Inline Script", func(t *testing.T) { diff --git a/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts b/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts index 704bf077e5f..2828ee85c34 100644 --- a/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts +++ b/public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts @@ -112,7 +112,7 @@ export const metricAggregationConfig: MetricsConfiguration = { label: 'Moving Average', requiresField: true, isPipelineAgg: true, - versionRange: '>=2.0.0', + versionRange: '>=2.0.0 <8.0.0', supportsMissing: false, supportsMultipleBucketPaths: false, hasSettings: true, diff --git a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx index 35f716b498b..139f8f74aa3 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx +++ b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx @@ -14,7 +14,7 @@ const indexPatternTypes: Array> = [ { label: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY' }, ]; -const esVersions = [ +const esVersions: SelectableValue[] = [ { label: '2.x', value: '2.0.0' }, { label: '5.x', value: '5.0.0' }, { label: '5.6+', value: '5.6.0' }, @@ -22,6 +22,11 @@ const esVersions = [ { label: '7.0+', value: '7.0.0' }, { label: '7.7+', value: '7.7.0' }, { label: '7.10+', value: '7.10.0' }, + { + label: '8.0+', + value: '8.0.0', + description: 'support for Elasticsearch 8 is currently experimental', + }, ]; type Props = { diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index 84cd5ba5731..300f8ce1fab 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -95,7 +95,7 @@ export class ElasticQueryBuilder { getDateHistogramAgg(aggDef: DateHistogram) { const esAgg: any = {}; const settings = aggDef.settings || {}; - esAgg.interval = settings.interval; + esAgg.field = aggDef.field || this.timeField; esAgg.min_doc_count = settings.min_doc_count || 0; esAgg.extended_bounds = { min: '$timeFrom', max: '$timeTo' }; @@ -108,8 +108,14 @@ export class ElasticQueryBuilder { esAgg.offset = settings.offset; } - if (esAgg.interval === 'auto') { - esAgg.interval = '$__interval'; + const interval = settings.interval === 'auto' ? '$__interval' : settings.interval; + + if (gte(this.esVersion, '8.0.0')) { + // The deprecation was actually introduced in 7.0.0, we might want to use that instead of the removal date, + // but it woudl be a breaking change on our side. + esAgg.fixed_interval = interval; + } else { + esAgg.interval = interval; } return esAgg; diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts index 2e956a6e3cd..718fc4e9c3c 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts @@ -9,8 +9,9 @@ describe('ElasticQueryBuilder', () => { const builder6x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '6.0.0' }); const builder7x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '7.0.0' }); const builder77 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '7.7.0' }); + const builder8 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '8.0.0' }); - const allBuilders = [builder, builder5x, builder56, builder6x, builder7x, builder77]; + const allBuilders = [builder, builder5x, builder56, builder6x, builder7x, builder77, builder8]; allBuilders.forEach((builder) => { describe(`version ${builder.esVersion}`, () => { @@ -728,7 +729,7 @@ describe('ElasticQueryBuilder', () => { { _doc: { order: 'desc' } }, ]); - const expectedAggs = { + const expectedAggs: any = { // FIXME: It's pretty weak to include this '1' in the test as it's not part of what we are testing here and // might change as a cause of unrelated changes 1: { @@ -742,6 +743,11 @@ describe('ElasticQueryBuilder', () => { }, }, }; + + if (gte(builder.esVersion, '8.0.0')) { + expectedAggs['1'].date_histogram.fixed_interval = expectedAggs['1'].date_histogram.interval; + delete expectedAggs['1'].date_histogram.interval; + } expect(query.aggs).toMatchObject(expectedAggs); }); @@ -921,6 +927,46 @@ describe('ElasticQueryBuilder', () => { expect(query.aggs['2'].date_histogram.field).toBe('@time'); }); + + describe('interval parameter', () => { + it('should use interval if Elasticsearch version <8.0.0', () => { + const query = builder77.build({ + refId: 'A', + metrics: [{ type: 'count', id: '1' }], + timeField: '@timestamp', + bucketAggs: [ + { + type: 'date_histogram', + id: '2', + field: '@time', + settings: { min_doc_count: '1', interval: '1d' }, + }, + ], + }); + + expect(query.aggs['2'].date_histogram.interval).toBe('1d'); + expect(query.aggs['2'].date_histogram.fixed_interval).toBeUndefined(); + }); + }); + + it('should use fixed_interval if Elasticsearch version >=8.0.0', () => { + const query = builder8.build({ + refId: 'A', + metrics: [{ type: 'count', id: '1' }], + timeField: '@timestamp', + bucketAggs: [ + { + type: 'date_histogram', + id: '2', + field: '@time', + settings: { min_doc_count: '1', interval: '1d' }, + }, + ], + }); + + expect(query.aggs['2'].date_histogram.interval).toBeUndefined(); + expect(query.aggs['2'].date_histogram.fixed_interval).toBe('1d'); + }); }); }); });