Elasticsearch: Add support for Elasticsearch 8.0 (Beta) (#41729)

* use fixed_interval in date_histogram

* Add 8.0 to available versions in datasource settings

* Remove moving_avg from available metric aggregations

* Add ES8 devenv

* Update public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts

Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com>

* Add FE tests

* Add BE test

* fix FE test

* fix BE test

Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com>

devenv/docker/blocks/elastic/docker-compose.yaml

@@ -1,19 +1,22 @@
 # You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
 
   elasticsearch-latest:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-beta1
     command: elasticsearch
     environment:
       - "discovery.type=single-node"
+      - "xpack.license.self_generated.type=basic"
+      - "xpack.security.enabled=false"
     ports:
       - "14200:9200"
       - "14300:9300"
 
-  fake-elastic-latest-data:
-    image: grafana/fake-data-gen
-    links:
-      - elasticsearch-latest
-    environment:
-      FD_SERVER: elasticsearch-latest
-      FD_DATASOURCE: elasticsearch7
-      FD_PORT: 9200
+  # TODO: uncomment when https://github.com/grafana/fake-data-gen/pull/20 is merged
+  # fake-elastic-latest-data:
+  #   image: grafana/fake-data-gen
+  #   links:
+  #     - elasticsearch-latest
+  #   environment:
+  #     FD_SERVER: elasticsearch-latest
+  #     FD_DATASOURCE: elasticsearch8
+  #     FD_PORT: 9200

devenv/docker/blocks/elastic/elasticsearch.yml

@@ -1,3 +0,0 @@
-script.inline: on
-script.indexed: on
-xpack.license.self_generated.type: basic

pkg/tsdb/elasticsearch/client/models.go

@@ -239,6 +239,7 @@ type HistogramAgg struct {
 type DateHistogramAgg struct {
 	Field          string          `json:"field"`
 	Interval       string          `json:"interval,omitempty"`
+	FixedInterval  string          `json:"fixed_interval,omitempty"`
 	MinDocCount    int             `json:"min_doc_count"`
 	Missing        *string         `json:"missing,omitempty"`
 	ExtendedBounds *ExtendedBounds `json:"extended_bounds"`

pkg/tsdb/elasticsearch/client/search_request.go

@@ -342,6 +342,11 @@ func (b *aggBuilderImpl) DateHistogram(key, field string, fn func(a *DateHistogr
 		fn(innerAgg, builder)
 	}
 
+	if b.version.Major() >= 8 {
+		innerAgg.FixedInterval = innerAgg.Interval
+		innerAgg.Interval = ""
+	}
+
 	b.aggDefs = append(b.aggDefs, aggDef)
 
 	return b

pkg/tsdb/elasticsearch/time_series_query_test.go

@@ -1049,6 +1049,62 @@ func TestSettingsCasting(t *testing.T) {
 
 			assert.Equal(t, 10, dateHistogramAgg.MinDocCount)
 		})
+
+		t.Run("interval parameter", func(t *testing.T) {
+			t.Run("Uses interval with ES < 8.0.0", func(t *testing.T) {
+				c := newFakeClient("7.7.0")
+				_, err := executeTsdbQuery(c, `{
+					"timeField": "@timestamp",
+					"bucketAggs": [
+						{
+							"type": "date_histogram",
+							"field": "@timestamp",
+							"id": "2",
+							"settings": {
+								"interval": "1d"							
+							}
+						}
+					],
+					"metrics": [
+						{ "id": "1", "type": "average", "field": "@value" }
+					]
+				}`, from, to, 15*time.Second)
+				assert.Nil(t, err)
+				sr := c.multisearchRequests[0].Requests[0]
+
+				dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg)
+
+				assert.Zero(t, dateHistogramAgg.FixedInterval)
+				assert.NotZero(t, dateHistogramAgg.Interval)
+			})
+
+			t.Run("Uses fixed_interval with ES >= 8.0.0", func(t *testing.T) {
+				c := newFakeClient("8.0.0")
+				_, err := executeTsdbQuery(c, `{
+					"timeField": "@timestamp",
+					"bucketAggs": [
+						{
+							"type": "date_histogram",
+							"field": "@timestamp",
+							"id": "2",
+							"settings": {
+								"interval": "1d"							
+							}
+						}
+					],
+					"metrics": [
+						{ "id": "1", "type": "average", "field": "@value" }
+					]
+				}`, from, to, 15*time.Second)
+				assert.Nil(t, err)
+				sr := c.multisearchRequests[0].Requests[0]
+
+				dateHistogramAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.DateHistogramAgg)
+
+				assert.NotZero(t, dateHistogramAgg.FixedInterval)
+				assert.Zero(t, dateHistogramAgg.Interval)
+			})
+		})
 	})
 
 	t.Run("Inline Script", func(t *testing.T) {

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts

@@ -112,7 +112,7 @@ export const metricAggregationConfig: MetricsConfiguration = {
     label: 'Moving Average',
     requiresField: true,
     isPipelineAgg: true,
-    versionRange: '>=2.0.0',
+    versionRange: '>=2.0.0 <8.0.0',
     supportsMissing: false,
     supportsMultipleBucketPaths: false,
     hasSettings: true,

public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx

@@ -14,7 +14,7 @@ const indexPatternTypes: Array<SelectableValue<'none' | Interval>> = [
   { label: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY' },
 ];
 
-const esVersions = [
+const esVersions: SelectableValue[] = [
   { label: '2.x', value: '2.0.0' },
   { label: '5.x', value: '5.0.0' },
   { label: '5.6+', value: '5.6.0' },
@@ -22,6 +22,11 @@ const esVersions = [
   { label: '7.0+', value: '7.0.0' },
   { label: '7.7+', value: '7.7.0' },
   { label: '7.10+', value: '7.10.0' },
+  {
+    label: '8.0+',
+    value: '8.0.0',
+    description: 'support for Elasticsearch 8 is currently experimental',
+  },
 ];
 
 type Props = {

public/app/plugins/datasource/elasticsearch/query_builder.ts

@@ -95,7 +95,7 @@ export class ElasticQueryBuilder {
   getDateHistogramAgg(aggDef: DateHistogram) {
     const esAgg: any = {};
     const settings = aggDef.settings || {};
-    esAgg.interval = settings.interval;
+
     esAgg.field = aggDef.field || this.timeField;
     esAgg.min_doc_count = settings.min_doc_count || 0;
     esAgg.extended_bounds = { min: '$timeFrom', max: '$timeTo' };
@@ -108,8 +108,14 @@ export class ElasticQueryBuilder {
       esAgg.offset = settings.offset;
     }
 
-    if (esAgg.interval === 'auto') {
-      esAgg.interval = '$__interval';
+    const interval = settings.interval === 'auto' ? '$__interval' : settings.interval;
+
+    if (gte(this.esVersion, '8.0.0')) {
+      // The deprecation was actually introduced in 7.0.0, we might want to use that instead of the removal date,
+      // but it woudl be a breaking change on our side.
+      esAgg.fixed_interval = interval;
+    } else {
+      esAgg.interval = interval;
     }
 
     return esAgg;

public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts

@@ -9,8 +9,9 @@ describe('ElasticQueryBuilder', () => {
   const builder6x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '6.0.0' });
   const builder7x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '7.0.0' });
   const builder77 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '7.7.0' });
+  const builder8 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: '8.0.0' });
 
-  const allBuilders = [builder, builder5x, builder56, builder6x, builder7x, builder77];
+  const allBuilders = [builder, builder5x, builder56, builder6x, builder7x, builder77, builder8];
 
   allBuilders.forEach((builder) => {
     describe(`version ${builder.esVersion}`, () => {
@@ -728,7 +729,7 @@ describe('ElasticQueryBuilder', () => {
             { _doc: { order: 'desc' } },
           ]);
 
-          const expectedAggs = {
+          const expectedAggs: any = {
             // FIXME: It's pretty weak to include this '1' in the test as it's not part of what we are testing here and
             // might change as a cause of unrelated changes
             1: {
@@ -742,6 +743,11 @@ describe('ElasticQueryBuilder', () => {
               },
             },
           };
+
+          if (gte(builder.esVersion, '8.0.0')) {
+            expectedAggs['1'].date_histogram.fixed_interval = expectedAggs['1'].date_histogram.interval;
+            delete expectedAggs['1'].date_histogram.interval;
+          }
           expect(query.aggs).toMatchObject(expectedAggs);
         });
 
@@ -921,6 +927,46 @@ describe('ElasticQueryBuilder', () => {
 
           expect(query.aggs['2'].date_histogram.field).toBe('@time');
         });
+
+        describe('interval parameter', () => {
+          it('should use interval if Elasticsearch version <8.0.0', () => {
+            const query = builder77.build({
+              refId: 'A',
+              metrics: [{ type: 'count', id: '1' }],
+              timeField: '@timestamp',
+              bucketAggs: [
+                {
+                  type: 'date_histogram',
+                  id: '2',
+                  field: '@time',
+                  settings: { min_doc_count: '1', interval: '1d' },
+                },
+              ],
+            });
+
+            expect(query.aggs['2'].date_histogram.interval).toBe('1d');
+            expect(query.aggs['2'].date_histogram.fixed_interval).toBeUndefined();
+          });
+        });
+
+        it('should use fixed_interval if Elasticsearch version >=8.0.0', () => {
+          const query = builder8.build({
+            refId: 'A',
+            metrics: [{ type: 'count', id: '1' }],
+            timeField: '@timestamp',
+            bucketAggs: [
+              {
+                type: 'date_histogram',
+                id: '2',
+                field: '@time',
+                settings: { min_doc_count: '1', interval: '1d' },
+              },
+            ],
+          });
+
+          expect(query.aggs['2'].date_histogram.interval).toBeUndefined();
+          expect(query.aggs['2'].date_histogram.fixed_interval).toBe('1d');
+        });
       });
     });
   });