diff --git a/devenv/dev-dashboards/datasource_tests_elasticsearch_compare.json b/devenv/dev-dashboards/datasource_tests_elasticsearch_compare.json index f07c6f46332..c631ea0a151 100644 --- a/devenv/dev-dashboards/datasource_tests_elasticsearch_compare.json +++ b/devenv/dev-dashboards/datasource_tests_elasticsearch_compare.json @@ -17,7 +17,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "iteration": 1542304484522, + "iteration": 1545263815779, "links": [ { "icon": "external link", @@ -66,6 +66,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -168,6 +169,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -270,6 +272,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -372,6 +375,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -474,6 +478,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -576,6 +581,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -2249,6 +2255,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -2366,6 +2373,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -2483,6 +2491,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -2600,6 +2609,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -2717,6 +2727,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -2834,6 +2845,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -2951,6 +2963,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -3068,6 +3081,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -3185,6 +3199,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -3302,6 +3317,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -3419,6 +3435,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -3536,6 +3553,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -3667,6 +3685,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -3780,6 +3799,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -3893,6 +3913,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -4006,6 +4027,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -4119,6 +4141,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -4232,6 +4255,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -4345,6 +4369,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -4458,6 +4483,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -4571,6 +4597,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -4684,6 +4711,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -4797,6 +4825,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -4910,6 +4939,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -5008,6 +5038,512 @@ "x": 0, "y": 4 }, + "id": 60, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$version_one", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 63, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "field": "select field", + "hide": true, + "id": "1", + "type": "count" + }, + { + "field": "select field", + "id": "3", + "meta": {}, + "pipelineVariables": [ + { + "name": "var1", + "pipelineAgg": "1" + } + ], + "settings": { + "script": "params.var1 * 1000" + }, + "type": "bucket_script" + } + ], + "refId": "A", + "timeField": "@timestamp" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "count * 1000 (version one) - interval auto", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$version_two", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 64, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "field": "select field", + "hide": true, + "id": "1", + "type": "count" + }, + { + "field": "select field", + "id": "3", + "meta": {}, + "pipelineVariables": [ + { + "name": "var1", + "pipelineAgg": "1" + } + ], + "settings": { + "script": "params.var1 * 1000" + }, + "type": "bucket_script" + } + ], + "refId": "A", + "timeField": "@timestamp" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "count * 1000 (version two) - interval auto", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$version_one", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "field": "select field", + "hide": true, + "id": "1", + "type": "count" + }, + { + "field": "@value", + "hide": true, + "id": "3", + "meta": {}, + "settings": {}, + "type": "avg" + }, + { + "field": "select field", + "id": "4", + "meta": {}, + "pipelineVariables": [ + { + "name": "var1", + "pipelineAgg": "1" + }, + { + "name": "var2", + "pipelineAgg": "3" + } + ], + "settings": { + "script": "params.var1 * params.var2" + }, + "type": "bucket_script" + } + ], + "refId": "A", + "timeField": "@timestamp" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "count * avg (version one) - interval auto", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$version_two", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 66, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": 0, + "trimEdges": 0 + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "field": "select field", + "hide": true, + "id": "1", + "type": "count" + }, + { + "field": "@value", + "hide": true, + "id": "3", + "meta": {}, + "settings": {}, + "type": "avg" + }, + { + "field": "select field", + "id": "4", + "meta": {}, + "pipelineVariables": [ + { + "name": "var1", + "pipelineAgg": "1" + }, + { + "name": "var2", + "pipelineAgg": "3" + } + ], + "settings": { + "script": "params.var1 * params.var2" + }, + "type": "bucket_script" + } + ], + "refId": "A", + "timeField": "@timestamp" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "count * avg (version two) - interval auto", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Basic date histogram with bucket script aggregation", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "id": 54, "panels": [ { @@ -5042,6 +5578,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -5193,6 +5730,7 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, @@ -5328,8 +5866,8 @@ "list": [ { "current": { - "text": "gdev-elasticsearch-v2-metrics", - "value": "gdev-elasticsearch-v2-metrics" + "text": "gdev-elasticsearch-v5-metrics", + "value": "gdev-elasticsearch-v5-metrics" }, "hide": 0, "label": "Version One", @@ -5343,8 +5881,8 @@ }, { "current": { - "text": "gdev-elasticsearch-v5-metrics", - "value": "gdev-elasticsearch-v5-metrics" + "text": "gdev-elasticsearch-v6-metrics", + "value": "gdev-elasticsearch-v6-metrics" }, "hide": 0, "label": "Version Two", @@ -5359,7 +5897,7 @@ ] }, "time": { - "from": "now-3h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -5390,5 +5928,5 @@ "timezone": "", "title": "Datasource tests - Elasticsearch comparison", "uid": "fuFWehBmk", - "version": 10 + "version": 4 } \ No newline at end of file diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index 5307dc34b27..fcc96aeba93 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -292,7 +292,7 @@ func (a *MetricAggregation) MarshalJSON() ([]byte, error) { // PipelineAggregation represents a metric aggregation type PipelineAggregation struct { - BucketPath string + BucketPath interface{} Settings map[string]interface{} } diff --git a/pkg/tsdb/elasticsearch/client/search_request.go b/pkg/tsdb/elasticsearch/client/search_request.go index d89a98cbadb..b8c9232f97b 100644 --- a/pkg/tsdb/elasticsearch/client/search_request.go +++ b/pkg/tsdb/elasticsearch/client/search_request.go @@ -268,7 +268,7 @@ type AggBuilder interface { Filters(key string, fn func(a *FiltersAggregation, b AggBuilder)) AggBuilder GeoHashGrid(key, field string, fn func(a *GeoHashGridAggregation, b AggBuilder)) AggBuilder Metric(key, metricType, field string, fn func(a *MetricAggregation)) AggBuilder - Pipeline(key, pipelineType, bucketPath string, fn func(a *PipelineAggregation)) AggBuilder + Pipeline(key, pipelineType string, bucketPath interface{}, fn func(a *PipelineAggregation)) AggBuilder Build() (AggArray, error) } @@ -438,7 +438,7 @@ func (b *aggBuilderImpl) Metric(key, metricType, field string, fn func(a *Metric return b } -func (b *aggBuilderImpl) Pipeline(key, pipelineType, bucketPath string, fn func(a *PipelineAggregation)) AggBuilder { +func (b *aggBuilderImpl) Pipeline(key, pipelineType string, bucketPath interface{}, fn func(a *PipelineAggregation)) AggBuilder { innerAgg := &PipelineAggregation{ BucketPath: bucketPath, Settings: make(map[string]interface{}), diff --git a/pkg/tsdb/elasticsearch/models.go b/pkg/tsdb/elasticsearch/models.go index 46af5e4b745..d38e21c3ebc 100644 --- a/pkg/tsdb/elasticsearch/models.go +++ b/pkg/tsdb/elasticsearch/models.go @@ -25,13 +25,14 @@ type BucketAgg struct { // MetricAgg represents a metric aggregation of the time series query model of the datasource type MetricAgg struct { - Field string `json:"field"` - Hide bool `json:"hide"` - ID string `json:"id"` - PipelineAggregate string `json:"pipelineAgg"` - Settings *simplejson.Json `json:"settings"` - Meta *simplejson.Json `json:"meta"` - Type string `json:"type"` + Field string `json:"field"` + Hide bool `json:"hide"` + ID string `json:"id"` + PipelineAggregate string `json:"pipelineAgg"` + PipelineVariables map[string]string `json:"pipelineVariables"` + Settings *simplejson.Json `json:"settings"` + Meta *simplejson.Json `json:"meta"` + Type string `json:"type"` } var metricAggType = map[string]string{ @@ -45,6 +46,7 @@ var metricAggType = map[string]string{ "cardinality": "Unique Count", "moving_avg": "Moving Average", "derivative": "Derivative", + "bucket_script": "Bucket Script", "raw_document": "Raw Document", } @@ -60,8 +62,13 @@ var extendedStats = map[string]string{ } var pipelineAggType = map[string]string{ - "moving_avg": "moving_avg", - "derivative": "derivative", + "moving_avg": "moving_avg", + "derivative": "derivative", + "bucket_script": "bucket_script", +} + +var pipelineAggWithMultipleBucketPathsType = map[string]string{ + "bucket_script": "bucket_script", } func isPipelineAgg(metricType string) bool { @@ -71,6 +78,13 @@ func isPipelineAgg(metricType string) bool { return false } +func isPipelineAggWithMultipleBucketPaths(metricType string) bool { + if _, ok := pipelineAggWithMultipleBucketPathsType[metricType]; ok { + return true + } + return false +} + func describeMetric(metricType, field string) string { text := metricAggType[metricType] if metricType == countType { diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index b2c724a9b93..6bbaa3df34b 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -260,6 +260,7 @@ func (rp *responseParser) processMetrics(esAgg *simplejson.Json, target *Query, newSeries.Tags["metric"] = metric.Type newSeries.Tags["field"] = metric.Field + newSeries.Tags["metricId"] = metric.ID for _, v := range esAgg.Get("buckets").MustArray() { bucket := simplejson.NewFromAny(v) key := castToNullFloat(bucket.Get("key")) @@ -459,20 +460,42 @@ func (rp *responseParser) getSeriesName(series *tsdb.TimeSeries, target *Query, } // todo, if field and pipelineAgg if field != "" && isPipelineAgg(metricType) { - found := false - for _, metric := range target.Metrics { - if metric.ID == field { - metricName += " " + describeMetric(metric.Type, field) - found = true + if isPipelineAggWithMultipleBucketPaths(metricType) { + metricID := "" + if v, ok := series.Tags["metricId"]; ok { + metricID = v + } + + for _, metric := range target.Metrics { + if metric.ID == metricID { + metricName = metric.Settings.Get("script").MustString() + for name, pipelineAgg := range metric.PipelineVariables { + for _, m := range target.Metrics { + if m.ID == pipelineAgg { + metricName = strings.Replace(metricName, "params."+name, describeMetric(m.Type, m.Field), -1) + } + } + } + } + } + } else { + found := false + for _, metric := range target.Metrics { + if metric.ID == field { + metricName += " " + describeMetric(metric.Type, field) + found = true + } + } + if !found { + metricName = "Unset" } - } - if !found { - metricName = "Unset" } } else if field != "" { metricName += " " + field } + delete(series.Tags, "metricId") + if len(series.Tags) == 0 { return metricName } diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index b00c14cf946..e9cd8ad0980 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -787,6 +787,84 @@ func TestResponseParser(t *testing.T) { So(rows[0][2].(null.Float).Float64, ShouldEqual, 3000) }) + Convey("With bucket_script", func() { + targets := map[string]string{ + "A": `{ + "timeField": "@timestamp", + "metrics": [ + { "id": "1", "type": "sum", "field": "@value" }, + { "id": "3", "type": "max", "field": "@value" }, + { + "id": "4", + "field": "select field", + "pipelineVariables": [{ "name": "var1", "pipelineAgg": "1" }, { "name": "var2", "pipelineAgg": "3" }], + "settings": { "script": "params.var1 * params.var2" }, + "type": "bucket_script" + } + ], + "bucketAggs": [{ "type": "date_histogram", "field": "@timestamp", "id": "2" }] + }`, + } + response := `{ + "responses": [ + { + "aggregations": { + "2": { + "buckets": [ + { + "1": { "value": 2 }, + "3": { "value": 3 }, + "4": { "value": 6 }, + "doc_count": 60, + "key": 1000 + }, + { + "1": { "value": 3 }, + "3": { "value": 4 }, + "4": { "value": 12 }, + "doc_count": 60, + "key": 2000 + } + ] + } + } + } + ] + }` + rp, err := newResponseParserForTest(targets, response) + So(err, ShouldBeNil) + result, err := rp.getTimeSeries() + So(err, ShouldBeNil) + So(result.Results, ShouldHaveLength, 1) + + queryRes := result.Results["A"] + So(queryRes, ShouldNotBeNil) + So(queryRes.Series, ShouldHaveLength, 3) + seriesOne := queryRes.Series[0] + So(seriesOne.Name, ShouldEqual, "Sum @value") + So(seriesOne.Points, ShouldHaveLength, 2) + So(seriesOne.Points[0][0].Float64, ShouldEqual, 2) + So(seriesOne.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesOne.Points[1][0].Float64, ShouldEqual, 3) + So(seriesOne.Points[1][1].Float64, ShouldEqual, 2000) + + seriesTwo := queryRes.Series[1] + So(seriesTwo.Name, ShouldEqual, "Max @value") + So(seriesTwo.Points, ShouldHaveLength, 2) + So(seriesTwo.Points[0][0].Float64, ShouldEqual, 3) + So(seriesTwo.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesTwo.Points[1][0].Float64, ShouldEqual, 4) + So(seriesTwo.Points[1][1].Float64, ShouldEqual, 2000) + + seriesThree := queryRes.Series[2] + So(seriesThree.Name, ShouldEqual, "Sum @value * Max @value") + So(seriesThree.Points, ShouldHaveLength, 2) + So(seriesThree.Points[0][0].Float64, ShouldEqual, 6) + So(seriesThree.Points[0][1].Float64, ShouldEqual, 1000) + So(seriesThree.Points[1][0].Float64, ShouldEqual, 12) + So(seriesThree.Points[1][1].Float64, ShouldEqual, 2000) + }) + // Convey("Raw documents query", func() { // targets := map[string]string{ // "A": `{ diff --git a/pkg/tsdb/elasticsearch/time_series_query.go b/pkg/tsdb/elasticsearch/time_series_query.go index 28a930df3a2..e1cd466275e 100644 --- a/pkg/tsdb/elasticsearch/time_series_query.go +++ b/pkg/tsdb/elasticsearch/time_series_query.go @@ -94,26 +94,56 @@ func (e *timeSeriesQuery) execute() (*tsdb.Response, error) { } if isPipelineAgg(m.Type) { - if _, err := strconv.Atoi(m.PipelineAggregate); err == nil { - var appliedAgg *MetricAgg - for _, pipelineMetric := range q.Metrics { - if pipelineMetric.ID == m.PipelineAggregate { - appliedAgg = pipelineMetric - break - } - } - if appliedAgg != nil { - bucketPath := m.PipelineAggregate - if appliedAgg.Type == countType { - bucketPath = "_count" + if isPipelineAggWithMultipleBucketPaths(m.Type) { + if len(m.PipelineVariables) > 0 { + bucketPaths := map[string]interface{}{} + for name, pipelineAgg := range m.PipelineVariables { + if _, err := strconv.Atoi(pipelineAgg); err == nil { + var appliedAgg *MetricAgg + for _, pipelineMetric := range q.Metrics { + if pipelineMetric.ID == pipelineAgg { + appliedAgg = pipelineMetric + break + } + } + if appliedAgg != nil { + if appliedAgg.Type == countType { + bucketPaths[name] = "_count" + } else { + bucketPaths[name] = pipelineAgg + } + } + } } - aggBuilder.Pipeline(m.ID, m.Type, bucketPath, func(a *es.PipelineAggregation) { + aggBuilder.Pipeline(m.ID, m.Type, bucketPaths, func(a *es.PipelineAggregation) { a.Settings = m.Settings.MustMap() }) + } else { + continue } } else { - continue + if _, err := strconv.Atoi(m.PipelineAggregate); err == nil { + var appliedAgg *MetricAgg + for _, pipelineMetric := range q.Metrics { + if pipelineMetric.ID == m.PipelineAggregate { + appliedAgg = pipelineMetric + break + } + } + if appliedAgg != nil { + bucketPath := m.PipelineAggregate + if appliedAgg.Type == countType { + bucketPath = "_count" + } + + aggBuilder.Pipeline(m.ID, m.Type, bucketPath, func(a *es.PipelineAggregation) { + a.Settings = m.Settings.MustMap() + }) + } + } else { + continue + } } } else { aggBuilder.Metric(m.ID, m.Type, m.Field, func(a *es.MetricAggregation) { @@ -328,12 +358,20 @@ func (p *timeSeriesQueryParser) parseMetrics(model *simplejson.Json) ([]*MetricA metric.PipelineAggregate = metricJSON.Get("pipelineAgg").MustString() metric.Settings = simplejson.NewFromAny(metricJSON.Get("settings").MustMap()) metric.Meta = simplejson.NewFromAny(metricJSON.Get("meta").MustMap()) - metric.Type, err = metricJSON.Get("type").String() if err != nil { return nil, err } + if isPipelineAggWithMultipleBucketPaths(metric.Type) { + metric.PipelineVariables = map[string]string{} + pvArr := metricJSON.Get("pipelineVariables").MustArray() + for _, v := range pvArr { + kv := v.(map[string]interface{}) + metric.PipelineVariables[kv["name"].(string)] = kv["pipelineAgg"].(string) + } + } + result = append(result, metric) } return result, nil diff --git a/pkg/tsdb/elasticsearch/time_series_query_test.go b/pkg/tsdb/elasticsearch/time_series_query_test.go index a4f305242fa..3a558c32782 100644 --- a/pkg/tsdb/elasticsearch/time_series_query_test.go +++ b/pkg/tsdb/elasticsearch/time_series_query_test.go @@ -543,6 +543,77 @@ func TestExecuteTimeSeriesQuery(t *testing.T) { plAgg := derivativeAgg.Aggregation.Aggregation.(*es.PipelineAggregation) So(plAgg.BucketPath, ShouldEqual, "_count") }) + + Convey("With bucket_script", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [ + { "id": "3", "type": "sum", "field": "@value" }, + { "id": "5", "type": "max", "field": "@value" }, + { + "id": "2", + "type": "bucket_script", + "pipelineVariables": [ + { "name": "var1", "pipelineAgg": "3" }, + { "name": "var2", "pipelineAgg": "5" } + ], + "settings": { "script": "params.var1 * params.var2" } + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "4") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + + bucketScriptAgg := firstLevel.Aggregation.Aggs[2] + So(bucketScriptAgg.Key, ShouldEqual, "2") + plAgg := bucketScriptAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(plAgg.BucketPath.(map[string]interface{}), ShouldResemble, map[string]interface{}{ + "var1": "3", + "var2": "5", + }) + }) + + Convey("With bucket_script doc count", func() { + c := newFakeClient(5) + _, err := executeTsdbQuery(c, `{ + "timeField": "@timestamp", + "bucketAggs": [ + { "type": "date_histogram", "field": "@timestamp", "id": "4" } + ], + "metrics": [ + { "id": "3", "type": "count", "field": "select field" }, + { + "id": "2", + "type": "bucket_script", + "pipelineVariables": [ + { "name": "var1", "pipelineAgg": "3" } + ], + "settings": { "script": "params.var1 * 1000" } + } + ] + }`, from, to, 15*time.Second) + So(err, ShouldBeNil) + sr := c.multisearchRequests[0].Requests[0] + + firstLevel := sr.Aggs[0] + So(firstLevel.Key, ShouldEqual, "4") + So(firstLevel.Aggregation.Type, ShouldEqual, "date_histogram") + + bucketScriptAgg := firstLevel.Aggregation.Aggs[0] + So(bucketScriptAgg.Key, ShouldEqual, "2") + plAgg := bucketScriptAgg.Aggregation.Aggregation.(*es.PipelineAggregation) + So(plAgg.BucketPath.(map[string]interface{}), ShouldResemble, map[string]interface{}{ + "var1": "_count", + }) + }) }) } diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.ts b/public/app/plugins/datasource/elasticsearch/elastic_response.ts index 7adec22c545..49f33e2963a 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.ts +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.ts @@ -88,6 +88,7 @@ export class ElasticResponse { datapoints: [], metric: metric.type, field: metric.field, + metricId: metric.id, props: props, }; for (i = 0; i < esAgg.buckets.length; i++) { @@ -240,7 +241,7 @@ export class ElasticResponse { return metricName; } if (group === 'field') { - return series.field; + return series.field || ''; } return match; @@ -248,11 +249,27 @@ export class ElasticResponse { } if (series.field && queryDef.isPipelineAgg(series.metric)) { - const appliedAgg = _.find(target.metrics, { id: series.field }); - if (appliedAgg) { - metricName += ' ' + queryDef.describeMetric(appliedAgg); + if (series.metric && queryDef.isPipelineAggWithMultipleBucketPaths(series.metric)) { + const agg = _.find(target.metrics, { id: series.metricId }); + if (agg && agg.settings.script) { + metricName = agg.settings.script; + + for (const pv of agg.pipelineVariables) { + const appliedAgg = _.find(target.metrics, { id: pv.pipelineAgg }); + if (appliedAgg) { + metricName = metricName.replace('params.' + pv.name, queryDef.describeMetric(appliedAgg)); + } + } + } else { + metricName = 'Unset'; + } } else { - metricName = 'Unset'; + const appliedAgg = _.find(target.metrics, { id: series.field }); + if (appliedAgg) { + metricName += ' ' + queryDef.describeMetric(appliedAgg); + } else { + metricName = 'Unset'; + } } } else if (series.field) { metricName += ' ' + series.field; diff --git a/public/app/plugins/datasource/elasticsearch/metric_agg.ts b/public/app/plugins/datasource/elasticsearch/metric_agg.ts index cae5be45720..735b4b4f7f9 100644 --- a/public/app/plugins/datasource/elasticsearch/metric_agg.ts +++ b/public/app/plugins/datasource/elasticsearch/metric_agg.ts @@ -35,11 +35,20 @@ export class ElasticMetricAggCtrl { $scope.isFirst = $scope.index === 0; $scope.isSingle = metricAggs.length === 1; $scope.settingsLinkText = ''; + $scope.variablesLinkText = ''; $scope.aggDef = _.find($scope.metricAggTypes, { value: $scope.agg.type }); if (queryDef.isPipelineAgg($scope.agg.type)) { - $scope.agg.pipelineAgg = $scope.agg.pipelineAgg || 'select metric'; - $scope.agg.field = $scope.agg.pipelineAgg; + if (queryDef.isPipelineAggWithMultipleBucketPaths($scope.agg.type)) { + $scope.variablesLinkText = 'Options'; + + if ($scope.agg.settings.script) { + $scope.variablesLinkText = 'Script: ' + $scope.agg.settings.script.replace(new RegExp('params.', 'g'), ''); + } + } else { + $scope.agg.pipelineAgg = $scope.agg.pipelineAgg || 'select metric'; + $scope.agg.field = $scope.agg.pipelineAgg; + } const pipelineOptions = queryDef.getPipelineOptions($scope.agg); if (pipelineOptions.length > 0) { @@ -119,6 +128,10 @@ export class ElasticMetricAggCtrl { $scope.updatePipelineAggOptions(); }; + $scope.toggleVariables = () => { + $scope.showVariables = !$scope.showVariables; + }; + $scope.onChangeInternal = () => { $scope.onChange(); }; @@ -152,6 +165,7 @@ export class ElasticMetricAggCtrl { $scope.target.bucketAggs = [queryDef.defaultBucketAgg()]; } + $scope.showVariables = queryDef.isPipelineAggWithMultipleBucketPaths($scope.agg.type); $scope.updatePipelineAggOptions(); $scope.onChange(); }; diff --git a/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html b/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html index b4a1a61ed32..362b0ddb486 100644 --- a/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html +++ b/public/app/plugins/datasource/elasticsearch/partials/metric_agg.html @@ -13,7 +13,17 @@
- + +
+ +
+
@@ -36,6 +46,20 @@
+
+ +
+ + +
+
+
@@ -103,5 +127,5 @@ The missing parameter defines how documents that are missing a value should be treated. By default they will be ignored but it is also possible to treat them as if they had a value -
+
diff --git a/public/app/plugins/datasource/elasticsearch/partials/pipeline_variables.html b/public/app/plugins/datasource/elasticsearch/partials/pipeline_variables.html new file mode 100644 index 00000000000..66bf5c8467b --- /dev/null +++ b/public/app/plugins/datasource/elasticsearch/partials/pipeline_variables.html @@ -0,0 +1,20 @@ +
+
+ + + + + +
+
+ + + +
+
diff --git a/public/app/plugins/datasource/elasticsearch/pipeline_variables.ts b/public/app/plugins/datasource/elasticsearch/pipeline_variables.ts new file mode 100644 index 00000000000..94e0e78d686 --- /dev/null +++ b/public/app/plugins/datasource/elasticsearch/pipeline_variables.ts @@ -0,0 +1,45 @@ +import coreModule from 'app/core/core_module'; +import _ from 'lodash'; + +export function elasticPipelineVariables() { + return { + templateUrl: 'public/app/plugins/datasource/elasticsearch/partials/pipeline_variables.html', + controller: 'ElasticPipelineVariablesCtrl', + restrict: 'E', + scope: { + onChange: '&', + variables: '=', + options: '=', + }, + }; +} + +const newVariable = index => { + return { + name: 'var' + index, + pipelineAgg: 'select metric', + }; +}; + +export class ElasticPipelineVariablesCtrl { + constructor($scope) { + $scope.variables = $scope.variables || [newVariable(1)]; + + $scope.onChangeInternal = () => { + $scope.onChange(); + }; + + $scope.add = () => { + $scope.variables.push(newVariable($scope.variables.length + 1)); + $scope.onChange(); + }; + + $scope.remove = index => { + $scope.variables.splice(index, 1); + $scope.onChange(); + }; + } +} + +coreModule.directive('elasticPipelineVariables', elasticPipelineVariables); +coreModule.controller('ElasticPipelineVariablesCtrl', ElasticPipelineVariablesCtrl); diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index cff7b5672f6..7d4d23aa6f0 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -189,7 +189,7 @@ export class ElasticQueryBuilder { target.bucketAggs = target.bucketAggs || [queryDef.defaultBucketAgg()]; target.timeField = this.timeField; - let i, nestedAggs, metric; + let i, j, pv, nestedAggs, metric; const query = { size: 0, query: { @@ -269,17 +269,42 @@ export class ElasticQueryBuilder { let metricAgg = null; if (queryDef.isPipelineAgg(metric.type)) { - if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) { - const appliedAgg = queryDef.findMetricById(target.metrics, metric.pipelineAgg); - if (appliedAgg) { - if (appliedAgg.type === 'count') { - metricAgg = { buckets_path: '_count' }; - } else { - metricAgg = { buckets_path: metric.pipelineAgg }; + if (queryDef.isPipelineAggWithMultipleBucketPaths(metric.type)) { + if (metric.pipelineVariables) { + metricAgg = { + buckets_path: {}, + }; + + for (j = 0; j < metric.pipelineVariables.length; j++) { + pv = metric.pipelineVariables[j]; + + if (pv.name && pv.pipelineAgg && /^\d*$/.test(pv.pipelineAgg)) { + const appliedAgg = queryDef.findMetricById(target.metrics, pv.pipelineAgg); + if (appliedAgg) { + if (appliedAgg.type === 'count') { + metricAgg.buckets_path[pv.name] = '_count'; + } else { + metricAgg.buckets_path[pv.name] = pv.pipelineAgg; + } + } + } } + } else { + continue; } } else { - continue; + if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) { + const appliedAgg = queryDef.findMetricById(target.metrics, metric.pipelineAgg); + if (appliedAgg) { + if (appliedAgg.type === 'count') { + metricAgg = { buckets_path: '_count' }; + } else { + metricAgg = { buckets_path: metric.pipelineAgg }; + } + } + } else { + continue; + } } } else { metricAgg = { field: metric.field }; diff --git a/public/app/plugins/datasource/elasticsearch/query_ctrl.ts b/public/app/plugins/datasource/elasticsearch/query_ctrl.ts index 81dbd5b8f8a..40420d7c6a3 100644 --- a/public/app/plugins/datasource/elasticsearch/query_ctrl.ts +++ b/public/app/plugins/datasource/elasticsearch/query_ctrl.ts @@ -1,5 +1,6 @@ import './bucket_agg'; import './metric_agg'; +import './pipeline_variables'; import angular from 'angular'; import _ from 'lodash'; @@ -70,6 +71,9 @@ export class ElasticQueryCtrl extends QueryCtrl { if (aggDef.requiresField) { text += metric.field; } + if (aggDef.supportsMultipleBucketPaths) { + text += metric.settings.script.replace(new RegExp('params.', 'g'), ''); + } text += '), '; }); diff --git a/public/app/plugins/datasource/elasticsearch/query_def.ts b/public/app/plugins/datasource/elasticsearch/query_def.ts index ae9a453b1fb..13797853a77 100644 --- a/public/app/plugins/datasource/elasticsearch/query_def.ts +++ b/public/app/plugins/datasource/elasticsearch/query_def.ts @@ -64,6 +64,14 @@ export const metricAggTypes = [ isPipelineAgg: true, minVersion: 2, }, + { + text: 'Bucket Script', + value: 'bucket_script', + requiresField: false, + isPipelineAgg: true, + supportsMultipleBucketPaths: true, + minVersion: 2, + }, { text: 'Raw Document', value: 'raw_document', requiresField: false }, ]; @@ -128,6 +136,7 @@ export const pipelineOptions = { { text: 'minimize', default: false }, ], derivative: [{ text: 'unit', default: undefined }], + bucket_script: [], }; export const movingAvgModelSettings = { @@ -171,6 +180,14 @@ export function isPipelineAgg(metricType) { return false; } +export function isPipelineAggWithMultipleBucketPaths(metricType) { + if (metricType) { + return metricAggTypes.find(t => t.value === metricType && t.supportsMultipleBucketPaths) !== undefined; + } + + return false; +} + export function getPipelineAggOptions(targets) { const result = []; _.each(targets.metrics, metric => { diff --git a/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts b/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts index 8b41e71145a..bedc71a0b58 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts @@ -665,4 +665,70 @@ describe('ElasticResponse', () => { expect(result.data[0].datapoints[0].fieldProp).toBe('field'); }); }); + + describe('with bucket_script ', () => { + let result; + + beforeEach(() => { + targets = [ + { + refId: 'A', + metrics: [ + { id: '1', type: 'sum', field: '@value' }, + { id: '3', type: 'max', field: '@value' }, + { + id: '4', + field: 'select field', + pipelineVariables: [{ name: 'var1', pipelineAgg: '1' }, { name: 'var2', pipelineAgg: '3' }], + settings: { script: 'params.var1 * params.var2' }, + type: 'bucket_script', + }, + ], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }], + }, + ]; + response = { + responses: [ + { + aggregations: { + '2': { + buckets: [ + { + 1: { value: 2 }, + 3: { value: 3 }, + 4: { value: 6 }, + doc_count: 60, + key: 1000, + }, + { + 1: { value: 3 }, + 3: { value: 4 }, + 4: { value: 12 }, + doc_count: 60, + key: 2000, + }, + ], + }, + }, + }, + ], + }; + + result = new ElasticResponse(targets, response).getTimeSeries(); + }); + + it('should return 3 series', () => { + expect(result.data.length).toBe(3); + expect(result.data[0].datapoints.length).toBe(2); + expect(result.data[0].target).toBe('Sum @value'); + expect(result.data[1].target).toBe('Max @value'); + expect(result.data[2].target).toBe('Sum @value * Max @value'); + expect(result.data[0].datapoints[0][0]).toBe(2); + expect(result.data[1].datapoints[0][0]).toBe(3); + expect(result.data[2].datapoints[0][0]).toBe(6); + expect(result.data[0].datapoints[1][0]).toBe(3); + expect(result.data[1].datapoints[1][0]).toBe(4); + expect(result.data[2].datapoints[1][0]).toBe(12); + }); + }); }); diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts index 4cd7deddb67..993eaccfbe2 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts @@ -353,6 +353,83 @@ describe('ElasticQueryBuilder', () => { expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('_count'); }); + it('with bucket_script', () => { + const query = builder.build({ + metrics: [ + { + id: '1', + type: 'sum', + field: '@value', + }, + { + id: '3', + type: 'max', + field: '@value', + }, + { + field: 'select field', + id: '4', + meta: {}, + pipelineVariables: [ + { + name: 'var1', + pipelineAgg: '1', + }, + { + name: 'var2', + pipelineAgg: '3', + }, + ], + settings: { + script: 'params.var1 * params.var2', + }, + type: 'bucket_script', + }, + ], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }], + }); + + const firstLevel = query.aggs['2']; + + expect(firstLevel.aggs['4']).not.toBe(undefined); + expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined); + expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '1', var2: '3' }); + }); + + it('with bucket_script doc count', () => { + const query = builder.build({ + metrics: [ + { + id: '3', + type: 'count', + field: 'select field', + }, + { + field: 'select field', + id: '4', + meta: {}, + pipelineVariables: [ + { + name: 'var1', + pipelineAgg: '3', + }, + ], + settings: { + script: 'params.var1 * 1000', + }, + type: 'bucket_script', + }, + ], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }], + }); + + const firstLevel = query.aggs['2']; + + expect(firstLevel.aggs['4']).not.toBe(undefined); + expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined); + expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '_count' }); + }); + it('with histogram', () => { const query = builder.build({ metrics: [{ id: '1', type: 'count' }], diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts b/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts index 471d400037c..f3d1874338f 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts @@ -65,6 +65,24 @@ describe('ElasticQueryDef', () => { }); }); + describe('isPipelineAggWithMultipleBucketPaths', () => { + describe('bucket_script', () => { + const result = queryDef.isPipelineAggWithMultipleBucketPaths('bucket_script'); + + test('should have multiple bucket paths support', () => { + expect(result).toBe(true); + }); + }); + + describe('moving_avg', () => { + const result = queryDef.isPipelineAggWithMultipleBucketPaths('moving_avg'); + + test('should not have multiple bucket paths support', () => { + expect(result).toBe(false); + }); + }); + }); + describe('pipeline aggs depending on esverison', () => { describe('using esversion undefined', () => { test('should not get pipeline aggs', () => { @@ -80,13 +98,13 @@ describe('ElasticQueryDef', () => { describe('using esversion 2', () => { test('should get pipeline aggs', () => { - expect(queryDef.getMetricAggTypes(2).length).toBe(11); + expect(queryDef.getMetricAggTypes(2).length).toBe(12); }); }); describe('using esversion 5', () => { test('should get pipeline aggs', () => { - expect(queryDef.getMetricAggTypes(5).length).toBe(11); + expect(queryDef.getMetricAggTypes(5).length).toBe(12); }); }); });