feat(elasticsearch): lots of work on elasticsearch metrics processing, handling grouped responses, etc, #1034

public/app/panels/graph/module.js

@@ -153,6 +153,8 @@ function (angular, app, $, _, kbn, moment, TimeSeries, PanelMeta) {
         return;
       }
 
+      console.log('graph data', results);
+
       $scope.datapointsWarning = false;
       $scope.datapointsCount = 0;
       $scope.datapointsOutside = false;

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -13,7 +13,7 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
 
   var module = angular.module('grafana.services');
 
-  module.factory('ElasticDatasource', function($q, backendSrv, templateSrv) {
+  module.factory('ElasticDatasource', function($q, backendSrv, templateSrv, timeSrv) {
 
     function ElasticDatasource(datasource) {
       this.type = 'elasticsearch';
@@ -160,7 +160,7 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
       payload = payload.replace(/\$maxDataPoints/g, options.maxDataPoints);
       payload = templateSrv.replace(payload, options.scopedVars);
 
-      var processTimeSeries = _.partial(this._processTimeSeries, sentTargets);
+      var processTimeSeries = _.bind(this._processTimeSeries, this, sentTargets);
       return this._post('/_msearch?search_type=count', payload).then(processTimeSeries);
     };
 
@@ -172,54 +172,109 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
       return date.getTime();
     };
 
-    ElasticDatasource._aggToSeries = function(agg) {
-      var datapoints = agg.date_histogram.buckets.map(function(entry) {
-        return [entry.stats.avg, entry.key];
-      });
-      return { target: agg.key, datapoints: datapoints };
-    };
+    ElasticDatasource.prototype._processBuckets = function(buckets, groupByFields, series, level, parentName, parentTime) {
+      var points = [];
+      var groupBy = groupByFields[level];
+
+      for (var i = 0; i < buckets.length; i++) {
+        var bucket = buckets[i];
+
+        if (groupBy) {
+          var seriesName = "";
+          var time = parentTime || bucket.key;
+          this._processBuckets(bucket[groupBy.field].buckets, groupByFields, series, level+1, seriesName, time)
+        } else {
+          var seriesName = parentName;
 
+          if (level > 0) {
+            seriesName += bucket.key;
+          } else {
+            parentTime = bucket.key;
+          }
+
+          var serie = series[seriesName] = series[seriesName] || {target: seriesName, datapoints: []};
+          serie.datapoints.push([bucket.doc_count, parentTime]);
+        }
+      }
+    };
 
     ElasticDatasource.prototype._processTimeSeries = function(targets, results) {
       var series = [];
 
-      _.each(results.responses, function(response, index) {
-        var buckets = response.aggregations.date_histogram.buckets;
-        var target = targets[index];
+      for (var i = 0; i < results.responses.length; i++) {
+        var buckets = results.responses[i].aggregations.histogram.buckets;
+        var target = targets[i];
         var points = [];
+        var querySeries = {}
 
-        for (var i = 0; i < buckets.length; i++) {
-          var bucket = buckets[i];
-          points[i] = [bucket.doc_count, bucket.key];
-        }
+        this._processBuckets(buckets, target.groupByFields, querySeries, 0, target.refId);
 
-        series.push({target: 'name', datapoints: points})
-        console.log('Nr DataPoints: ' + points.length);
-      });
-
-      console.log(series);
+        _.each(querySeries, function(value) {
+          series.push(value);
+        });
+      };
 
       return { data: series };
     };
 
     ElasticDatasource.prototype.metricFindQuery = function(query) {
-      var region;
-      var namespace;
-      var metricName;
-
-      var transformSuggestData = function(suggestData) {
-        return _.map(suggestData, function(v) {
-          return { text: v };
-        });
+      var timeFrom = this.translateTime(timeSrv.time.from);
+      var timeTo = this.translateTime(timeSrv.time.to);
+
+      var query = {
+        size: 10,
+        "query": {
+          "filtered": {
+            "filter": {
+              "bool": {
+                "must": [
+                  {
+                    "range": {
+                      "@timestamp": {
+                        "gte": timeFrom,
+                        "lte": timeTo
+                      }
+                    }
+                  }
+                ],
+              }
+            }
+          }
+        }
       };
 
-      var d = $q.defer();
+      return this._post('/_search?', query).then(function(res) {
+        var fields = {};
 
-      var regionQuery = query.match(/^region\(\)/);
-      if (regionQuery) {
-        d.resolve(transformSuggestData(this.performSuggestRegion()));
-        return d.promise;
-      }
+        for (var i = 0; i < res.hits.hits.length; i++) {
+          var hit = res.hits.hits[i];
+          for (var field in hit) {
+            if (hit.hasOwnProperty(field) && field[0] !== '_') {
+              fields[field] = 1;
+            }
+          }
+
+          if (hit._source) {
+            for (var field in hit._source) {
+              if (hit._source.hasOwnProperty(field)) {
+                fields[field] = 1;
+              }
+            }
+          }
+        }
+
+        fields = _.map(_.keys(fields), function(field) {
+          return {text: field};
+        })
+        console.log('metricFindQuery:',  fields);
+        return fields;
+      });
+      // var d = $q.defer();
+      //
+      // var fieldsQuery = query.match(/^fields\(\)/);
+      // if (fieldsQuery) {
+      //   return d.promise;
+      // }
     };
 
     return ElasticDatasource;

public/app/plugins/datasource/elasticsearch/partials/query.editor.html

@@ -77,11 +77,10 @@
 		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
 			Group By
 		</li>
-		<li>
-			<metric-segment segment="groupByFieldSegment" on-value-changed="groupByFieldChanged()"></metric-segment>
+		<li ng-repeat="segment in groupBySegments">
+			<metric-segment segment="segment" get-alt-segments="getGroupByFields(segment, $index)" on-value-changed="groupByChanged(segment, $index)"></metric-segment>
 		</li>
 	</ul>
-
 	<div class="clearfix"></div>
 </div>
 

public/app/plugins/datasource/elasticsearch/queryBuilder.js

@@ -42,7 +42,7 @@ function () {
     };
 
     query.aggs = {
-      "date_histogram": {
+      "histogram": {
         "date_histogram": {
           "interval": target.interval || "$interval",
           "field": target.timeField,
@@ -55,6 +55,19 @@ function () {
       }
     };
 
+    var nestedAggs = {};
+
+    if (target.groupByFields && target.groupByFields.length > 0) {
+      var field = target.groupByFields[0].field;
+      nestedAggs[field] = {
+        terms:  {
+          field: field,
+        }
+      }
+    }
+
+    query.aggs.histogram.aggs = nestedAggs;
+
     query = JSON.stringify(query);
     return query;
   };

public/app/plugins/datasource/elasticsearch/queryCtrl.js

@@ -23,18 +23,25 @@ function (angular, _, ElasticQueryBuilder) {
       target.function = target.function || 'mean';
       target.timeField = target.timeField || '@timestamp';
       target.select = target.select || [{ agg: 'count' }];
+      target.groupByFields = target.groupByFields || [];
 
       $scope.timeSegment = uiSegmentSrv.newSegment(target.timeField);
-      $scope.groupByFieldSegment = uiSegmentSrv.getSegmentForValue(target.groupByField, 'add group by');
+
+      $scope.groupBySegments = _.map(target.groupByFields, function(group) {
+        return uiSegmentSrv.newSegment(group.field);
+      });
 
       $scope.selectSegments = _.map(target.select, function(select) {
         return uiSegmentSrv.newSegment(select.agg);
       });
+
+      $scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
+      $scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'});
+      $scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'});
     };
 
     $scope.getFields = function() {
-      return $scope.datasource.metricFindQuery('fields()')
-        .then($scope.transformToSegments(true));
+      return $scope.datasource.metricFindQuery('fields()').then($scope.transformToSegments(true));
     };
 
     $scope.transformToSegments = function(addTemplateVars) {
@@ -53,6 +60,36 @@ function (angular, _, ElasticQueryBuilder) {
       };
     };
 
+    $scope.getGroupByFields = function(segment) {
+      return $scope.datasource.metricFindQuery('fields()').then($scope.transformToSegments(false))
+      .then(function(results) {
+        if (segment.type !== 'plus-button') {
+          results.splice(0, 0, angular.copy($scope.removeGroupBySegment));
+        }
+        return results;
+      })
+      .then(null, $scope.handleQueryError);
+    };
+
+    $scope.groupByChanged = function(segment, index) {
+      if (segment.value === $scope.removeGroupBySegment.value) {
+        $scope.target.groupByFields.splice(index, 1);
+        $scope.groupBySegments.splice(index, 1);
+        $scope.$parent.get_data();
+        return;
+      }
+
+      if (index === $scope.groupBySegments.length-1) {
+        $scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
+      }
+
+      segment.type = 'group-by-key';
+      segment.fake = false;
+
+      $scope.target.groupByFields[index] = {field: segment.value};
+      $scope.$parent.get_data();
+    };
+
     $scope.valueFieldChanged = function() {
       $scope.target.valueField = $scope.valueFieldSegment.value;
       $scope.$parent.get_data();
@@ -88,19 +125,6 @@ function (angular, _, ElasticQueryBuilder) {
       return [];
     };
 
-    $scope.transformToSegments = function(results) {
-      return _.map(results, function(segment) {
-        return new MetricSegment({ value: segment.text, expandable: segment.expandable });
-      });
-    };
-
-    $scope.addTemplateVariableSegments = function(segments) {
-      _.each(templateSrv.variables, function(variable) {
-        segments.unshift(new MetricSegment({ type: 'template', value: '$' + variable.name, expandable: true }));
-      });
-      return segments;
-    };
-
     $scope.toggleQueryMode = function () {
       $scope.target.rawQuery = !$scope.target.rawQuery;
     };

public/test/specs/elasticsearch-specs.js

@@ -0,0 +1,108 @@
+define([
+  'helpers',
+  'plugins/datasource/elasticsearch/datasource',
+  'aws-sdk',
+], function(helpers) {
+  'use strict';
+
+  describe('ElasticDatasource', function() {
+    var ctx = new helpers.ServiceTestContext();
+
+    beforeEach(module('grafana.services'));
+    beforeEach(ctx.providePhase(['templateSrv']));
+    beforeEach(ctx.createService('ElasticDatasource'));
+    beforeEach(function() {
+      ctx.ds = new ctx.service({});
+    });
+
+    describe('When processing es response', function() {
+
+      describe('simple query', function() {
+        var result;
+
+        beforeEach(function() {
+          result = ctx.ds._processTimeSeries([{
+            refId: 'A',
+            groupByFields: [],
+          }], {
+            responses: [{
+              aggregations: {
+                histogram: {
+                  buckets: [
+                    {
+                      doc_count: 10,
+                      key: 1000
+                    },
+                    {
+                      doc_count: 15,
+                      key: 2000
+                    }
+                  ]
+                }
+              }
+            }]
+          })
+        });
+
+        it('should return 1 series', function() {
+          expect(result.data.length).to.be(1);
+          expect(result.data[0].datapoints.length).to.be(2);
+          expect(result.data[0].datapoints[0][0]).to.be(10);
+          expect(result.data[0].datapoints[0][1]).to.be(1000);
+        });
+
+      });
+
+      describe('single group by query', function() {
+        var result;
+
+        beforeEach(function() {
+          result = ctx.ds._processTimeSeries([
+            {
+              refId: 'A',
+              groupByFields: [
+                {field: 'host' }
+              ]
+            }
+          ], {
+            responses: [{
+              aggregations: {
+                histogram: {
+                  buckets: [
+                    {
+                      host: {
+                        buckets: [
+                           {doc_count: 4, key: 'server1'},
+                           {doc_count: 6, key: 'server2'},
+                        ]
+                      },
+                      doc_count: 10,
+                      key: 1000
+                    },
+                    {
+                      host: {
+                        buckets: [
+                           {doc_count: 4, key: 'server1'},
+                           {doc_count: 6, key: 'server2'},
+                        ]
+                      },
+                      doc_count: 15,
+                      key: 2000
+                    }
+                  ]
+                }
+              }
+            }]
+          })
+        });
+
+        it('should return 2 series', function() {
+          expect(result.data.length).to.be(2);
+          expect(result.data[0].datapoints.length).to.be(2);
+          expect(result.data[0].target).to.be('server1');
+          expect(result.data[1].target).to.be('server2');
+        });
+      });
+    });
+  });
+});

public/test/test-main.js

@@ -152,6 +152,7 @@ require([
     'specs/valueSelectDropdown-specs',
     'specs/opentsdbDatasource-specs',
     'specs/cloudwatch-datasource-specs',
+    'specs/elasticsearch-specs',
   ];
 
   var pluginSpecs = (config.plugins.specs || []).map(function (spec) {