From 3be84b00d57b2c451caed41eb2884d9a0894794d Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Thu, 15 Sep 2016 08:30:08 +0300 Subject: [PATCH] ES nested fields autocomplete (#6043) * Re-create PR #4527 from @arcolife: fixes #4526 - add nested support to fieldname autocomplete; Also: - update _mapping to first use given time range for deducting index mapping, then fallback to today's date based index name * (elasticsearch): refactor getFields() method. * (elasticsearch): add tests for getFields() method. * (elasticsearch): fixed _get() method (tests was broken after @arcolife commit). --- .../datasource/elasticsearch/datasource.js | 69 +++++++++---- .../elasticsearch/specs/datasource_specs.ts | 99 ++++++++++++++++++- 2 files changed, 148 insertions(+), 20 deletions(-) diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index c952bf4d4f6..410cb8fb736 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -47,10 +47,19 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes }; this._get = function(url) { - return this._request('GET', this.indexPattern.getIndexForToday() + url).then(function(results) { - results.data.$$config = results.config; - return results.data; - }); + var range = timeSrv.timeRange(); + var index_list = this.indexPattern.getIndexList(range.from.valueOf(), range.to.valueOf()); + if (_.isArray(index_list) && index_list.length) { + return this._request('GET', index_list[0] + url).then(function(results) { + results.data.$$config = results.config; + return results.data; + }); + } else { + return this._request('GET', this.indexPattern.getIndexForToday() + url).then(function(results) { + results.data.$$config = results.config; + return results.data; + }); + } }; this._post = function(url, data) { @@ -210,8 +219,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes } this.getFields = function(query) { - return this._get('/_mapping').then(function(res) { - var fields = {}; + return this._get('/_mapping').then(function(result) { var typeMap = { 'float': 'number', 'double': 'number', @@ -219,24 +227,47 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes 'long': 'number', 'date': 'date', 'string': 'string', + 'nested': 'nested' }; - for (var indexName in res) { - var index = res[indexName]; - var mappings = index.mappings; - if (!mappings) { continue; } - for (var typeName in mappings) { - var properties = mappings[typeName].properties; - for (var field in properties) { - var prop = properties[field]; - if (query.type && typeMap[prop.type] !== query.type) { - continue; - } - if (prop.type && field[0] !== '_') { - fields[field] = {text: field, type: prop.type}; + // Store subfield names: [system, process, cpu, total] -> system.process.cpu.total + var fieldNameParts = []; + var fields = {}; + function getFieldsRecursively(obj) { + for (var key in obj) { + var subObj = obj[key]; + + // Check mapping field for nested fields + if (subObj.hasOwnProperty('properties')) { + fieldNameParts.push(key); + getFieldsRecursively(subObj.properties); + } else { + var fieldName = fieldNameParts.concat(key).join('.'); + + // Hide meta-fields and check field type + if (key[0] !== '_' && + (!query.type || + query.type && typeMap[subObj.type] === query.type)) { + + fields[fieldName] = { + text: fieldName, + type: subObj.type + }; } } } + fieldNameParts.pop(); + } + + for (var indexName in result) { + var index = result[indexName]; + if (index && index.mappings) { + var mappings = index.mappings; + for (var typeName in mappings) { + var properties = mappings[typeName].properties; + getFieldsRecursively(properties); + } + } } // transform to array diff --git a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts index 7358249793e..43b295a3d64 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts @@ -1,4 +1,4 @@ - +import _ from 'lodash'; import {describe, beforeEach, it, sinon, expect, angularMocks} from 'test/lib/common'; import moment from 'moment'; import angular from 'angular'; @@ -112,4 +112,101 @@ describe('ElasticDatasource', function() { }); }); + describe('When getting fields', function() { + var requestOptions, parts, header; + + beforeEach(function() { + createDatasource({url: 'http://es.com', index: 'metricbeat'}); + + ctx.backendSrv.datasourceRequest = function(options) { + requestOptions = options; + return ctx.$q.when({data: { + metricbeat: { + mappings: { + metricsets: { + _all: {}, + properties: { + '@timestamp': {type: 'date'}, + beat: { + properties: { + name: {type: 'string'}, + hostname: {type: 'string'}, + } + }, + system: { + properties: { + cpu: { + properties: { + system: {type: 'float'}, + user: {type: 'float'}, + } + }, + process: { + properties: { + cpu: { + properties: { + total: {type: 'float'} + } + }, + name: {type: 'string'}, + } + }, + } + } + } + } + } + } + }}); + }; + }); + + it('should return nested fields', function() { + ctx.ds.getFields({ + find: 'fields', + query: '*' + }).then((fieldObjects) => { + var fields = _.map(fieldObjects, 'text'); + expect(fields).to.eql([ + '@timestamp', + 'beat.name', + 'beat.hostname', + 'system.cpu.system', + 'system.cpu.user', + 'system.process.cpu.total', + 'system.process.name' + ]); + }); + ctx.$rootScope.$apply(); + }); + + it('should return fields related to query type', function() { + ctx.ds.getFields({ + find: 'fields', + query: '*', + type: 'number' + }).then((fieldObjects) => { + var fields = _.map(fieldObjects, 'text'); + expect(fields).to.eql([ + 'system.cpu.system', + 'system.cpu.user', + 'system.process.cpu.total' + ]); + }); + + ctx.ds.getFields({ + find: 'fields', + query: '*', + type: 'date' + }).then((fieldObjects) => { + var fields = _.map(fieldObjects, 'text'); + expect(fields).to.eql([ + '@timestamp' + ]); + }); + + ctx.$rootScope.$apply(); + }); + }); + });