diff --git a/packages/grafana-data/src/dataframe/MutableDataFrame.test.ts b/packages/grafana-data/src/dataframe/MutableDataFrame.test.ts index 7e00dd1e594..65094078cd2 100644 --- a/packages/grafana-data/src/dataframe/MutableDataFrame.test.ts +++ b/packages/grafana-data/src/dataframe/MutableDataFrame.test.ts @@ -58,7 +58,7 @@ describe('Apending DataFrame', () => { ]); // Add a time value that has an array type - frame.add({ time: [300] }); + frame.add({ time: 300 }); expect(frame.toArray()).toEqual([ { time: 100, name: 'a', value: 1, value2: null }, // 1 { time: 200, name: 'BB', value: 20, value2: null }, // 2 diff --git a/packages/grafana-data/src/dataframe/MutableDataFrame.ts b/packages/grafana-data/src/dataframe/MutableDataFrame.ts index 834f7aa96a0..96a2a38ec82 100644 --- a/packages/grafana-data/src/dataframe/MutableDataFrame.ts +++ b/packages/grafana-data/src/dataframe/MutableDataFrame.ts @@ -231,11 +231,6 @@ export class MutableDataFrame implements DataFrame, MutableVector { field.parse = makeFieldParser(val, field); } val = field.parse(val); - } else if (field.type === FieldType.time && isArray(val)) { - if (!field.parse) { - field.parse = (val: any[]) => val[0] || undefined; - } - val = field.parse(val); } if (val === undefined) { diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.ts b/public/app/plugins/datasource/elasticsearch/elastic_response.ts index 9ba53f73d12..0216c852e44 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.ts +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.ts @@ -424,40 +424,26 @@ export class ElasticResponse { throw this.getErrorFromElasticResponse(this.response, response.error); } - const hits = response.hits; + // We keep a list of all props so that we can create all the fields in the dataFrame, this can lead + // to wide sparse dataframes in case the scheme is different per document. let propNames: string[] = []; - let propName, hit, doc: any, i; - for (i = 0; i < hits.hits.length; i++) { - hit = hits.hits[i]; + for (const hit of response.hits.hits) { const flattened = hit._source ? flatten(hit._source, null) : {}; - doc = {}; - doc[this.targets[0].timeField] = null; - doc = { - ...doc, + const doc = { _id: hit._id, _type: hit._type, _index: hit._index, + _source: { ...flattened }, ...flattened, }; - // Note: the order of for...in is arbitrary amd implementation dependant - // and should probably not be relied upon. - for (propName in hit.fields) { + for (const propName of Object.keys(doc)) { if (propNames.indexOf(propName) === -1) { propNames.push(propName); } - doc[propName] = hit.fields[propName]; } - for (propName in doc) { - if (propNames.indexOf(propName) === -1) { - propNames.push(propName); - } - } - - doc._source = { ...flattened }; - docs.push(doc); } @@ -468,9 +454,7 @@ export class ElasticResponse { series.addField({ name: this.targets[0].timeField, type: FieldType.time, - }).parse = (v: any) => { - return v[0] || ''; - }; + }); if (logMessageField) { series.addField({ diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.ts b/public/app/plugins/datasource/elasticsearch/query_builder.ts index e857202edd0..570a9b19c33 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.ts +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -5,7 +5,7 @@ export class ElasticQueryBuilder { timeField: string; esVersion: number; - constructor(options: any) { + constructor(options: { timeField: string; esVersion: number }) { this.timeField = options.timeField; this.esVersion = options.esVersion; } @@ -129,11 +129,6 @@ export class ElasticQueryBuilder { } query.script_fields = {}; - if (this.esVersion < 5) { - query.fielddata_fields = [this.timeField]; - } else { - query.docvalue_fields = [this.timeField]; - } return query; } diff --git a/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts b/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts index fffcedfbba8..7beecd63a14 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts @@ -871,9 +871,6 @@ describe('ElasticResponse', () => { host: 'djisaodjsoad', message: 'hello, i am a message', }, - fields: { - '@timestamp': ['2019-06-24T09:51:19.765Z'], - }, }, { _id: 'kdospaidopa', @@ -884,9 +881,6 @@ describe('ElasticResponse', () => { host: 'dsalkdakdop', message: 'hello, i am also message', }, - fields: { - '@timestamp': ['2019-06-24T09:52:19.765Z'], - }, }, ], }, diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts index 645fde00acf..83462888668 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts @@ -1,573 +1,507 @@ import { ElasticQueryBuilder } from '../query_builder'; describe('ElasticQueryBuilder', () => { - let builder: any; - - beforeEach(() => { - builder = new ElasticQueryBuilder({ timeField: '@timestamp' }); - }); - - it('with defaults', () => { - const query = builder.build({ - metrics: [{ type: 'Count', id: '0' }], - timeField: '@timestamp', - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }], - }); - - expect(query.query.bool.filter[0].range['@timestamp'].gte).toBe('$timeFrom'); - expect(query.aggs['1'].date_histogram.extended_bounds.min).toBe('$timeFrom'); - }); - - it('with defaults on es5.x', () => { - const builder5x = new ElasticQueryBuilder({ - timeField: '@timestamp', - esVersion: 5, - }); - - const query = builder5x.build({ - metrics: [{ type: 'Count', id: '0' }], - timeField: '@timestamp', - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }], - }); - - expect(query.query.bool.filter[0].range['@timestamp'].gte).toBe('$timeFrom'); - expect(query.aggs['1'].date_histogram.extended_bounds.min).toBe('$timeFrom'); - }); - - it('with multiple bucket aggs', () => { - const query = builder.build({ - metrics: [{ type: 'count', id: '1' }], - timeField: '@timestamp', - bucketAggs: [ - { type: 'terms', field: '@host', id: '2' }, - { type: 'date_histogram', field: '@timestamp', id: '3' }, - ], - }); - - expect(query.aggs['2'].terms.field).toBe('@host'); - expect(query.aggs['2'].aggs['3'].date_histogram.field).toBe('@timestamp'); - }); + const builder = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 2 }); + const builder5x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 5 }); + const builder56 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 56 }); + const builder6x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 60 }); + const builder7x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 70 }); + + const allBuilders = [builder, builder5x, builder56, builder6x, builder7x]; + + allBuilders.forEach(builder => { + describe(`version ${builder.esVersion}`, () => { + it('should return query with defaults', () => { + const query = builder.build({ + metrics: [{ type: 'Count', id: '0' }], + timeField: '@timestamp', + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }], + }); + + expect(query.query.bool.filter[0].range['@timestamp'].gte).toBe('$timeFrom'); + expect(query.aggs['1'].date_histogram.extended_bounds.min).toBe('$timeFrom'); + }); + + it('with multiple bucket aggs', () => { + const query = builder.build({ + metrics: [{ type: 'count', id: '1' }], + timeField: '@timestamp', + bucketAggs: [ + { type: 'terms', field: '@host', id: '2' }, + { type: 'date_histogram', field: '@timestamp', id: '3' }, + ], + }); - it('with select field', () => { - const query = builder.build( - { - metrics: [{ type: 'avg', field: '@value', id: '1' }], - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }], - }, - 100, - 1000 - ); - - const aggs = query.aggs['2'].aggs; - expect(aggs['1'].avg.field).toBe('@value'); - }); + expect(query.aggs['2'].terms.field).toBe('@host'); + expect(query.aggs['2'].aggs['3'].date_histogram.field).toBe('@timestamp'); + }); - it('with term agg and order by term', () => { - const query = builder.build( - { - metrics: [ - { type: 'count', id: '1' }, - { type: 'avg', field: '@value', id: '5' }, - ], - bucketAggs: [ + it('with select field', () => { + const query = builder.build( { - type: 'terms', - field: '@host', - settings: { size: 5, order: 'asc', orderBy: '_term' }, - id: '2', + metrics: [{ type: 'avg', field: '@value', id: '1' }], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }], }, - { type: 'date_histogram', field: '@timestamp', id: '3' }, - ], - }, - 100, - 1000 - ); - - const firstLevel = query.aggs['2']; - expect(firstLevel.terms.order._term).toBe('asc'); - }); - - it('with term agg and order by term on es6.x', () => { - const builder6x = new ElasticQueryBuilder({ - timeField: '@timestamp', - esVersion: 60, - }); - const query = builder6x.build( - { - metrics: [ - { type: 'count', id: '1' }, - { type: 'avg', field: '@value', id: '5' }, - ], - bucketAggs: [ + 100, + '1000' + ); + + const aggs = query.aggs['2'].aggs; + expect(aggs['1'].avg.field).toBe('@value'); + }); + + it('term agg and order by term', () => { + const target = { + metrics: [ + { type: 'count', id: '1' }, + { type: 'avg', field: '@value', id: '5' }, + ], + bucketAggs: [ + { + type: 'terms', + field: '@host', + settings: { size: 5, order: 'asc', orderBy: '_term' }, + id: '2', + }, + { type: 'date_histogram', field: '@timestamp', id: '3' }, + ], + }; + const query = builder.build(target, 100, '1000'); + const firstLevel = query.aggs['2']; + if (builder.esVersion >= 60) { + expect(firstLevel.terms.order._key).toBe('asc'); + } else { + expect(firstLevel.terms.order._term).toBe('asc'); + } + }); + + it('with term agg and order by metric agg', () => { + const query = builder.build( { - type: 'terms', - field: '@host', - settings: { size: 5, order: 'asc', orderBy: '_term' }, - id: '2', + metrics: [ + { type: 'count', id: '1' }, + { type: 'avg', field: '@value', id: '5' }, + ], + bucketAggs: [ + { + type: 'terms', + field: '@host', + settings: { size: 5, order: 'asc', orderBy: '5' }, + id: '2', + }, + { type: 'date_histogram', field: '@timestamp', id: '3' }, + ], }, - { type: 'date_histogram', field: '@timestamp', id: '3' }, - ], - }, - 100, - // @ts-ignore - 1000 - ); - - const firstLevel = query.aggs['2']; - expect(firstLevel.terms.order._key).toBe('asc'); - }); + 100, + '1000' + ); - it('with term agg and order by metric agg', () => { - const query = builder.build( - { - metrics: [ - { type: 'count', id: '1' }, - { type: 'avg', field: '@value', id: '5' }, - ], - bucketAggs: [ - { - type: 'terms', - field: '@host', - settings: { size: 5, order: 'asc', orderBy: '5' }, - id: '2', - }, - { type: 'date_histogram', field: '@timestamp', id: '3' }, - ], - }, - 100, - 1000 - ); - - const firstLevel = query.aggs['2']; - const secondLevel = firstLevel.aggs['3']; - - expect(firstLevel.aggs['5'].avg.field).toBe('@value'); - expect(secondLevel.aggs['5'].avg.field).toBe('@value'); - }); + const firstLevel = query.aggs['2']; + const secondLevel = firstLevel.aggs['3']; + + expect(firstLevel.aggs['5'].avg.field).toBe('@value'); + expect(secondLevel.aggs['5'].avg.field).toBe('@value'); + }); - it('with metric percentiles', () => { - const query = builder.build( - { - metrics: [ + it('with metric percentiles', () => { + const query = builder.build( { - id: '1', - type: 'percentiles', - field: '@load_time', - settings: { - percents: [1, 2, 3, 4], - }, + metrics: [ + { + id: '1', + type: 'percentiles', + field: '@load_time', + settings: { + percents: [1, 2, 3, 4], + }, + }, + ], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], }, - ], - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], - }, - 100, - 1000 - ); + 100, + '1000' + ); - const firstLevel = query.aggs['3']; + const firstLevel = query.aggs['3']; - expect(firstLevel.aggs['1'].percentiles.field).toBe('@load_time'); - expect(firstLevel.aggs['1'].percentiles.percents).toEqual([1, 2, 3, 4]); - }); + expect(firstLevel.aggs['1'].percentiles.field).toBe('@load_time'); + expect(firstLevel.aggs['1'].percentiles.percents).toEqual([1, 2, 3, 4]); + }); - it('with filters aggs', () => { - const query = builder.build({ - metrics: [{ type: 'count', id: '1' }], - timeField: '@timestamp', - bucketAggs: [ - { - id: '2', - type: 'filters', - settings: { - filters: [{ query: '@metric:cpu' }, { query: '@metric:logins.count' }], + it('with filters aggs', () => { + const query = builder.build({ + metrics: [{ type: 'count', id: '1' }], + timeField: '@timestamp', + bucketAggs: [ + { + id: '2', + type: 'filters', + settings: { + filters: [{ query: '@metric:cpu' }, { query: '@metric:logins.count' }], + }, + }, + { type: 'date_histogram', field: '@timestamp', id: '4' }, + ], + }); + + expect(query.aggs['2'].filters.filters['@metric:cpu'].query_string.query).toBe('@metric:cpu'); + expect(query.aggs['2'].filters.filters['@metric:logins.count'].query_string.query).toBe('@metric:logins.count'); + expect(query.aggs['2'].aggs['4'].date_histogram.field).toBe('@timestamp'); + }); + + it('should return correct query for raw_document metric', () => { + const target = { + metrics: [{ type: 'raw_document', id: '1', settings: {} }], + timeField: '@timestamp', + bucketAggs: [] as any[], + }; + + const query = builder.build(target); + expect(query).toMatchObject({ + size: 500, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + format: 'epoch_millis', + gte: '$timeFrom', + lte: '$timeTo', + }, + }, + }, + { + query_string: { + analyze_wildcard: true, + query: undefined, + }, + }, + ], + }, }, - }, - { type: 'date_histogram', field: '@timestamp', id: '4' }, - ], - }); - - expect(query.aggs['2'].filters.filters['@metric:cpu'].query_string.query).toBe('@metric:cpu'); - expect(query.aggs['2'].filters.filters['@metric:logins.count'].query_string.query).toBe('@metric:logins.count'); - expect(query.aggs['2'].aggs['4'].date_histogram.field).toBe('@timestamp'); - }); - - it('with filters aggs on es5.x', () => { - const builder5x = new ElasticQueryBuilder({ - timeField: '@timestamp', - esVersion: 5, - }); - const query = builder5x.build({ - metrics: [{ type: 'count', id: '1' }], - timeField: '@timestamp', - bucketAggs: [ - { - id: '2', - type: 'filters', - settings: { - filters: [{ query: '@metric:cpu' }, { query: '@metric:logins.count' }], + sort: { + '@timestamp': { + order: 'desc', + unmapped_type: 'boolean', + }, }, - }, - { type: 'date_histogram', field: '@timestamp', id: '4' }, - ], - }); - - expect(query.aggs['2'].filters.filters['@metric:cpu'].query_string.query).toBe('@metric:cpu'); - expect(query.aggs['2'].filters.filters['@metric:logins.count'].query_string.query).toBe('@metric:logins.count'); - expect(query.aggs['2'].aggs['4'].date_histogram.field).toBe('@timestamp'); - }); - - it('with raw_document metric', () => { - const query = builder.build({ - metrics: [{ type: 'raw_document', id: '1', settings: {} }], - timeField: '@timestamp', - bucketAggs: [], - }); - - expect(query.size).toBe(500); - }); - it('with raw_document metric size set', () => { - const query = builder.build({ - metrics: [{ type: 'raw_document', id: '1', settings: { size: 1337 } }], - timeField: '@timestamp', - bucketAggs: [], - }); - - expect(query.size).toBe(1337); - }); - - it('with moving average', () => { - const query = builder.build({ - metrics: [ - { - id: '3', - type: 'sum', - field: '@value', - }, - { - id: '2', - type: 'moving_avg', - field: '3', - pipelineAgg: '3', - }, - ], - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], - }); - - const firstLevel = query.aggs['3']; - - expect(firstLevel.aggs['2']).not.toBe(undefined); - expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined); - expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3'); - }); - - it('with moving average doc count', () => { - const query = builder.build({ - metrics: [ - { - id: '3', - type: 'count', - field: 'select field', - }, - { - id: '2', - type: 'moving_avg', - field: '3', - pipelineAgg: '3', - }, - ], - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }], - }); - - const firstLevel = query.aggs['4']; - - expect(firstLevel.aggs['2']).not.toBe(undefined); - expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined); - expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('_count'); - }); - - it('with broken moving average', () => { - const query = builder.build({ - metrics: [ - { - id: '3', - type: 'sum', - field: '@value', - }, - { - id: '2', - type: 'moving_avg', - pipelineAgg: '3', - }, - { - id: '4', - type: 'moving_avg', - pipelineAgg: 'Metric to apply moving average', - }, - ], - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], - }); - - const firstLevel = query.aggs['3']; - - expect(firstLevel.aggs['2']).not.toBe(undefined); - expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined); - expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3'); - expect(firstLevel.aggs['4']).toBe(undefined); - }); - - it('with derivative', () => { - const query = builder.build({ - metrics: [ - { - id: '3', - type: 'sum', - field: '@value', - }, - { - id: '2', - type: 'derivative', - pipelineAgg: '3', - }, - ], - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], - }); + script_fields: {}, + }); + }); + + it('should set query size from settings when raw_documents', () => { + const query = builder.build({ + metrics: [{ type: 'raw_document', id: '1', settings: { size: 1337 } }], + timeField: '@timestamp', + bucketAggs: [], + }); + + expect(query.size).toBe(1337); + }); + + it('with moving average', () => { + const query = builder.build({ + metrics: [ + { + id: '3', + type: 'sum', + field: '@value', + }, + { + id: '2', + type: 'moving_avg', + field: '3', + pipelineAgg: '3', + }, + ], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], + }); - const firstLevel = query.aggs['3']; + const firstLevel = query.aggs['3']; - expect(firstLevel.aggs['2']).not.toBe(undefined); - expect(firstLevel.aggs['2'].derivative).not.toBe(undefined); - expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('3'); - }); + expect(firstLevel.aggs['2']).not.toBe(undefined); + expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined); + expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3'); + }); - it('with derivative doc count', () => { - const query = builder.build({ - metrics: [ - { - id: '3', - type: 'count', - field: 'select field', - }, - { - id: '2', - type: 'derivative', - pipelineAgg: '3', - }, - ], - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }], - }); + it('with moving average doc count', () => { + const query = builder.build({ + metrics: [ + { + id: '3', + type: 'count', + field: 'select field', + }, + { + id: '2', + type: 'moving_avg', + field: '3', + pipelineAgg: '3', + }, + ], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }], + }); - const firstLevel = query.aggs['4']; + const firstLevel = query.aggs['4']; - expect(firstLevel.aggs['2']).not.toBe(undefined); - expect(firstLevel.aggs['2'].derivative).not.toBe(undefined); - expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('_count'); - }); + expect(firstLevel.aggs['2']).not.toBe(undefined); + expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined); + expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('_count'); + }); - it('with bucket_script', () => { - const query = builder.build({ - metrics: [ - { - id: '1', - type: 'sum', - field: '@value', - }, - { - id: '3', - type: 'max', - field: '@value', - }, - { - field: 'select field', - id: '4', - meta: {}, - pipelineVariables: [ + it('with broken moving average', () => { + const query = builder.build({ + metrics: [ { - name: 'var1', - pipelineAgg: '1', + id: '3', + type: 'sum', + field: '@value', }, { - name: 'var2', + id: '2', + type: 'moving_avg', pipelineAgg: '3', }, + { + id: '4', + type: 'moving_avg', + pipelineAgg: 'Metric to apply moving average', + }, ], - settings: { - script: 'params.var1 * params.var2', - }, - type: 'bucket_script', - }, - ], - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }], - }); + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], + }); - const firstLevel = query.aggs['2']; + const firstLevel = query.aggs['3']; - expect(firstLevel.aggs['4']).not.toBe(undefined); - expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined); - expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '1', var2: '3' }); - }); + expect(firstLevel.aggs['2']).not.toBe(undefined); + expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined); + expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3'); + expect(firstLevel.aggs['4']).toBe(undefined); + }); - it('with bucket_script doc count', () => { - const query = builder.build({ - metrics: [ - { - id: '3', - type: 'count', - field: 'select field', - }, - { - field: 'select field', - id: '4', - meta: {}, - pipelineVariables: [ + it('with derivative', () => { + const query = builder.build({ + metrics: [ { - name: 'var1', + id: '3', + type: 'sum', + field: '@value', + }, + { + id: '2', + type: 'derivative', pipelineAgg: '3', }, ], - settings: { - script: 'params.var1 * 1000', - }, - type: 'bucket_script', - }, - ], - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }], - }); + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], + }); - const firstLevel = query.aggs['2']; + const firstLevel = query.aggs['3']; - expect(firstLevel.aggs['4']).not.toBe(undefined); - expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined); - expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '_count' }); - }); - - it('with histogram', () => { - const query = builder.build({ - metrics: [{ id: '1', type: 'count' }], - bucketAggs: [ - { - type: 'histogram', - field: 'bytes', - id: '3', - settings: { interval: 10, min_doc_count: 2, missing: 5 }, - }, - ], - }); + expect(firstLevel.aggs['2']).not.toBe(undefined); + expect(firstLevel.aggs['2'].derivative).not.toBe(undefined); + expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('3'); + }); - const firstLevel = query.aggs['3']; - expect(firstLevel.histogram.field).toBe('bytes'); - expect(firstLevel.histogram.interval).toBe(10); - expect(firstLevel.histogram.min_doc_count).toBe(2); - expect(firstLevel.histogram.missing).toBe(5); - }); + it('with derivative doc count', () => { + const query = builder.build({ + metrics: [ + { + id: '3', + type: 'count', + field: 'select field', + }, + { + id: '2', + type: 'derivative', + pipelineAgg: '3', + }, + ], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }], + }); - it('with adhoc filters', () => { - const query = builder.build( - { - metrics: [{ type: 'Count', id: '0' }], - timeField: '@timestamp', - bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], - }, - [ - { key: 'key1', operator: '=', value: 'value1' }, - { key: 'key2', operator: '=', value: 'value2' }, - { key: 'key2', operator: '!=', value: 'value2' }, - { key: 'key3', operator: '<', value: 'value3' }, - { key: 'key4', operator: '>', value: 'value4' }, - { key: 'key5', operator: '=~', value: 'value5' }, - { key: 'key6', operator: '!~', value: 'value6' }, - ] - ); - - expect(query.query.bool.must[0].match_phrase['key1'].query).toBe('value1'); - expect(query.query.bool.must[1].match_phrase['key2'].query).toBe('value2'); - expect(query.query.bool.must_not[0].match_phrase['key2'].query).toBe('value2'); - expect(query.query.bool.filter[2].range['key3'].lt).toBe('value3'); - expect(query.query.bool.filter[3].range['key4'].gt).toBe('value4'); - expect(query.query.bool.filter[4].regexp['key5']).toBe('value5'); - expect(query.query.bool.filter[5].bool.must_not.regexp['key6']).toBe('value6'); - }); + const firstLevel = query.aggs['4']; - // terms query ES<6.0 - check ordering for _term and doc_type + expect(firstLevel.aggs['2']).not.toBe(undefined); + expect(firstLevel.aggs['2'].derivative).not.toBe(undefined); + expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('_count'); + }); - it('getTermsQuery(default case) es<6.0 should set asc sorting on _term', () => { - const query = builder.getTermsQuery({}); - expect(query.aggs['1'].terms.order._term).toBe('asc'); - expect(query.aggs['1'].terms.order._key).toBeUndefined(); - expect(query.aggs['1'].terms.order._count).toBeUndefined(); - }); + it('with bucket_script', () => { + const query = builder.build({ + metrics: [ + { + id: '1', + type: 'sum', + field: '@value', + }, + { + id: '3', + type: 'max', + field: '@value', + }, + { + field: 'select field', + id: '4', + meta: {}, + pipelineVariables: [ + { + name: 'var1', + pipelineAgg: '1', + }, + { + name: 'var2', + pipelineAgg: '3', + }, + ], + settings: { + script: 'params.var1 * params.var2', + }, + type: 'bucket_script', + }, + ], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }], + }); - it('getTermsQuery(order:desc) es<6.0 should set desc sorting on _term', () => { - const query = builder.getTermsQuery({ order: 'desc' }); - expect(query.aggs['1'].terms.order._term).toBe('desc'); - expect(query.aggs['1'].terms.order._key).toBeUndefined(); - expect(query.aggs['1'].terms.order._count).toBeUndefined(); - }); + const firstLevel = query.aggs['2']; - it('getTermsQuery(orderBy:doc_count) es<6.0 should set desc sorting on _count', () => { - const query = builder.getTermsQuery({ orderBy: 'doc_count' }); - expect(query.aggs['1'].terms.order._term).toBeUndefined(); - expect(query.aggs['1'].terms.order._key).toBeUndefined(); - expect(query.aggs['1'].terms.order._count).toBe('desc'); - }); + expect(firstLevel.aggs['4']).not.toBe(undefined); + expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined); + expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '1', var2: '3' }); + }); - it('getTermsQuery(orderBy:doc_count, order:asc) es<6.0 should set asc sorting on _count', () => { - const query = builder.getTermsQuery({ orderBy: 'doc_count', order: 'asc' }); - expect(query.aggs['1'].terms.order._term).toBeUndefined(); - expect(query.aggs['1'].terms.order._key).toBeUndefined(); - expect(query.aggs['1'].terms.order._count).toBe('asc'); - }); + it('with bucket_script doc count', () => { + const query = builder.build({ + metrics: [ + { + id: '3', + type: 'count', + field: 'select field', + }, + { + field: 'select field', + id: '4', + meta: {}, + pipelineVariables: [ + { + name: 'var1', + pipelineAgg: '3', + }, + ], + settings: { + script: 'params.var1 * 1000', + }, + type: 'bucket_script', + }, + ], + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }], + }); - // terms query ES>=6.0 - check ordering for _key and doc_type + const firstLevel = query.aggs['2']; - it('getTermsQuery(default case) es6.x should set asc sorting on _key', () => { - const builder6x = new ElasticQueryBuilder({ - timeField: '@timestamp', - esVersion: 60, - }); - const query = builder6x.getTermsQuery({}); - expect(query.aggs['1'].terms.order._term).toBeUndefined(); - expect(query.aggs['1'].terms.order._key).toBe('asc'); - expect(query.aggs['1'].terms.order._count).toBeUndefined(); - }); + expect(firstLevel.aggs['4']).not.toBe(undefined); + expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined); + expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '_count' }); + }); - it('getTermsQuery(order:desc) es6.x should set desc sorting on _key', () => { - const builder6x = new ElasticQueryBuilder({ - timeField: '@timestamp', - esVersion: 60, - }); - const query = builder6x.getTermsQuery({ order: 'desc' }); - expect(query.aggs['1'].terms.order._term).toBeUndefined(); - expect(query.aggs['1'].terms.order._key).toBe('desc'); - expect(query.aggs['1'].terms.order._count).toBeUndefined(); - }); + it('with histogram', () => { + const query = builder.build({ + metrics: [{ id: '1', type: 'count' }], + bucketAggs: [ + { + type: 'histogram', + field: 'bytes', + id: '3', + settings: { interval: 10, min_doc_count: 2, missing: 5 }, + }, + ], + }); - it('getTermsQuery(orderBy:doc_count) es6.x should set desc sorting on _count', () => { - const builder6x = new ElasticQueryBuilder({ - timeField: '@timestamp', - esVersion: 60, - }); - const query = builder6x.getTermsQuery({ orderBy: 'doc_count' }); - expect(query.aggs['1'].terms.order._term).toBeUndefined(); - expect(query.aggs['1'].terms.order._key).toBeUndefined(); - expect(query.aggs['1'].terms.order._count).toBe('desc'); - }); + const firstLevel = query.aggs['3']; + expect(firstLevel.histogram.field).toBe('bytes'); + expect(firstLevel.histogram.interval).toBe(10); + expect(firstLevel.histogram.min_doc_count).toBe(2); + expect(firstLevel.histogram.missing).toBe(5); + }); - it('getTermsQuery(orderBy:doc_count, order:asc) es6.x should set asc sorting on _count', () => { - const builder6x = new ElasticQueryBuilder({ - timeField: '@timestamp', - esVersion: 60, + it('with adhoc filters', () => { + const query = builder.build( + { + metrics: [{ type: 'Count', id: '0' }], + timeField: '@timestamp', + bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }], + }, + [ + { key: 'key1', operator: '=', value: 'value1' }, + { key: 'key2', operator: '=', value: 'value2' }, + { key: 'key2', operator: '!=', value: 'value2' }, + { key: 'key3', operator: '<', value: 'value3' }, + { key: 'key4', operator: '>', value: 'value4' }, + { key: 'key5', operator: '=~', value: 'value5' }, + { key: 'key6', operator: '!~', value: 'value6' }, + ] + ); + + expect(query.query.bool.must[0].match_phrase['key1'].query).toBe('value1'); + expect(query.query.bool.must[1].match_phrase['key2'].query).toBe('value2'); + expect(query.query.bool.must_not[0].match_phrase['key2'].query).toBe('value2'); + expect(query.query.bool.filter[2].range['key3'].lt).toBe('value3'); + expect(query.query.bool.filter[3].range['key4'].gt).toBe('value4'); + expect(query.query.bool.filter[4].regexp['key5']).toBe('value5'); + expect(query.query.bool.filter[5].bool.must_not.regexp['key6']).toBe('value6'); + }); + + describe('getTermsQuery', () => { + function testGetTermsQuery(queryDef: any) { + const query = builder.getTermsQuery(queryDef); + return query.aggs['1'].terms.order; + } + + function checkSort(order: any, expected: string) { + if (builder.esVersion < 60) { + expect(order._term).toBe(expected); + expect(order._key).toBeUndefined(); + } else { + expect(order._term).toBeUndefined(); + expect(order._key).toBe(expected); + } + } + + it('should set correct default sorting', () => { + const order = testGetTermsQuery({}); + checkSort(order, 'asc'); + expect(order._count).toBeUndefined(); + }); + + it('should set correct explicit sorting', () => { + const order = testGetTermsQuery({ order: 'desc' }); + console.log({ order }); + checkSort(order, 'desc'); + expect(order._count).toBeUndefined(); + }); + + it('getTermsQuery(orderBy:doc_count) should set desc sorting on _count', () => { + const query = builder.getTermsQuery({ orderBy: 'doc_count' }); + expect(query.aggs['1'].terms.order._term).toBeUndefined(); + expect(query.aggs['1'].terms.order._key).toBeUndefined(); + expect(query.aggs['1'].terms.order._count).toBe('desc'); + }); + + it('getTermsQuery(orderBy:doc_count, order:asc) should set asc sorting on _count', () => { + const query = builder.getTermsQuery({ orderBy: 'doc_count', order: 'asc' }); + expect(query.aggs['1'].terms.order._term).toBeUndefined(); + expect(query.aggs['1'].terms.order._key).toBeUndefined(); + expect(query.aggs['1'].terms.order._count).toBe('asc'); + }); + }); + + it('getTermsQuery should request documents and date histogram', () => { + const query = builder.getLogsQuery({}, ''); + console.log({ query }); + expect(query).toHaveProperty('query.bool.filter'); + expect(query.aggs['2']).toHaveProperty('date_histogram'); + }); }); - const query = builder6x.getTermsQuery({ orderBy: 'doc_count', order: 'asc' }); - expect(query.aggs['1'].terms.order._term).toBeUndefined(); - expect(query.aggs['1'].terms.order._key).toBeUndefined(); - expect(query.aggs['1'].terms.order._count).toBe('asc'); - }); - - // Logs query - - it('getTermsQuery should request documents and date histogram', () => { - const query = builder.getLogsQuery({}); - expect(query).toHaveProperty('query.bool.filter'); - expect(query.aggs['2']).toHaveProperty('date_histogram'); }); });