The open and composable observability and data visualization platform. Visualize metrics, logs, and traces from multiple sources like Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 
grafana/public/app/plugins/datasource/elasticsearch/QueryBuilder.test.ts

924 lines
27 KiB

import { ElasticQueryBuilder } from './QueryBuilder';
import { ElasticsearchQuery, TermsQuery } from './types';
describe('ElasticQueryBuilder', () => {
const builder = new ElasticQueryBuilder({ timeField: '@timestamp' }); // es2
it('should return query with defaults', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '0' }],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }],
});
expect(query.query.bool.filter[0].range['@timestamp'].gte).toBe('$timeFrom');
expect(query.aggs['1'].date_histogram.extended_bounds.min).toBe('$timeFrom');
});
it('should clean settings from null values', () => {
const query = builder.build({
refId: 'A',
// The following `missing: null as any` is because previous versions of the DS where
// storing null in the query model when inputting an empty string,
// which were then removed in the query builder.
// The new version doesn't store empty strings at all. This tests ensures backward compatibility.
metrics: [{ type: 'avg', id: '0', settings: { missing: null as any, script: '1' } }],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }],
});
expect(query.aggs['1'].aggs['0'].avg.missing).not.toBeDefined();
expect(query.aggs['1'].aggs['0'].avg.script).toBeDefined();
});
it('with multiple bucket aggs', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{ type: 'terms', field: '@host', id: '2' },
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
});
expect(query.aggs['2'].terms.field).toBe('@host');
expect(query.aggs['2'].aggs['3'].date_histogram.field).toBe('@timestamp');
});
it('with select field', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'avg', field: '@value', id: '1' }],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
});
const aggs = query.aggs['2'].aggs;
expect(aggs['1'].avg.field).toBe('@value');
});
it('term agg and order by term', () => {
const target: ElasticsearchQuery = {
refId: 'A',
metrics: [
{ type: 'count', id: '1' },
{ type: 'avg', field: '@value', id: '5' },
],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: '5', order: 'asc', orderBy: '_term' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
};
const query = builder.build(target);
const firstLevel = query.aggs['2'];
expect(firstLevel.terms.order._key).toBe('asc');
});
it('with term agg and order by metric agg', () => {
const query = builder.build({
refId: 'A',
metrics: [
{ type: 'count', id: '1' },
{ type: 'avg', field: '@value', id: '5' },
],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: '5', order: 'asc', orderBy: '5' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
});
const firstLevel = query.aggs['2'];
const secondLevel = firstLevel.aggs['3'];
expect(firstLevel.aggs['5'].avg.field).toBe('@value');
expect(secondLevel.aggs['5'].avg.field).toBe('@value');
});
it('with term agg and order by count agg', () => {
const query = builder.build({
refId: 'A',
metrics: [
{ type: 'count', id: '1' },
{ type: 'avg', field: '@value', id: '5' },
],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: '5', order: 'asc', orderBy: '1' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
});
expect(query.aggs['2'].terms.order._count).toEqual('asc');
expect(query.aggs['2'].aggs).not.toHaveProperty('1');
});
it('with term agg and order by extended_stats agg', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'extended_stats', id: '1', field: '@value', meta: { std_deviation: true } }],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: '5', order: 'asc', orderBy: '1[std_deviation]' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
});
const firstLevel = query.aggs['2'];
const secondLevel = firstLevel.aggs['3'];
expect(firstLevel.aggs['1'].extended_stats.field).toBe('@value');
expect(secondLevel.aggs['1'].extended_stats.field).toBe('@value');
});
it('with term agg and order by percentiles agg', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'percentiles', id: '1', field: '@value', settings: { percents: ['95', '99'] } }],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: '5', order: 'asc', orderBy: '1[95.0]' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
});
const firstLevel = query.aggs['2'];
const secondLevel = firstLevel.aggs['3'];
expect(firstLevel.aggs['1'].percentiles.field).toBe('@value');
expect(secondLevel.aggs['1'].percentiles.field).toBe('@value');
});
it('with term agg and valid min_doc_count', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { min_doc_count: '1' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
});
const firstLevel = query.aggs['2'];
expect(firstLevel.terms.min_doc_count).toBe(1);
});
it('with term agg and variable as min_doc_count', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { min_doc_count: '$min_doc_count' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
});
const firstLevel = query.aggs['2'];
expect(firstLevel.terms.min_doc_count).toBe('$min_doc_count');
});
it('with metric percentiles', () => {
const percents = ['1', '2', '3', '4'];
const field = '@load_time';
const query = builder.build({
refId: 'A',
metrics: [
{
id: '1',
type: 'percentiles',
field,
settings: {
percents,
},
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['1'].percentiles.field).toBe(field);
expect(firstLevel.aggs['1'].percentiles.percents).toEqual(percents);
});
it('with filters aggs', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
id: '2',
type: 'filters',
settings: {
filters: [
{ query: '@metric:cpu', label: '' },
{ query: '@metric:logins.count', label: '' },
],
},
},
{ type: 'date_histogram', field: '@timestamp', id: '4' },
],
});
expect(query.aggs['2'].filters.filters['@metric:cpu'].query_string.query).toBe('@metric:cpu');
expect(query.aggs['2'].filters.filters['@metric:logins.count'].query_string.query).toBe('@metric:logins.count');
expect(query.aggs['2'].aggs['4'].date_histogram.field).toBe('@timestamp');
});
it('should return correct query for raw_document metric', () => {
const target: ElasticsearchQuery = {
refId: 'A',
metrics: [{ type: 'raw_document', id: '1', settings: {} }],
timeField: '@timestamp',
bucketAggs: [],
};
const query = builder.build(target);
expect(query).toMatchObject({
size: 500,
query: {
bool: {
filter: [
{
range: {
'@timestamp': {
format: 'epoch_millis',
gte: '$timeFrom',
lte: '$timeTo',
},
},
},
],
},
},
sort: [{ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } }, { _doc: { order: 'desc' } }],
script_fields: {},
});
});
it('should set query size from settings when raw_documents', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'raw_document', id: '1', settings: { size: '1337' } }],
timeField: '@timestamp',
bucketAggs: [],
});
expect(query.size).toBe(1337);
});
it('with moving average', () => {
const query = builder.build({
refId: 'A',
metrics: [
{
id: '3',
type: 'sum',
field: '@value',
},
{
id: '2',
type: 'moving_avg',
field: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3');
});
it('with moving average doc count', () => {
const query = builder.build({
refId: 'A',
metrics: [
{
id: '3',
type: 'count',
},
{
id: '2',
type: 'moving_avg',
field: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }],
});
const firstLevel = query.aggs['4'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('_count');
});
it('with broken moving average', () => {
const query = builder.build({
refId: 'A',
metrics: [
{
id: '3',
type: 'sum',
field: '@value',
},
{
id: '2',
type: 'moving_avg',
field: '3',
},
{
id: '4',
type: 'moving_avg',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3');
expect(firstLevel.aggs['4']).toBe(undefined);
});
it('with top_metrics', () => {
const query = builder.build({
refId: 'A',
metrics: [
{
id: '2',
type: 'top_metrics',
settings: {
order: 'desc',
orderBy: '@timestamp',
metrics: ['@value'],
},
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].top_metrics).not.toBe(undefined);
expect(firstLevel.aggs['2'].top_metrics.metrics).not.toBe(undefined);
expect(firstLevel.aggs['2'].top_metrics.size).not.toBe(undefined);
expect(firstLevel.aggs['2'].top_metrics.sort).not.toBe(undefined);
expect(firstLevel.aggs['2'].top_metrics.metrics.length).toBe(1);
expect(firstLevel.aggs['2'].top_metrics.metrics).toEqual([{ field: '@value' }]);
expect(firstLevel.aggs['2'].top_metrics.sort).toEqual([{ '@timestamp': 'desc' }]);
expect(firstLevel.aggs['2'].top_metrics.size).toBe(1);
});
it('with derivative', () => {
const query = builder.build({
refId: 'A',
metrics: [
{
id: '3',
type: 'sum',
field: '@value',
},
{
id: '2',
type: 'derivative',
field: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('3');
});
it('with derivative doc count', () => {
const query = builder.build({
refId: 'A',
metrics: [
{
id: '3',
type: 'count',
},
{
id: '2',
type: 'derivative',
field: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }],
});
const firstLevel = query.aggs['4'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('_count');
});
it('with serial_diff', () => {
const query = builder.build({
refId: 'A',
metrics: [
{
id: '3',
type: 'max',
field: '@value',
},
{
id: '2',
type: 'serial_diff',
field: '3',
settings: {
lag: '5',
},
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].serial_diff).not.toBe(undefined);
expect(firstLevel.aggs['2'].serial_diff.buckets_path).toBe('3');
expect(firstLevel.aggs['2'].serial_diff.lag).toBe(5);
});
it('with bucket_script', () => {
const query = builder.build({
refId: 'A',
metrics: [
{
id: '1',
type: 'sum',
field: '@value',
},
{
id: '3',
type: 'max',
field: '@value',
},
{
id: '4',
pipelineVariables: [
{
name: 'var1',
pipelineAgg: '1',
},
{
name: 'var2',
pipelineAgg: '3',
},
],
settings: {
script: 'params.var1 * params.var2',
},
type: 'bucket_script',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
});
const firstLevel = query.aggs['2'];
expect(firstLevel.aggs['4']).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '1', var2: '3' });
});
it('with bucket_script doc count', () => {
const query = builder.build({
refId: 'A',
metrics: [
{
id: '3',
type: 'count',
},
{
id: '4',
pipelineVariables: [
{
name: 'var1',
pipelineAgg: '3',
},
],
settings: {
script: 'params.var1 * 1000',
},
type: 'bucket_script',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
});
const firstLevel = query.aggs['2'];
expect(firstLevel.aggs['4']).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '_count' });
});
it('with histogram', () => {
const query = builder.build({
refId: 'A',
metrics: [{ id: '1', type: 'count' }],
bucketAggs: [
{
type: 'histogram',
field: 'bytes',
id: '3',
settings: {
interval: '10',
min_doc_count: '2',
},
},
],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.histogram.field).toBe('bytes');
expect(firstLevel.histogram.interval).toBe('10');
expect(firstLevel.histogram.min_doc_count).toBe('2');
});
it('with nested', () => {
const query = builder.build({
refId: 'A',
metrics: [{ id: '1', type: 'count' }],
bucketAggs: [
{
type: 'nested',
field: 'nested_field',
id: '3',
},
],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.nested.path).toBe('nested_field');
});
describe('getTermsQuery', () => {
function testGetTermsQuery(queryDef: TermsQuery) {
const query = builder.getTermsQuery(queryDef);
return query.aggs['1'].terms.order;
}
function checkSort(
order: {
[key: string]: string;
},
expected: string
) {
expect(order._term).toBeUndefined();
expect(order._key).toBe(expected);
}
it('should set correct default sorting', () => {
const order = testGetTermsQuery({});
checkSort(order, 'asc');
expect(order._count).toBeUndefined();
});
it('should set correct explicit sorting', () => {
const order = testGetTermsQuery({ order: 'desc' });
checkSort(order, 'desc');
expect(order._count).toBeUndefined();
});
it('getTermsQuery(orderBy:doc_count) should set desc sorting on _count', () => {
const query = builder.getTermsQuery({ orderBy: 'doc_count' });
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBe('desc');
});
it('getTermsQuery(orderBy:doc_count, order:asc) should set asc sorting on _count', () => {
const query = builder.getTermsQuery({ orderBy: 'doc_count', order: 'asc' });
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBe('asc');
});
describe('lucene query', () => {
it('should add query_string filter when query is not empty', () => {
const luceneQuery = 'foo';
const query = builder.getTermsQuery({ orderBy: 'doc_count', order: 'asc', query: luceneQuery });
expect(query.query.bool.filter).toContainEqual({
query_string: { analyze_wildcard: true, query: luceneQuery },
});
});
it('should not add query_string filter when query is empty', () => {
const query = builder.getTermsQuery({ orderBy: 'doc_count', order: 'asc' });
expect(
query.query.bool.filter.find((filter: object) => Object.keys(filter).includes('query_string'))
).toBeFalsy();
});
});
});
describe('lucene query', () => {
it('should add query_string filter when query is not empty', () => {
const luceneQuery = 'foo';
const query = builder.build({ refId: 'A', query: luceneQuery });
expect(query.query.bool.filter).toContainEqual({
query_string: { analyze_wildcard: true, query: luceneQuery },
});
});
it('should not add query_string filter when query is empty', () => {
const query = builder.build({ refId: 'A' });
expect(
query.query.bool.filter.find((filter: object) => Object.keys(filter).includes('query_string'))
).toBeFalsy();
});
});
describe('getLogsQuery', () => {
it('should return query with defaults', () => {
const query = builder.getLogsQuery({ refId: 'A' }, 500);
expect(query.size).toEqual(500);
const expectedQuery = {
bool: {
filter: [{ range: { '@timestamp': { gte: '$timeFrom', lte: '$timeTo', format: 'epoch_millis' } } }],
},
};
expect(query.query).toEqual(expectedQuery);
expect(query.sort).toEqual([
{ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } },
{ _doc: { order: 'desc' } },
]);
const expectedAggs = {
// FIXME: It's pretty weak to include this '1' in the test as it's not part of what we are testing here and
// might change as a cause of unrelated changes
1: {
aggs: {},
date_histogram: {
extended_bounds: { max: '$timeTo', min: '$timeFrom' },
field: '@timestamp',
format: 'epoch_millis',
fixed_interval: '${__interval_ms}ms',
min_doc_count: 0,
},
},
};
expect(query.aggs).toMatchObject(expectedAggs);
});
describe('lucene query', () => {
it('should add query_string filter when query is not empty', () => {
const luceneQuery = 'foo';
const query = builder.getLogsQuery({ refId: 'A', query: luceneQuery }, 500);
expect(query.query.bool.filter).toContainEqual({
query_string: { analyze_wildcard: true, query: luceneQuery },
});
});
it('should not add query_string filter when query is empty', () => {
const query = builder.getLogsQuery({ refId: 'A' }, 500);
expect(
query.query.bool.filter.find((filter: object) => Object.keys(filter).includes('query_string'))
).toBeFalsy();
});
});
});
describe('Value casting for settings', () => {
it('correctly casts values in moving_avg ', () => {
const query = builder.build({
refId: 'A',
metrics: [
{ type: 'avg', id: '2' },
{
type: 'moving_avg',
id: '3',
field: '2',
settings: {
window: '5',
model: 'holt_winters',
predict: '10',
settings: {
alpha: '1',
beta: '2',
gamma: '3',
period: '4',
},
},
},
],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }],
});
const movingAvg = query.aggs['1'].aggs['3'].moving_avg;
expect(movingAvg.window).toBe(5);
expect(movingAvg.predict).toBe(10);
expect(movingAvg.settings.alpha).toBe(1);
expect(movingAvg.settings.beta).toBe(2);
expect(movingAvg.settings.gamma).toBe(3);
expect(movingAvg.settings.period).toBe(4);
});
it('correctly casts values in serial_diff ', () => {
const query = builder.build({
refId: 'A',
metrics: [
{ type: 'avg', id: '2' },
{
type: 'serial_diff',
id: '3',
field: '2',
settings: {
lag: '1',
},
},
],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }],
});
const serialDiff = query.aggs['1'].aggs['3'].serial_diff;
expect(serialDiff.lag).toBe(1);
});
describe('date_histogram', () => {
it('should not include time_zone if not present in the query model', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2', settings: { min_doc_count: '1' } }],
});
expect(query.aggs['2'].date_histogram.time_zone).not.toBeDefined();
});
it('should not include time_zone if "utc" in the query model', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{ type: 'date_histogram', field: '@timestamp', id: '2', settings: { min_doc_count: '1', timeZone: 'utc' } },
],
});
expect(query.aggs['2'].date_histogram.time_zone).not.toBeDefined();
});
it('should include time_zone if not "utc" in the query model', () => {
const expectedTimezone = 'America/Los_angeles';
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
type: 'date_histogram',
field: '@timestamp',
id: '2',
settings: { min_doc_count: '1', timeZone: expectedTimezone },
},
],
});
expect(query.aggs['2'].date_histogram.time_zone).toBe(expectedTimezone);
});
describe('field property', () => {
it('should use timeField from datasource when not specified', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
type: 'date_histogram',
id: '2',
settings: { min_doc_count: '1' },
},
],
});
expect(query.aggs['2'].date_histogram.field).toBe('@timestamp');
});
it('should use field from bucket agg when specified', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
type: 'date_histogram',
id: '2',
field: '@time',
settings: { min_doc_count: '1' },
},
],
});
expect(query.aggs['2'].date_histogram.field).toBe('@time');
});
it('should use fixed_interval', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
type: 'date_histogram',
id: '2',
field: '@time',
settings: { min_doc_count: '1', interval: '1d' },
},
],
});
expect(query.aggs['2'].date_histogram.interval).toBeUndefined();
expect(query.aggs['2'].date_histogram.fixed_interval).toBe('1d');
});
it('should use calendar_interval', () => {
const query = builder.build({
refId: 'A',
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
type: 'date_histogram',
id: '2',
field: '@time',
settings: { min_doc_count: '1', interval: '1w' },
},
],
});
expect(query.aggs['2'].date_histogram.interval).toBeUndefined();
expect(query.aggs['2'].date_histogram.calendar_interval).toBe('1w');
});
});
});
});
});