apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Elasticsearch: Remove timeSrv dependency (#29770)

Browse Source 
* Elasticsearch: use time range from queries or default one

* Fix tests

* Fix language provider tests

* Elasticsearch: remove timeSrv dependency for logs context (#29841)

* Elasticsearch: remove timeSrv dependency for logs context

* Add tests & slighlty improve getIndexList

Co-authored-by: Elfo404 <gio.ricci@grafana.com>
pull/29898/head
Alexander Zobnin 5 years ago committed by GitHub
parent 1ee9745794
commit ae575158ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 113 additions and 75 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx
  2. 37
      public/app/plugins/datasource/elasticsearch/datasource.test.ts
  3. 46
      public/app/plugins/datasource/elasticsearch/datasource.ts
  4. 28
      public/app/plugins/datasource/elasticsearch/index_pattern.ts
  5. 15
      public/app/plugins/datasource/elasticsearch/language_provider.test.ts
  6. 52
      public/app/plugins/datasource/elasticsearch/specs/index_pattern.test.ts
  7. 4
      public/app/plugins/datasource/elasticsearch/types.ts

6
public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx
Unescape View File

@ -1,7 +1,7 @@
import React from 'react';
import { EventsWithValidation, regexValidation, LegacyForms } from '@grafana/ui';
const { Select, Input, FormField } = LegacyForms;
import { ElasticsearchOptions } from '../types';
import { ElasticsearchOptions, Interval } from '../types';
import { DataSourceSettings, SelectableValue } from '@grafana/data';
const indexPatternTypes = [
@ -170,7 +170,9 @@ const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['va
});
};
const intervalHandler = (value: Props['value'], onChange: Props['onChange']) => (option: SelectableValue<string>) => {
const intervalHandler = (value: Props['value'], onChange: Props['onChange']) => (
option: SelectableValue<Interval | 'none'>
) => {
const { database } = value;
// If option value is undefined it will send its label instead so we have to convert made up value to undefined here.
const newInterval = option.value === 'none' ? undefined : option.value;

37
public/app/plugins/datasource/elasticsearch/datasource.test.ts
Unescape View File

@ -15,7 +15,6 @@ import {
import _ from 'lodash';
import { ElasticDatasource, enhanceDataFrame } from './datasource';
import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { ElasticsearchOptions, ElasticsearchQuery } from './types';
import { Filters } from './components/QueryEditor/BucketAggregationsEditor/aggregations';
@ -61,8 +60,6 @@ describe('ElasticDatasource', function(this: any) {
getAdhocFilters: jest.fn(() => []),
};
const timeSrv: any = createTimeSrv('now-1h');
interface TestContext {
ds: ElasticDatasource;
}
@ -88,15 +85,8 @@ describe('ElasticDatasource', function(this: any) {
}
function createDatasource(instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>) {
createDatasourceWithTime(instanceSettings, timeSrv as TimeSrv);
}
function createDatasourceWithTime(
instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
timeSrv: TimeSrv
) {
instanceSettings.jsonData = instanceSettings.jsonData || ({} as ElasticsearchOptions);
ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv, timeSrv);
ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv);
}
describe('When testing datasource with index pattern', () => {
@ -506,14 +496,11 @@ describe('ElasticDatasource', function(this: any) {
};
beforeEach(() => {
createDatasourceWithTime(
{
url: ELASTICSEARCH_MOCK_URL,
database: '[asd-]YYYY.MM.DD',
jsonData: { interval: 'Daily', esVersion: 50 } as ElasticsearchOptions,
} as DataSourceInstanceSettings<ElasticsearchOptions>,
twoWeekTimeSrv
);
createDatasource({
url: ELASTICSEARCH_MOCK_URL,
database: '[asd-]YYYY.MM.DD',
jsonData: { interval: 'Daily', esVersion: 50 } as ElasticsearchOptions,
} as DataSourceInstanceSettings<ElasticsearchOptions>);
});
it('should return fields of the newest available index', async () => {
@ -534,7 +521,8 @@ describe('ElasticDatasource', function(this: any) {
return Promise.reject({ status: 404 });
});
const fieldObjects = await ctx.ds.getFields();
const range = twoWeekTimeSrv.timeRange();
const fieldObjects = await ctx.ds.getFields(undefined, range);
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp', 'beat.hostname']);
@ -545,6 +533,7 @@ describe('ElasticDatasource', function(this: any) {
.subtract(2, 'day')
.format('YYYY.MM.DD');
const range = twoWeekTimeSrv.timeRange();
datasourceRequestMock.mockImplementation(options => {
if (options.url === `${ELASTICSEARCH_MOCK_URL}/asd-${twoDaysBefore}/_mapping`) {
return Promise.resolve(basicResponse);
@ -554,7 +543,7 @@ describe('ElasticDatasource', function(this: any) {
expect.assertions(2);
try {
await ctx.ds.getFields();
await ctx.ds.getFields(undefined, range);
} catch (e) {
expect(e).toStrictEqual({ status: 500 });
expect(datasourceRequestMock).toBeCalledTimes(1);
@ -562,13 +551,14 @@ describe('ElasticDatasource', function(this: any) {
});
it('should not retry more than 7 indices', async () => {
const range = twoWeekTimeSrv.timeRange();
datasourceRequestMock.mockImplementation(() => {
return Promise.reject({ status: 404 });
});
expect.assertions(2);
try {
await ctx.ds.getFields();
await ctx.ds.getFields(undefined, range);
} catch (e) {
expect(e).toStrictEqual({ status: 404 });
expect(datasourceRequestMock).toBeCalledTimes(7);
@ -840,8 +830,7 @@ describe('ElasticDatasource', function(this: any) {
timeField: '@time',
},
} as DataSourceInstanceSettings<ElasticsearchOptions>,
templateSrv as TemplateSrv,
timeSrv as TimeSrv
templateSrv as TemplateSrv
);
(dataSource as any).post = jest.fn(() => Promise.resolve({ responses: [] }));
dataSource.query(createElasticQuery());

46
public/app/plugins/datasource/elasticsearch/datasource.ts
Unescape View File

@ -12,6 +12,10 @@ import {
LogRowModel,
Field,
MetricFindValue,
TimeRange,
DefaultTimeRange,
DateTime,
dateTime,
} from '@grafana/data';
import LanguageProvider from './language_provider';
import { ElasticResponse } from './elastic_response';
@ -21,7 +25,6 @@ import { toUtc } from '@grafana/data';
import { defaultBucketAgg, hasMetricOfType } from './query_def';
import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery } from './types';
import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider';
import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils';
@ -65,8 +68,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
constructor(
instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
private readonly templateSrv: TemplateSrv = getTemplateSrv(),
private readonly timeSrv: TimeSrv = getTimeSrv()
private readonly templateSrv: TemplateSrv = getTemplateSrv()
) {
super(instanceSettings);
this.basicAuth = instanceSettings.basicAuth;
@ -140,9 +142,8 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
*
* @param url the url to query the index on, for example `/_mapping`.
*/
private get(url: string) {
const range = this.timeSrv.timeRange();
const indexList = this.indexPattern.getIndexList(range.from.valueOf(), range.to.valueOf());
private get(url: string, range = DefaultTimeRange) {
const indexList = this.indexPattern.getIndexList(range.from, range.to);
if (_.isArray(indexList) && indexList.length) {
return this.requestAllIndices(indexList, url).then((results: any) => {
results.data.$$config = results.config;
@ -370,7 +371,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
);
}
getQueryHeader(searchType: any, timeFrom: any, timeTo: any) {
getQueryHeader(searchType: any, timeFrom?: DateTime, timeTo?: DateTime): string {
const queryHeader: any = {
search_type: searchType,
ignore_unavailable: true,
@ -447,9 +448,13 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
getLogRowContext = async (row: LogRowModel, options?: RowContextOptions): Promise<{ data: DataFrame[] }> => {
const sortField = row.dataFrame.fields.find(f => f.name === 'sort');
const searchAfter = sortField?.values.get(row.rowIndex) || [row.timeEpochMs];
const range = this.timeSrv.timeRange();
const direction = options?.direction === 'FORWARD' ? 'asc' : 'desc';
const header = this.getQueryHeader('query_then_fetch', range.from, range.to);
const sort = options?.direction === 'FORWARD' ? 'asc' : 'desc';
const header =
options?.direction === 'FORWARD'
? this.getQueryHeader('query_then_fetch', dateTime(row.timeEpochMs))
: this.getQueryHeader('query_then_fetch', undefined, dateTime(row.timeEpochMs));
const limit = options?.limit ?? 10;
const esQuery = JSON.stringify({
size: limit,
@ -459,8 +464,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
{
range: {
[this.timeField]: {
gte: range.from.valueOf(),
lte: range.to.valueOf(),
[options?.direction === 'FORWARD' ? 'gte' : 'lte']: row.timeEpochMs,
format: 'epoch_millis',
},
},
@ -468,14 +472,14 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
],
},
},
sort: [{ [this.timeField]: direction }, { _doc: direction }],
sort: [{ [this.timeField]: sort }, { _doc: sort }],
search_after: searchAfter,
});
const payload = [header, esQuery].join('\n') + '\n';
const url = this.getMultiSearchUrl();
const response = await this.post(url, payload);
const targets: ElasticsearchQuery[] = [{ refId: `${row.dataFrame.refId}`, metrics: [], isLogsQuery: true }];
const elasticResponse = new ElasticResponse(targets, transformHitsBasedOnDirection(response, direction));
const elasticResponse = new ElasticResponse(targets, transformHitsBasedOnDirection(response, sort));
const logResponse = elasticResponse.getLogs(this.logMessageField, this.logLevelField);
const dataFrame = _.first(logResponse.data);
if (!dataFrame) {
@ -576,9 +580,9 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
}
// TODO: instead of being a string, this could be a custom type representing all the elastic types
async getFields(type?: string): Promise<MetricFindValue[]> {
async getFields(type?: string, range?: TimeRange): Promise<MetricFindValue[]> {
const configuredEsVersion = this.esVersion;
return this.get('/_mapping').then((result: any) => {
return this.get('/_mapping', range).then((result: any) => {
const typeMap: any = {
float: 'number',
double: 'number',
@ -663,8 +667,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
});
}
getTerms(queryDef: any) {
const range = this.timeSrv.timeRange();
getTerms(queryDef: any, range = DefaultTimeRange) {
const searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count';
const header = this.getQueryHeader(searchType, range.from, range.to);
let esQuery = JSON.stringify(this.queryBuilder.getTermsQuery(queryDef));
@ -698,18 +701,19 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
return '_msearch';
}
metricFindQuery(query: string): Promise<MetricFindValue[]> {
metricFindQuery(query: string, options?: any): Promise<MetricFindValue[]> {
const range = options?.range;
const parsedQuery = JSON.parse(query);
if (query) {
if (parsedQuery.find === 'fields') {
parsedQuery.field = this.templateSrv.replace(parsedQuery.field, {}, 'lucene');
return this.getFields(query);
return this.getFields(query, range);
}
if (parsedQuery.find === 'terms') {
parsedQuery.field = this.templateSrv.replace(parsedQuery.field, {}, 'lucene');
parsedQuery.query = this.templateSrv.replace(parsedQuery.query || '*', {}, 'lucene');
return this.getTerms(query);
return this.getTerms(query, range);
}
}

28
public/app/plugins/datasource/elasticsearch/index_pattern.ts
Unescape View File

@ -1,9 +1,18 @@
import { toUtc, dateTime } from '@grafana/data';
import { toUtc, dateTime, DateTime, DurationUnit } from '@grafana/data';
import { Interval } from './types';
type IntervalMap = Record<
Interval,
{
startOf: DurationUnit;
amount: DurationUnit;
}
>;
const intervalMap: any = {
const intervalMap: IntervalMap = {
Hourly: { startOf: 'hour', amount: 'hours' },
Daily: { startOf: 'day', amount: 'days' },
Weekly: { startOf: 'isoWeek', amount: 'weeks' },
Weekly: { startOf: 'week', amount: 'weeks' },
Monthly: { startOf: 'month', amount: 'months' },
Yearly: { startOf: 'year', amount: 'years' },
};
@ -11,7 +20,7 @@ const intervalMap: any = {
export class IndexPattern {
private dateLocale = 'en';
constructor(private pattern: any, private interval?: string) {}
constructor(private pattern: string, private interval?: keyof typeof intervalMap) {}
getIndexForToday() {
if (this.interval) {
@ -23,16 +32,21 @@ export class IndexPattern {
}
}
getIndexList(from: any, to: any) {
getIndexList(from?: DateTime, to?: DateTime) {
// When no `from` or `to` is provided, we request data from 7 subsequent/previous indices
// for the provided index pattern.
// This is useful when requesting log context where the only time data we have is the log
// timestamp.
const indexOffset = 7;
if (!this.interval) {
return this.pattern;
}
const intervalInfo = intervalMap[this.interval];
const start = dateTime(from)
const start = dateTime(from || dateTime(to).add(-indexOffset, intervalInfo.amount))
.utc()
.startOf(intervalInfo.startOf);
const endEpoch = dateTime(to)
const endEpoch = dateTime(to || dateTime(from).add(indexOffset, intervalInfo.amount))
.utc()
.startOf(intervalInfo.startOf)
.valueOf();

15
public/app/plugins/datasource/elasticsearch/language_provider.test.ts
Unescape View File

@ -1,25 +1,15 @@
import LanguageProvider from './language_provider';
import { PromQuery } from '../prometheus/types';
import { ElasticDatasource } from './datasource';
import { DataSourceInstanceSettings, dateTime } from '@grafana/data';
import { DataSourceInstanceSettings } from '@grafana/data';
import { ElasticsearchOptions } from './types';
import { TemplateSrv } from '../../../features/templating/template_srv';
import { TimeSrv } from '../../../features/dashboard/services/TimeSrv';
const templateSrvStub = {
getAdhocFilters: jest.fn(() => [] as any[]),
replace: jest.fn((a: string) => a),
} as any;
const timeSrvStub = {
timeRange(): any {
return {
from: dateTime(1531468681),
to: dateTime(1531489712),
};
},
} as any;
const dataSource = new ElasticDatasource(
{
url: 'http://es.com',
@ -30,8 +20,7 @@ const dataSource = new ElasticDatasource(
timeField: '@time',
},
} as DataSourceInstanceSettings<ElasticsearchOptions>,
templateSrvStub as TemplateSrv,
timeSrvStub as TimeSrv
templateSrvStub as TemplateSrv
);
describe('transform prometheus query to elasticsearch query', () => {
it('Prometheus query with exact equals labels ( 2 labels ) and metric __name__', () => {

52
public/app/plugins/datasource/elasticsearch/specs/index_pattern.test.ts
Unescape View File

@ -1,7 +1,7 @@
///<amd-dependency path="test/specs/helpers" name="helpers" />
import { IndexPattern } from '../index_pattern';
import { toUtc, getLocale, setLocale } from '@grafana/data';
import { toUtc, getLocale, setLocale, dateTime } from '@grafana/data';
describe('IndexPattern', () => {
const originalLocale = getLocale();
@ -31,8 +31,8 @@ describe('IndexPattern', () => {
describe('no interval', () => {
test('should return correct index', () => {
const pattern = new IndexPattern('my-metrics');
const from = new Date(2015, 4, 30, 1, 2, 3);
const to = new Date(2015, 5, 1, 12, 5, 6);
const from = dateTime(new Date(2015, 4, 30, 1, 2, 3));
const to = dateTime(new Date(2015, 5, 1, 12, 5, 6));
expect(pattern.getIndexList(from, to)).toEqual('my-metrics');
});
});
@ -40,8 +40,8 @@ describe('IndexPattern', () => {
describe('daily', () => {
test('should return correct index list', () => {
const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
const from = new Date(1432940523000);
const to = new Date(1433153106000);
const from = dateTime(new Date(1432940523000));
const to = dateTime(new Date(1433153106000));
const expected = ['asd-2015.05.29', 'asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01'];
@ -51,8 +51,8 @@ describe('IndexPattern', () => {
test('should format date using western arabic numerals regardless of locale', () => {
setLocale('ar_SA'); // saudi-arabic, formatting for YYYY.MM.DD looks like "٢٠٢٠.٠٩.٠٣"
const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
const from = new Date(1432940523000);
const to = new Date(1433153106000);
const from = dateTime(new Date(1432940523000));
const to = dateTime(new Date(1433153106000));
const expected = ['asd-2015.05.29', 'asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01'];
@ -60,4 +60,42 @@ describe('IndexPattern', () => {
});
});
});
describe('when getting index list from single date', () => {
it('Should return index matching the starting time and subsequent ones', () => {
const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
const from = dateTime(new Date(1432940523000));
const expected = [
'asd-2015.05.29',
'asd-2015.05.30',
'asd-2015.05.31',
'asd-2015.06.01',
'asd-2015.06.02',
'asd-2015.06.03',
'asd-2015.06.04',
'asd-2015.06.05',
];
expect(pattern.getIndexList(from)).toEqual(expected);
});
it('Should return index matching the starting time and previous ones', () => {
const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
const to = dateTime(new Date(1432940523000));
const expected = [
'asd-2015.05.22',
'asd-2015.05.23',
'asd-2015.05.24',
'asd-2015.05.25',
'asd-2015.05.26',
'asd-2015.05.27',
'asd-2015.05.28',
'asd-2015.05.29',
];
expect(pattern.getIndexList(undefined, to)).toEqual(expected);
});
});
});

4
public/app/plugins/datasource/elasticsearch/types.ts
Unescape View File

@ -8,10 +8,12 @@ import {
MetricAggregationType,
} from './components/QueryEditor/MetricAggregationsEditor/aggregations';
export type Interval = 'Hourly' | 'Daily' | 'Weekly' | 'Monthly' | 'Yearly';
export interface ElasticsearchOptions extends DataSourceJsonData {
timeField: string;
esVersion: number;
interval?: string;
interval?: Interval;
timeInterval: string;
maxConcurrentShardRequests?: number;
logMessageField?: string;

Write Preview
Loading…
Cancel
Save