diff --git a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx index 0cc38d6f63a..bdfc4c0ff53 100644 --- a/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx +++ b/public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { EventsWithValidation, regexValidation, LegacyForms } from '@grafana/ui'; const { Select, Input, FormField } = LegacyForms; -import { ElasticsearchOptions } from '../types'; +import { ElasticsearchOptions, Interval } from '../types'; import { DataSourceSettings, SelectableValue } from '@grafana/data'; const indexPatternTypes = [ @@ -170,7 +170,9 @@ const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['va }); }; -const intervalHandler = (value: Props['value'], onChange: Props['onChange']) => (option: SelectableValue) => { +const intervalHandler = (value: Props['value'], onChange: Props['onChange']) => ( + option: SelectableValue +) => { const { database } = value; // If option value is undefined it will send its label instead so we have to convert made up value to undefined here. const newInterval = option.value === 'none' ? undefined : option.value; diff --git a/public/app/plugins/datasource/elasticsearch/datasource.test.ts b/public/app/plugins/datasource/elasticsearch/datasource.test.ts index 33bc654b68b..2baf8844013 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.test.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.test.ts @@ -15,7 +15,6 @@ import { import _ from 'lodash'; import { ElasticDatasource, enhanceDataFrame } from './datasource'; import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__ -import { TimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { TemplateSrv } from 'app/features/templating/template_srv'; import { ElasticsearchOptions, ElasticsearchQuery } from './types'; import { Filters } from './components/QueryEditor/BucketAggregationsEditor/aggregations'; @@ -61,8 +60,6 @@ describe('ElasticDatasource', function(this: any) { getAdhocFilters: jest.fn(() => []), }; - const timeSrv: any = createTimeSrv('now-1h'); - interface TestContext { ds: ElasticDatasource; } @@ -88,15 +85,8 @@ describe('ElasticDatasource', function(this: any) { } function createDatasource(instanceSettings: DataSourceInstanceSettings) { - createDatasourceWithTime(instanceSettings, timeSrv as TimeSrv); - } - - function createDatasourceWithTime( - instanceSettings: DataSourceInstanceSettings, - timeSrv: TimeSrv - ) { instanceSettings.jsonData = instanceSettings.jsonData || ({} as ElasticsearchOptions); - ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv, timeSrv); + ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv); } describe('When testing datasource with index pattern', () => { @@ -506,14 +496,11 @@ describe('ElasticDatasource', function(this: any) { }; beforeEach(() => { - createDatasourceWithTime( - { - url: ELASTICSEARCH_MOCK_URL, - database: '[asd-]YYYY.MM.DD', - jsonData: { interval: 'Daily', esVersion: 50 } as ElasticsearchOptions, - } as DataSourceInstanceSettings, - twoWeekTimeSrv - ); + createDatasource({ + url: ELASTICSEARCH_MOCK_URL, + database: '[asd-]YYYY.MM.DD', + jsonData: { interval: 'Daily', esVersion: 50 } as ElasticsearchOptions, + } as DataSourceInstanceSettings); }); it('should return fields of the newest available index', async () => { @@ -534,7 +521,8 @@ describe('ElasticDatasource', function(this: any) { return Promise.reject({ status: 404 }); }); - const fieldObjects = await ctx.ds.getFields(); + const range = twoWeekTimeSrv.timeRange(); + const fieldObjects = await ctx.ds.getFields(undefined, range); const fields = _.map(fieldObjects, 'text'); expect(fields).toEqual(['@timestamp', 'beat.hostname']); @@ -545,6 +533,7 @@ describe('ElasticDatasource', function(this: any) { .subtract(2, 'day') .format('YYYY.MM.DD'); + const range = twoWeekTimeSrv.timeRange(); datasourceRequestMock.mockImplementation(options => { if (options.url === `${ELASTICSEARCH_MOCK_URL}/asd-${twoDaysBefore}/_mapping`) { return Promise.resolve(basicResponse); @@ -554,7 +543,7 @@ describe('ElasticDatasource', function(this: any) { expect.assertions(2); try { - await ctx.ds.getFields(); + await ctx.ds.getFields(undefined, range); } catch (e) { expect(e).toStrictEqual({ status: 500 }); expect(datasourceRequestMock).toBeCalledTimes(1); @@ -562,13 +551,14 @@ describe('ElasticDatasource', function(this: any) { }); it('should not retry more than 7 indices', async () => { + const range = twoWeekTimeSrv.timeRange(); datasourceRequestMock.mockImplementation(() => { return Promise.reject({ status: 404 }); }); expect.assertions(2); try { - await ctx.ds.getFields(); + await ctx.ds.getFields(undefined, range); } catch (e) { expect(e).toStrictEqual({ status: 404 }); expect(datasourceRequestMock).toBeCalledTimes(7); @@ -840,8 +830,7 @@ describe('ElasticDatasource', function(this: any) { timeField: '@time', }, } as DataSourceInstanceSettings, - templateSrv as TemplateSrv, - timeSrv as TimeSrv + templateSrv as TemplateSrv ); (dataSource as any).post = jest.fn(() => Promise.resolve({ responses: [] })); dataSource.query(createElasticQuery()); diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index 656511c6e22..061bbca3f06 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -12,6 +12,10 @@ import { LogRowModel, Field, MetricFindValue, + TimeRange, + DefaultTimeRange, + DateTime, + dateTime, } from '@grafana/data'; import LanguageProvider from './language_provider'; import { ElasticResponse } from './elastic_response'; @@ -21,7 +25,6 @@ import { toUtc } from '@grafana/data'; import { defaultBucketAgg, hasMetricOfType } from './query_def'; import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; -import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery } from './types'; import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider'; import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils'; @@ -65,8 +68,7 @@ export class ElasticDatasource extends DataSourceApi, - private readonly templateSrv: TemplateSrv = getTemplateSrv(), - private readonly timeSrv: TimeSrv = getTimeSrv() + private readonly templateSrv: TemplateSrv = getTemplateSrv() ) { super(instanceSettings); this.basicAuth = instanceSettings.basicAuth; @@ -140,9 +142,8 @@ export class ElasticDatasource extends DataSourceApi { results.data.$$config = results.config; @@ -370,7 +371,7 @@ export class ElasticDatasource extends DataSourceApi => { const sortField = row.dataFrame.fields.find(f => f.name === 'sort'); const searchAfter = sortField?.values.get(row.rowIndex) || [row.timeEpochMs]; - const range = this.timeSrv.timeRange(); - const direction = options?.direction === 'FORWARD' ? 'asc' : 'desc'; - const header = this.getQueryHeader('query_then_fetch', range.from, range.to); + const sort = options?.direction === 'FORWARD' ? 'asc' : 'desc'; + + const header = + options?.direction === 'FORWARD' + ? this.getQueryHeader('query_then_fetch', dateTime(row.timeEpochMs)) + : this.getQueryHeader('query_then_fetch', undefined, dateTime(row.timeEpochMs)); + const limit = options?.limit ?? 10; const esQuery = JSON.stringify({ size: limit, @@ -459,8 +464,7 @@ export class ElasticDatasource extends DataSourceApi { + async getFields(type?: string, range?: TimeRange): Promise { const configuredEsVersion = this.esVersion; - return this.get('/_mapping').then((result: any) => { + return this.get('/_mapping', range).then((result: any) => { const typeMap: any = { float: 'number', double: 'number', @@ -663,8 +667,7 @@ export class ElasticDatasource extends DataSourceApi= 5 ? 'query_then_fetch' : 'count'; const header = this.getQueryHeader(searchType, range.from, range.to); let esQuery = JSON.stringify(this.queryBuilder.getTermsQuery(queryDef)); @@ -698,18 +701,19 @@ export class ElasticDatasource extends DataSourceApi { + metricFindQuery(query: string, options?: any): Promise { + const range = options?.range; const parsedQuery = JSON.parse(query); if (query) { if (parsedQuery.find === 'fields') { parsedQuery.field = this.templateSrv.replace(parsedQuery.field, {}, 'lucene'); - return this.getFields(query); + return this.getFields(query, range); } if (parsedQuery.find === 'terms') { parsedQuery.field = this.templateSrv.replace(parsedQuery.field, {}, 'lucene'); parsedQuery.query = this.templateSrv.replace(parsedQuery.query || '*', {}, 'lucene'); - return this.getTerms(query); + return this.getTerms(query, range); } } diff --git a/public/app/plugins/datasource/elasticsearch/index_pattern.ts b/public/app/plugins/datasource/elasticsearch/index_pattern.ts index 0e946c71a6a..f8e7664cd7a 100644 --- a/public/app/plugins/datasource/elasticsearch/index_pattern.ts +++ b/public/app/plugins/datasource/elasticsearch/index_pattern.ts @@ -1,9 +1,18 @@ -import { toUtc, dateTime } from '@grafana/data'; +import { toUtc, dateTime, DateTime, DurationUnit } from '@grafana/data'; +import { Interval } from './types'; + +type IntervalMap = Record< + Interval, + { + startOf: DurationUnit; + amount: DurationUnit; + } +>; -const intervalMap: any = { +const intervalMap: IntervalMap = { Hourly: { startOf: 'hour', amount: 'hours' }, Daily: { startOf: 'day', amount: 'days' }, - Weekly: { startOf: 'isoWeek', amount: 'weeks' }, + Weekly: { startOf: 'week', amount: 'weeks' }, Monthly: { startOf: 'month', amount: 'months' }, Yearly: { startOf: 'year', amount: 'years' }, }; @@ -11,7 +20,7 @@ const intervalMap: any = { export class IndexPattern { private dateLocale = 'en'; - constructor(private pattern: any, private interval?: string) {} + constructor(private pattern: string, private interval?: keyof typeof intervalMap) {} getIndexForToday() { if (this.interval) { @@ -23,16 +32,21 @@ export class IndexPattern { } } - getIndexList(from: any, to: any) { + getIndexList(from?: DateTime, to?: DateTime) { + // When no `from` or `to` is provided, we request data from 7 subsequent/previous indices + // for the provided index pattern. + // This is useful when requesting log context where the only time data we have is the log + // timestamp. + const indexOffset = 7; if (!this.interval) { return this.pattern; } const intervalInfo = intervalMap[this.interval]; - const start = dateTime(from) + const start = dateTime(from || dateTime(to).add(-indexOffset, intervalInfo.amount)) .utc() .startOf(intervalInfo.startOf); - const endEpoch = dateTime(to) + const endEpoch = dateTime(to || dateTime(from).add(indexOffset, intervalInfo.amount)) .utc() .startOf(intervalInfo.startOf) .valueOf(); diff --git a/public/app/plugins/datasource/elasticsearch/language_provider.test.ts b/public/app/plugins/datasource/elasticsearch/language_provider.test.ts index a3b2ed3c472..ac17e77a8be 100644 --- a/public/app/plugins/datasource/elasticsearch/language_provider.test.ts +++ b/public/app/plugins/datasource/elasticsearch/language_provider.test.ts @@ -1,25 +1,15 @@ import LanguageProvider from './language_provider'; import { PromQuery } from '../prometheus/types'; import { ElasticDatasource } from './datasource'; -import { DataSourceInstanceSettings, dateTime } from '@grafana/data'; +import { DataSourceInstanceSettings } from '@grafana/data'; import { ElasticsearchOptions } from './types'; import { TemplateSrv } from '../../../features/templating/template_srv'; -import { TimeSrv } from '../../../features/dashboard/services/TimeSrv'; const templateSrvStub = { getAdhocFilters: jest.fn(() => [] as any[]), replace: jest.fn((a: string) => a), } as any; -const timeSrvStub = { - timeRange(): any { - return { - from: dateTime(1531468681), - to: dateTime(1531489712), - }; - }, -} as any; - const dataSource = new ElasticDatasource( { url: 'http://es.com', @@ -30,8 +20,7 @@ const dataSource = new ElasticDatasource( timeField: '@time', }, } as DataSourceInstanceSettings, - templateSrvStub as TemplateSrv, - timeSrvStub as TimeSrv + templateSrvStub as TemplateSrv ); describe('transform prometheus query to elasticsearch query', () => { it('Prometheus query with exact equals labels ( 2 labels ) and metric __name__', () => { diff --git a/public/app/plugins/datasource/elasticsearch/specs/index_pattern.test.ts b/public/app/plugins/datasource/elasticsearch/specs/index_pattern.test.ts index b6556a1b0a6..99fab513f47 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/index_pattern.test.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/index_pattern.test.ts @@ -1,7 +1,7 @@ /// import { IndexPattern } from '../index_pattern'; -import { toUtc, getLocale, setLocale } from '@grafana/data'; +import { toUtc, getLocale, setLocale, dateTime } from '@grafana/data'; describe('IndexPattern', () => { const originalLocale = getLocale(); @@ -31,8 +31,8 @@ describe('IndexPattern', () => { describe('no interval', () => { test('should return correct index', () => { const pattern = new IndexPattern('my-metrics'); - const from = new Date(2015, 4, 30, 1, 2, 3); - const to = new Date(2015, 5, 1, 12, 5, 6); + const from = dateTime(new Date(2015, 4, 30, 1, 2, 3)); + const to = dateTime(new Date(2015, 5, 1, 12, 5, 6)); expect(pattern.getIndexList(from, to)).toEqual('my-metrics'); }); }); @@ -40,8 +40,8 @@ describe('IndexPattern', () => { describe('daily', () => { test('should return correct index list', () => { const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily'); - const from = new Date(1432940523000); - const to = new Date(1433153106000); + const from = dateTime(new Date(1432940523000)); + const to = dateTime(new Date(1433153106000)); const expected = ['asd-2015.05.29', 'asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01']; @@ -51,8 +51,8 @@ describe('IndexPattern', () => { test('should format date using western arabic numerals regardless of locale', () => { setLocale('ar_SA'); // saudi-arabic, formatting for YYYY.MM.DD looks like "٢٠٢٠.٠٩.٠٣" const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily'); - const from = new Date(1432940523000); - const to = new Date(1433153106000); + const from = dateTime(new Date(1432940523000)); + const to = dateTime(new Date(1433153106000)); const expected = ['asd-2015.05.29', 'asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01']; @@ -60,4 +60,42 @@ describe('IndexPattern', () => { }); }); }); + + describe('when getting index list from single date', () => { + it('Should return index matching the starting time and subsequent ones', () => { + const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily'); + const from = dateTime(new Date(1432940523000)); + + const expected = [ + 'asd-2015.05.29', + 'asd-2015.05.30', + 'asd-2015.05.31', + 'asd-2015.06.01', + 'asd-2015.06.02', + 'asd-2015.06.03', + 'asd-2015.06.04', + 'asd-2015.06.05', + ]; + + expect(pattern.getIndexList(from)).toEqual(expected); + }); + + it('Should return index matching the starting time and previous ones', () => { + const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily'); + const to = dateTime(new Date(1432940523000)); + + const expected = [ + 'asd-2015.05.22', + 'asd-2015.05.23', + 'asd-2015.05.24', + 'asd-2015.05.25', + 'asd-2015.05.26', + 'asd-2015.05.27', + 'asd-2015.05.28', + 'asd-2015.05.29', + ]; + + expect(pattern.getIndexList(undefined, to)).toEqual(expected); + }); + }); }); diff --git a/public/app/plugins/datasource/elasticsearch/types.ts b/public/app/plugins/datasource/elasticsearch/types.ts index 5a92902ab94..fee88a5984a 100644 --- a/public/app/plugins/datasource/elasticsearch/types.ts +++ b/public/app/plugins/datasource/elasticsearch/types.ts @@ -8,10 +8,12 @@ import { MetricAggregationType, } from './components/QueryEditor/MetricAggregationsEditor/aggregations'; +export type Interval = 'Hourly' | 'Daily' | 'Weekly' | 'Monthly' | 'Yearly'; + export interface ElasticsearchOptions extends DataSourceJsonData { timeField: string; esVersion: number; - interval?: string; + interval?: Interval; timeInterval: string; maxConcurrentShardRequests?: number; logMessageField?: string;