diff --git a/.betterer.results b/.betterer.results index 02f138aeec2..c1f01ae34a7 100644 --- a/.betterer.results +++ b/.betterer.results @@ -4606,6 +4606,9 @@ exports[`better eslint`] = { "public/app/plugins/datasource/loki/LiveStreams.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] ], + "public/app/plugins/datasource/loki/LogContextProvider.ts:5381": [ + [0, 0, 0, "Do not use any type assertions.", "0"] + ], "public/app/plugins/datasource/loki/components/LokiLabelBrowser.tsx:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"], [0, 0, 0, "Do not use any type assertions.", "1"] @@ -4632,10 +4635,9 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "1"], [0, 0, 0, "Unexpected any. Specify a different type.", "2"], [0, 0, 0, "Unexpected any. Specify a different type.", "3"], - [0, 0, 0, "Do not use any type assertions.", "4"], + [0, 0, 0, "Unexpected any. Specify a different type.", "4"], [0, 0, 0, "Unexpected any. Specify a different type.", "5"], - [0, 0, 0, "Unexpected any. Specify a different type.", "6"], - [0, 0, 0, "Unexpected any. Specify a different type.", "7"] + [0, 0, 0, "Unexpected any. Specify a different type.", "6"] ], "public/app/plugins/datasource/loki/getDerivedFields.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] diff --git a/public/app/plugins/datasource/loki/LogContextProvider.ts b/public/app/plugins/datasource/loki/LogContextProvider.ts new file mode 100644 index 00000000000..7549e6dede3 --- /dev/null +++ b/public/app/plugins/datasource/loki/LogContextProvider.ts @@ -0,0 +1,152 @@ +import { FieldCache, FieldType, LogRowModel, TimeRange, toUtc } from '@grafana/data'; +import { DataQuery } from '@grafana/schema'; + +import LokiLanguageProvider from './LanguageProvider'; +import { LokiContextUi } from './components/LokiContextUi'; +import { REF_ID_STARTER_LOG_ROW_CONTEXT } from './datasource'; +import { escapeLabelValueInExactSelector } from './languageUtils'; +import { addLabelToQuery, addParserToQuery } from './modifyQuery'; +import { getParserFromQuery } from './queryUtils'; +import { ContextFilter, LokiQuery, LokiQueryDirection, LokiQueryType } from './types'; + +export class LogContextProvider { + languageProvider: LokiLanguageProvider; + onContextClose: (() => void) | undefined; + + constructor(languageProvider: LokiLanguageProvider) { + this.languageProvider = languageProvider; + } + + async prepareLogRowContextQueryTarget( + row: LogRowModel, + limit: number, + direction: 'BACKWARD' | 'FORWARD', + origQuery?: DataQuery + ): Promise<{ query: LokiQuery; range: TimeRange }> { + let expr = await this.prepareContextExpr(row, origQuery); + + const contextTimeBuffer = 2 * 60 * 60 * 1000; // 2h buffer + + const queryDirection = direction === 'FORWARD' ? LokiQueryDirection.Forward : LokiQueryDirection.Backward; + + const query: LokiQuery = { + expr, + queryType: LokiQueryType.Range, + refId: `${REF_ID_STARTER_LOG_ROW_CONTEXT}${row.dataFrame.refId || ''}`, + maxLines: limit, + direction: queryDirection, + }; + + const fieldCache = new FieldCache(row.dataFrame); + const tsField = fieldCache.getFirstFieldOfType(FieldType.time); + if (tsField === undefined) { + throw new Error('loki: data frame missing time-field, should never happen'); + } + const tsValue = tsField.values.get(row.rowIndex); + const timestamp = toUtc(tsValue); + + const range = + queryDirection === LokiQueryDirection.Forward + ? { + // start param in Loki API is inclusive so we'll have to filter out the row that this request is based from + // and any other that were logged in the same ns but before the row. Right now these rows will be lost + // because the are before but came it he response that should return only rows after. + from: timestamp, + // convert to ns, we lose some precision here but it is not that important at the far points of the context + to: toUtc(row.timeEpochMs + contextTimeBuffer), + } + : { + // convert to ns, we lose some precision here but it is not that important at the far points of the context + from: toUtc(row.timeEpochMs - contextTimeBuffer), + to: timestamp, + }; + + return { + query, + range: { + from: range.from, + to: range.to, + raw: range, + }, + }; + } + + getLogRowContextUi(row: LogRowModel, runContextQuery: () => void): React.ReactNode { + const updateFilter = (contextFilters: ContextFilter[]) => { + this.prepareContextExpr = async (row: LogRowModel, origQuery?: DataQuery) => { + await this.languageProvider.start(); + const labels = this.languageProvider.getLabelKeys(); + + let expr = contextFilters + .map((filter) => { + const label = filter.value; + if (filter && !filter.fromParser && filter.enabled && labels.includes(label)) { + // escape backslashes in label as users can't escape them by themselves + return `${label}="${escapeLabelValueInExactSelector(row.labels[label])}"`; + } + return ''; + }) + // Filter empty strings + .filter((label) => !!label) + .join(','); + + expr = `{${expr}}`; + + const parserContextFilters = contextFilters.filter((filter) => filter.fromParser && filter.enabled); + if (parserContextFilters.length) { + // we should also filter for labels from parsers, let's find the right parser + if (origQuery) { + const parser = getParserFromQuery((origQuery as LokiQuery).expr); + if (parser) { + expr = addParserToQuery(expr, parser); + } + } + for (const filter of parserContextFilters) { + if (filter.enabled) { + expr = addLabelToQuery(expr, filter.label, '=', row.labels[filter.label]); + } + } + } + return expr; + }; + if (runContextQuery) { + runContextQuery(); + } + }; + + // we need to cache this function so that it doesn't get recreated on every render + this.onContextClose = + this.onContextClose ?? + (() => { + this.prepareContextExpr = this.prepareContextExprWithoutParsedLabels; + }); + + return LokiContextUi({ + row, + updateFilter, + languageProvider: this.languageProvider, + onClose: this.onContextClose, + }); + } + + async prepareContextExpr(row: LogRowModel, origQuery?: DataQuery): Promise { + return await this.prepareContextExprWithoutParsedLabels(row, origQuery); + } + + private async prepareContextExprWithoutParsedLabels(row: LogRowModel, origQuery?: DataQuery): Promise { + await this.languageProvider.start(); + const labels = this.languageProvider.getLabelKeys(); + const expr = Object.keys(row.labels) + .map((label: string) => { + if (labels.includes(label)) { + // escape backslashes in label as users can't escape them by themselves + return `${label}="${escapeLabelValueInExactSelector(row.labels[label])}"`; + } + return ''; + }) + .filter((label) => !!label) + .join(','); + + return `{${expr}}`; + } +} diff --git a/public/app/plugins/datasource/loki/LogContextProvidet.test.ts b/public/app/plugins/datasource/loki/LogContextProvidet.test.ts new file mode 100644 index 00000000000..26a4989059e --- /dev/null +++ b/public/app/plugins/datasource/loki/LogContextProvidet.test.ts @@ -0,0 +1,60 @@ +import { FieldType, LogRowModel, MutableDataFrame } from '@grafana/data'; + +import LokiLanguageProvider from './LanguageProvider'; +import { LogContextProvider } from './LogContextProvider'; + +const defaultLanguageProviderMock = { + start: jest.fn(), + getLabelKeys: jest.fn(() => ['foo']), +} as unknown as LokiLanguageProvider; + +const defaultLogRow = { + rowIndex: 0, + dataFrame: new MutableDataFrame({ + fields: [ + { + name: 'ts', + type: FieldType.time, + values: [0], + }, + ], + }), + labels: { bar: 'baz', foo: 'uniqueParsedLabel' }, + uid: '1', +} as unknown as LogRowModel; + +describe('new context ui', () => { + it('returns expression with 1 label', async () => { + const lcp = new LogContextProvider(defaultLanguageProviderMock); + const result = await lcp.prepareContextExpr(defaultLogRow); + + expect(result).toEqual('{foo="uniqueParsedLabel"}'); + }); + + it('returns empty expression for parsed labels', async () => { + const languageProviderMock = { + ...defaultLanguageProviderMock, + getLabelKeys: jest.fn(() => []), + } as unknown as LokiLanguageProvider; + + const lcp = new LogContextProvider(languageProviderMock); + const result = await lcp.prepareContextExpr(defaultLogRow); + + expect(result).toEqual('{}'); + }); +}); + +describe('prepareLogRowContextQueryTarget', () => { + const lcp = new LogContextProvider(defaultLanguageProviderMock); + it('creates query with only labels from /labels API', async () => { + const contextQuery = await lcp.prepareLogRowContextQueryTarget(defaultLogRow, 10, 'BACKWARD'); + + expect(contextQuery.query.expr).toContain('uniqueParsedLabel'); + expect(contextQuery.query.expr).not.toContain('baz'); + }); + + it('should call languageProvider.start to fetch labels', async () => { + await lcp.prepareLogRowContextQueryTarget(defaultLogRow, 10, 'BACKWARD'); + expect(lcp.languageProvider.start).toBeCalled(); + }); +}); diff --git a/public/app/plugins/datasource/loki/datasource.test.ts b/public/app/plugins/datasource/loki/datasource.test.ts index 1ff0e55149b..bfa63d0791d 100644 --- a/public/app/plugins/datasource/loki/datasource.test.ts +++ b/public/app/plugins/datasource/loki/datasource.test.ts @@ -13,8 +13,6 @@ import { DataSourceInstanceSettings, dateTime, FieldType, - LogRowModel, - MutableDataFrame, SupplementaryQueryType, } from '@grafana/data'; import { @@ -839,57 +837,6 @@ describe('LokiDatasource', () => { }); }); - describe('prepareLogRowContextQueryTarget', () => { - const ds = createLokiDatasource(templateSrvStub); - it('creates query with only labels from /labels API', async () => { - const row: LogRowModel = { - rowIndex: 0, - dataFrame: new MutableDataFrame({ - fields: [ - { - name: 'ts', - type: FieldType.time, - values: [0], - }, - ], - }), - labels: { bar: 'baz', foo: 'uniqueParsedLabel' }, - uid: '1', - } as unknown as LogRowModel; - - //Mock stored labels to only include "bar" label - jest.spyOn(ds.languageProvider, 'start').mockImplementation(() => Promise.resolve([])); - jest.spyOn(ds.languageProvider, 'getLabelKeys').mockImplementation(() => ['bar']); - const contextQuery = await ds.prepareLogRowContextQueryTarget(row, 10, 'BACKWARD'); - - expect(contextQuery.query.expr).toContain('baz'); - expect(contextQuery.query.expr).not.toContain('uniqueParsedLabel'); - }); - - it('should call languageProvider.start to fetch labels', async () => { - const row: LogRowModel = { - rowIndex: 0, - dataFrame: new MutableDataFrame({ - fields: [ - { - name: 'ts', - type: FieldType.time, - values: [0], - }, - ], - }), - labels: { bar: 'baz', foo: 'uniqueParsedLabel' }, - uid: '1', - } as unknown as LogRowModel; - - //Mock stored labels to only include "bar" label - jest.spyOn(ds.languageProvider, 'start').mockImplementation(() => Promise.resolve([])); - await ds.prepareLogRowContextQueryTarget(row, 10, 'BACKWARD'); - - expect(ds.languageProvider.start).toBeCalled(); - }); - }); - describe('logs volume data provider', () => { let ds: LokiDatasource; beforeEach(() => { @@ -1228,57 +1175,3 @@ function makeAnnotationQueryRequest(options = {}): AnnotationQueryRequest { - it('returns expression with 1 label', async () => { - const ds = createLokiDatasource(templateSrvStub); - - const row: LogRowModel = { - rowIndex: 0, - dataFrame: new MutableDataFrame({ - fields: [ - { - name: 'ts', - type: FieldType.time, - values: [0], - }, - ], - }), - labels: { bar: 'baz', foo: 'uniqueParsedLabel' }, - uid: '1', - } as unknown as LogRowModel; - - jest.spyOn(ds.languageProvider, 'start').mockImplementation(() => Promise.resolve([])); - jest.spyOn(ds.languageProvider, 'getLabelKeys').mockImplementation(() => ['foo']); - - const result = await ds.prepareContextExpr(row); - - expect(result).toEqual('{foo="uniqueParsedLabel"}'); - }); - - it('returns empty expression for parsed labels', async () => { - const ds = createLokiDatasource(templateSrvStub); - - const row: LogRowModel = { - rowIndex: 0, - dataFrame: new MutableDataFrame({ - fields: [ - { - name: 'ts', - type: FieldType.time, - values: [0], - }, - ], - }), - labels: { bar: 'baz', foo: 'uniqueParsedLabel' }, - uid: '1', - } as unknown as LogRowModel; - - jest.spyOn(ds.languageProvider, 'start').mockImplementation(() => Promise.resolve([])); - jest.spyOn(ds.languageProvider, 'getLabelKeys').mockImplementation(() => []); - - const result = await ds.prepareContextExpr(row); - - expect(result).toEqual('{}'); - }); -}); diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index 04f99951524..06f1c31092b 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -31,7 +31,6 @@ import { rangeUtil, ScopedVars, TimeRange, - toUtc, } from '@grafana/data'; import { BackendSrvRequest, config, DataSourceWithBackend, FetchError } from '@grafana/runtime'; import { DataQuery } from '@grafana/schema'; @@ -48,10 +47,10 @@ import { replaceVariables, returnVariables } from '../prometheus/querybuilder/sh import LanguageProvider from './LanguageProvider'; import { LiveStreams, LokiLiveTarget } from './LiveStreams'; +import { LogContextProvider } from './LogContextProvider'; import { transformBackendResult } from './backendResultTransformer'; import { LokiAnnotationsQueryEditor } from './components/AnnotationsQueryEditor'; -import { LokiContextUi } from './components/LokiContextUi'; -import { escapeLabelValueInExactSelector, escapeLabelValueInSelector, isRegexSelector } from './languageUtils'; +import { escapeLabelValueInSelector, isRegexSelector } from './languageUtils'; import { labelNamesRegex, labelValuesRegex } from './migrations/variableQueryMigrations'; import { addLabelFormatToQuery, @@ -72,7 +71,6 @@ import { getLogQueryFromMetricsQuery, getNormalizedLokiQuery, getStreamSelectorsFromQuery, - getParserFromQuery, isLogsQuery, isValidQuery, requestSupportsSplitting, @@ -81,10 +79,8 @@ import { sortDataFrameByTime, SortDirection } from './sortDataFrame'; import { doLokiChannelStream } from './streaming'; import { trackQuery } from './tracking'; import { - ContextFilter, LokiOptions, LokiQuery, - LokiQueryDirection, LokiQueryType, LokiVariableQuery, LokiVariableQueryType, @@ -135,8 +131,8 @@ export class LokiDatasource { private streams = new LiveStreams(); languageProvider: LanguageProvider; + logContextProvider: LogContextProvider; maxLines: number; - onContextClose: (() => void) | undefined; constructor( private instanceSettings: DataSourceInstanceSettings, @@ -152,6 +148,7 @@ export class LokiDatasource QueryEditor: LokiAnnotationsQueryEditor, }; this.variables = new LokiVariableSupport(this); + this.logContextProvider = new LogContextProvider(this.languageProvider); } getDataProvider( @@ -654,7 +651,12 @@ export class LokiDatasource ): Promise<{ data: DataFrame[] }> => { const direction = (options && options.direction) || 'BACKWARD'; const limit = (options && options.limit) || 10; - const { query, range } = await this.prepareLogRowContextQueryTarget(row, limit, direction, origQuery); + const { query, range } = await this.logContextProvider.prepareLogRowContextQueryTarget( + row, + limit, + direction, + origQuery + ); const processDataFrame = (frame: DataFrame): DataFrame => { // log-row-context requires specific field-names to work, so we set them here: "ts", "line", "id" @@ -717,137 +719,8 @@ export class LokiDatasource ); }; - prepareLogRowContextQueryTarget = async ( - row: LogRowModel, - limit: number, - direction: 'BACKWARD' | 'FORWARD', - origQuery?: DataQuery - ): Promise<{ query: LokiQuery; range: TimeRange }> => { - let expr = await this.prepareContextExpr(row, origQuery); - - const contextTimeBuffer = 2 * 60 * 60 * 1000; // 2h buffer - - const queryDirection = direction === 'FORWARD' ? LokiQueryDirection.Forward : LokiQueryDirection.Backward; - - const query: LokiQuery = { - expr, - queryType: LokiQueryType.Range, - refId: `${REF_ID_STARTER_LOG_ROW_CONTEXT}${row.dataFrame.refId || ''}`, - maxLines: limit, - direction: queryDirection, - }; - - const fieldCache = new FieldCache(row.dataFrame); - const tsField = fieldCache.getFirstFieldOfType(FieldType.time); - if (tsField === undefined) { - throw new Error('loki: dataframe missing time-field, should never happen'); - } - const tsValue = tsField.values.get(row.rowIndex); - const timestamp = toUtc(tsValue); - - const range = - queryDirection === LokiQueryDirection.Forward - ? { - // start param in Loki API is inclusive so we'll have to filter out the row that this request is based from - // and any other that were logged in the same ns but before the row. Right now these rows will be lost - // because the are before but came it he response that should return only rows after. - from: timestamp, - // convert to ns, we lose some precision here but it is not that important at the far points of the context - to: toUtc(row.timeEpochMs + contextTimeBuffer), - } - : { - // convert to ns, we lose some precision here but it is not that important at the far points of the context - from: toUtc(row.timeEpochMs - contextTimeBuffer), - to: timestamp, - }; - - return { - query, - range: { - from: range.from, - to: range.to, - raw: range, - }, - }; - }; - - async prepareContextExprWithoutParsedLabels(row: LogRowModel, origQuery?: DataQuery): Promise { - await this.languageProvider.start(); - const labels = this.languageProvider.getLabelKeys(); - const expr = Object.keys(row.labels) - .map((label: string) => { - if (labels.includes(label)) { - // escape backslashes in label as users can't escape them by themselves - return `${label}="${escapeLabelValueInExactSelector(row.labels[label])}"`; - } - return ''; - }) - .filter((label) => !!label) - .join(','); - - return `{${expr}}`; - } - - async prepareContextExpr(row: LogRowModel, origQuery?: DataQuery): Promise { - return await this.prepareContextExprWithoutParsedLabels(row, origQuery); - } - getLogRowContextUi(row: LogRowModel, runContextQuery: () => void): React.ReactNode { - const updateFilter = (contextFilters: ContextFilter[]) => { - this.prepareContextExpr = async (row: LogRowModel, origQuery?: DataQuery) => { - await this.languageProvider.start(); - const labels = this.languageProvider.getLabelKeys(); - - let expr = contextFilters - .map((filter) => { - const label = filter.value; - if (filter && !filter.fromParser && filter.enabled && labels.includes(label)) { - // escape backslashes in label as users can't escape them by themselves - return `${label}="${escapeLabelValueInExactSelector(row.labels[label])}"`; - } - return ''; - }) - // Filter empty strings - .filter((label) => !!label) - .join(','); - - expr = `{${expr}}`; - - const parserContextFilters = contextFilters.filter((filter) => filter.fromParser && filter.enabled); - if (parserContextFilters.length) { - // we should also filter for labels from parsers, let's find the right parser - if (origQuery) { - const parser = getParserFromQuery((origQuery as LokiQuery).expr); - if (parser) { - expr = addParserToQuery(expr, parser); - } - } - for (const filter of parserContextFilters) { - if (filter.enabled) { - expr = addLabelToQuery(expr, filter.label, '=', row.labels[filter.label]); - } - } - } - return expr; - }; - if (runContextQuery) { - runContextQuery(); - } - }; - - // we need to cache this function so that it doesn't get recreated on every render - this.onContextClose = - this.onContextClose ?? - (() => { - this.prepareContextExpr = this.prepareContextExprWithoutParsedLabels; - }); - - return LokiContextUi({ - row, - updateFilter, - languageProvider: this.languageProvider, - onClose: this.onContextClose, - }); + return this.logContextProvider.getLogRowContextUi(row, runContextQuery); } testDatasource(): Promise<{ status: string; message: string }> {