diff --git a/public/app/plugins/datasource/loki/addToQuery.ts b/public/app/plugins/datasource/loki/addToQuery.ts index 61706170ca8..9dc08a3fbdc 100644 --- a/public/app/plugins/datasource/loki/addToQuery.ts +++ b/public/app/plugins/datasource/loki/addToQuery.ts @@ -1,3 +1,5 @@ +import { sortBy } from 'lodash'; + import { parser } from '@grafana/lezer-logql'; import { QueryBuilderLabelFilter } from '../prometheus/querybuilder/shared/types'; @@ -76,6 +78,18 @@ export function addNoPipelineErrorToQuery(query: string): string { return addFilterAsLabelFilter(query, parserPositions, filter); } +/** + * Adds label format to existing query. Useful for query modification for hints. + * It uses LogQL parser to find log query and add label format at the end. + * + * @param query + * @param labelFormat + */ +export function addLabelFormatToQuery(query: string, labelFormat: { originalLabel: string; renameTo: string }): string { + const logQueryPositions = getLogQueryPositions(query); + return addLabelFormat(query, logQueryPositions, labelFormat); +} + /** * Parse the string and get all Selector positions in the query together with parsed representation of the * selector. @@ -149,6 +163,50 @@ function getLineFiltersPositions(query: string): Position[] { return positions; } +/** + * Parse the string and get all Log query positions in the query. + * @param query + */ +function getLogQueryPositions(query: string): Position[] { + const tree = parser.parse(query); + const positions: Position[] = []; + tree.iterate({ + enter: (type, from, to, get): false | void => { + if (type.name === 'LogExpr') { + positions.push({ from, to }); + return false; + } + + // This is a case in metrics query + if (type.name === 'LogRangeExpr') { + // Unfortunately, LogRangeExpr includes both log and non-log (e.g. Duration/Range/...) parts of query. + // We get position of all log-parts within LogRangeExpr: Selector, PipelineExpr and UnwrapExpr. + const logPartsPositions: Position[] = []; + const selector = get().getChild('Selector'); + if (selector) { + logPartsPositions.push({ from: selector.from, to: selector.to }); + } + + const pipeline = get().getChild('PipelineExpr'); + if (pipeline) { + logPartsPositions.push({ from: pipeline.from, to: pipeline.to }); + } + + const unwrap = get().getChild('UnwrapExpr'); + if (unwrap) { + logPartsPositions.push({ from: unwrap.from, to: unwrap.to }); + } + + // We sort them and then pick "from" from first position and "to" from last position. + const sorted = sortBy(logPartsPositions, (position) => position.to); + positions.push({ from: sorted[0].from, to: sorted[sorted.length - 1].to }); + return false; + } + }, + }); + return positions; +} + function toLabelFilter(key: string, value: string, operator: string): QueryBuilderLabelFilter { // We need to make sure that we convert the value back to string because it may be a number return { label: key, op: operator, value }; @@ -244,6 +302,35 @@ function addParser(query: string, queryPartPositions: Position[], parser: string return newQuery; } +/** + * Add filter as label filter after the parsers + * @param query + * @param logQueryPositions + * @param labelFormat + */ +function addLabelFormat( + query: string, + logQueryPositions: Position[], + labelFormat: { originalLabel: string; renameTo: string } +): string { + let newQuery = ''; + let prev = 0; + + for (let i = 0; i < logQueryPositions.length; i++) { + // This is basically just doing splice on a string for each matched vector selector. + const match = logQueryPositions[i]; + const isLast = i === logQueryPositions.length - 1; + + const start = query.substring(prev, match.to); + const end = isLast ? query.substring(match.to) : ''; + + const labelFilter = ` | label_format ${labelFormat.renameTo}=${labelFormat.originalLabel}`; + newQuery += start + labelFilter + end; + prev = match.to; + } + return newQuery; +} + /** * Check if label exists in the list of labels but ignore the operator. * @param labels diff --git a/public/app/plugins/datasource/loki/addtoQuery.test.ts b/public/app/plugins/datasource/loki/addtoQuery.test.ts index 81b47f00c33..a5201fbfb22 100644 --- a/public/app/plugins/datasource/loki/addtoQuery.test.ts +++ b/public/app/plugins/datasource/loki/addtoQuery.test.ts @@ -1,4 +1,4 @@ -import { addLabelToQuery, addNoPipelineErrorToQuery, addParserToQuery } from './addToQuery'; +import { addLabelFormatToQuery, addLabelToQuery, addNoPipelineErrorToQuery, addParserToQuery } from './addToQuery'; describe('addLabelToQuery()', () => { it('should add label to simple query', () => { @@ -193,3 +193,43 @@ describe('addNoPipelineErrorToQuery', () => { expect(addNoPipelineErrorToQuery('{job="grafana"} |="no parser"')).toBe('{job="grafana"} |="no parser"'); }); }); + +describe('addLabelFormatToQuery', () => { + it('should add label format at the end of log query when parser', () => { + expect(addLabelFormatToQuery('{job="grafana"} | logfmt', { originalLabel: 'lvl', renameTo: 'level' })).toBe( + '{job="grafana"} | logfmt | label_format level=lvl' + ); + }); + + it('should add label format at the end of log query when no parser', () => { + expect(addLabelFormatToQuery('{job="grafana"}', { originalLabel: 'lvl', renameTo: 'level' })).toBe( + '{job="grafana"} | label_format level=lvl' + ); + }); + + it('should add label format at the end of log query when more label parser', () => { + expect( + addLabelFormatToQuery('{job="grafana"} | logfmt | label_format a=b', { originalLabel: 'lvl', renameTo: 'level' }) + ).toBe('{job="grafana"} | logfmt | label_format a=b | label_format level=lvl'); + }); + + it('should add label format at the end of log query part of metrics query', () => { + expect( + addLabelFormatToQuery('rate({job="grafana"} | logfmt | label_format a=b [5m])', { + originalLabel: 'lvl', + renameTo: 'level', + }) + ).toBe('rate({job="grafana"} | logfmt | label_format a=b | label_format level=lvl [5m])'); + }); + + it('should add label format at the end of multiple log query part of metrics query', () => { + expect( + addLabelFormatToQuery( + 'rate({job="grafana"} | logfmt | label_format a=b [5m]) + rate({job="grafana"} | logfmt | label_format a=b [5m])', + { originalLabel: 'lvl', renameTo: 'level' } + ) + ).toBe( + 'rate({job="grafana"} | logfmt | label_format a=b | label_format level=lvl [5m]) + rate({job="grafana"} | logfmt | label_format a=b | label_format level=lvl [5m])' + ); + }); +}); diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index a30ce914eee..477eb02ed76 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -44,7 +44,7 @@ import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_sr import { serializeParams } from '../../../core/utils/fetch'; import { renderLegendFormat } from '../prometheus/legend'; -import { addLabelToQuery, addNoPipelineErrorToQuery, addParserToQuery } from './addToQuery'; +import { addLabelFormatToQuery, addLabelToQuery, addNoPipelineErrorToQuery, addParserToQuery } from './addToQuery'; import { transformBackendResult } from './backendResultTransformer'; import { LokiAnnotationsQueryEditor } from './components/AnnotationsQueryEditor'; import LanguageProvider from './language_provider'; @@ -358,7 +358,7 @@ export class LokiDatasource expr: query.expr, queryType: LokiQueryType.Range, refId: 'log-samples', - maxLines: 50, + maxLines: 10, }; // For samples, we use defaultTimeRange (now-6h/now) and limit od 10 lines so queries are small and fast @@ -417,6 +417,15 @@ export class LokiDatasource expression = addNoPipelineErrorToQuery(expression); break; } + case 'ADD_LEVEL_LABEL_FORMAT': { + if (action.options?.originalLabel && action.options?.renameTo) { + expression = addLabelFormatToQuery(expression, { + renameTo: action.options.renameTo, + originalLabel: action.options.originalLabel, + }); + } + break; + } default: break; } diff --git a/public/app/plugins/datasource/loki/queryHints.test.ts b/public/app/plugins/datasource/loki/queryHints.test.ts index d7165f315fc..9f01591a537 100644 --- a/public/app/plugins/datasource/loki/queryHints.test.ts +++ b/public/app/plugins/datasource/loki/queryHints.test.ts @@ -70,4 +70,45 @@ describe('getQueryHints', () => { expect(getQueryHints('{job="grafana" | json', [jsonAndLogfmtSeries])).toEqual([]); }); }); + + describe('when series with level-like label', () => { + const createSeriesWithLabel = (labelName?: string): DataFrame => { + const labelVariable: { [key: string]: string } = { job: 'a' }; + if (labelName) { + labelVariable[labelName] = 'error'; + } + return { + name: 'logs', + length: 2, + fields: [ + { + name: 'Line', + type: FieldType.string, + config: {}, + values: new ArrayVector(['{"foo": "bar", "bar": "baz"}', 'foo="bar" bar="baz"']), + }, + { + name: 'labels', + type: FieldType.other, + config: {}, + values: new ArrayVector([labelVariable, { job: 'baz', foo: 'bar' }]), + }, + ], + }; + }; + + it('suggest level renaming when no level label', () => { + expect(getQueryHints('{job="grafana"', [createSeriesWithLabel('lvl')])).toMatchObject([ + { type: 'ADD_JSON_PARSER' }, + { type: 'ADD_LOGFMT_PARSER' }, + { type: 'ADD_LEVEL_LABEL_FORMAT' }, + ]); + }); + it('does not suggest level renaming if level label', () => { + expect(getQueryHints('{job="grafana"', [createSeriesWithLabel('level')])).toMatchObject([ + { type: 'ADD_JSON_PARSER' }, + { type: 'ADD_LOGFMT_PARSER' }, + ]); + }); + }); }); diff --git a/public/app/plugins/datasource/loki/queryHints.ts b/public/app/plugins/datasource/loki/queryHints.ts index a31a5dfbdf4..fcad80179cb 100644 --- a/public/app/plugins/datasource/loki/queryHints.ts +++ b/public/app/plugins/datasource/loki/queryHints.ts @@ -1,7 +1,12 @@ import { DataFrame, QueryHint } from '@grafana/data'; -import { isQueryPipelineErrorFiltering, isQueryWithParser } from './query_utils'; -import { extractHasErrorLabelFromDataFrame, extractLogParserFromDataFrame } from './responseUtils'; +import { isQueryPipelineErrorFiltering, isQueryWithLabelFormat, isQueryWithParser } from './query_utils'; +import { + dataFrameHasLevelLabel, + extractHasErrorLabelFromDataFrame, + extractLevelLikeLabelFromDataFrame, + extractLogParserFromDataFrame, +} from './responseUtils'; export function getQueryHints(query: string, series: DataFrame[]): QueryHint[] { if (series.length === 0) { @@ -63,5 +68,30 @@ export function getQueryHints(query: string, series: DataFrame[]): QueryHint[] { } } + const queryWithLabelFormat = isQueryWithLabelFormat(query); + if (!queryWithLabelFormat) { + const hasLevel = dataFrameHasLevelLabel(series[0]); + const levelLikeLabel = extractLevelLikeLabelFromDataFrame(series[0]); + + // Add hint only if we don't have "level" label and have level-like label + if (!hasLevel && levelLikeLabel) { + hints.push({ + type: 'ADD_LEVEL_LABEL_FORMAT', + label: `Some logs in your selected log stream have "${levelLikeLabel}" label.`, + fix: { + label: `If ${levelLikeLabel} label has level values, consider using label_format to rename it to "level". Level label can be then visualized in log volumes.`, + action: { + type: 'ADD_LEVEL_LABEL_FORMAT', + query, + options: { + renameTo: 'level', + originalLabel: levelLikeLabel, + }, + }, + }, + }); + } + } + return hints; } diff --git a/public/app/plugins/datasource/loki/query_utils.test.ts b/public/app/plugins/datasource/loki/query_utils.test.ts index 870cc256f02..5e815467886 100644 --- a/public/app/plugins/datasource/loki/query_utils.test.ts +++ b/public/app/plugins/datasource/loki/query_utils.test.ts @@ -2,6 +2,7 @@ import { getHighlighterExpressionsFromQuery, getNormalizedLokiQuery, isLogsQuery, + isQueryWithLabelFormat, isQueryWithParser, isValidQuery, } from './query_utils'; @@ -187,3 +188,21 @@ describe('isQueryWithParser', () => { }); }); }); + +describe('isQueryWithLabelFormat', () => { + it('returns true if log query with label format', () => { + expect(isQueryWithLabelFormat('{job="grafana"} | label_format level=lvl')).toBe(true); + }); + + it('returns true if metrics query with label format', () => { + expect(isQueryWithLabelFormat('rate({job="grafana"} | label_format level=lvl [5m])')).toBe(true); + }); + + it('returns false if log query without label format', () => { + expect(isQueryWithLabelFormat('{job="grafana"} | json')).toBe(false); + }); + + it('returns false if metrics query without label format', () => { + expect(isQueryWithLabelFormat('rate({job="grafana"} [5m])')).toBe(false); + }); +}); diff --git a/public/app/plugins/datasource/loki/query_utils.ts b/public/app/plugins/datasource/loki/query_utils.ts index 3afa1bb8df2..e99efe6dfd8 100644 --- a/public/app/plugins/datasource/loki/query_utils.ts +++ b/public/app/plugins/datasource/loki/query_utils.ts @@ -159,3 +159,16 @@ export function isQueryPipelineErrorFiltering(query: string): boolean { return isQueryPipelineErrorFiltering; } + +export function isQueryWithLabelFormat(query: string): boolean { + let queryWithLabelFormat = false; + const tree = parser.parse(query); + tree.iterate({ + enter: (type): false | void => { + if (type.name === 'LabelFormatExpr') { + queryWithLabelFormat = true; + } + }, + }); + return queryWithLabelFormat; +} diff --git a/public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.test.ts b/public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.test.ts index eef0ce74dca..a7ef55e61e4 100644 --- a/public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.test.ts +++ b/public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.test.ts @@ -170,9 +170,9 @@ describe('LokiQueryModeller', () => { expect( modeller.renderQuery({ labels: [{ label: 'app', op: '=', value: 'grafana' }], - operations: [{ id: LokiOperationId.LabelFormat, params: ['new', 'old'] }], + operations: [{ id: LokiOperationId.LabelFormat, params: ['original', 'renameTo'] }], }) - ).toBe('{app="grafana"} | label_format old=`new`'); + ).toBe('{app="grafana"} | label_format renameTo=original'); }); it('Can render simply binary operation with scalar', () => { diff --git a/public/app/plugins/datasource/loki/querybuilder/operations.ts b/public/app/plugins/datasource/loki/querybuilder/operations.ts index 632c8c5884f..02bdd90dd94 100644 --- a/public/app/plugins/datasource/loki/querybuilder/operations.ts +++ b/public/app/plugins/datasource/loki/querybuilder/operations.ts @@ -187,13 +187,13 @@ export function getOperationDefinitions(): QueryBuilderOperationDef[] { name: 'Label format', params: [ { name: 'Label', type: 'string' }, - { name: 'Rename', type: 'string' }, + { name: 'Rename to', type: 'string' }, ], defaultParams: ['', ''], alternativesKey: 'format', category: LokiVisualQueryOperationCategory.Formats, orderRank: LokiOperationOrder.LineFormats, - renderer: (model, def, innerExpr) => `${innerExpr} | label_format ${model.params[1]}=\`${model.params[0]}\``, + renderer: (model, def, innerExpr) => `${innerExpr} | label_format ${model.params[1]}=${model.params[0]}`, addOperationHandler: addLokiOperation, explainHandler: () => `This will change name of label to desired new label. In the example below, label "error_level" will be renamed to "level". diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts index 0c458a533f2..3c074989ece 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts @@ -456,7 +456,7 @@ describe('buildVisualQueryFromString', () => { }); it('parses query with label format', () => { - expect(buildVisualQueryFromString('{app="frontend"} | label_format newLabel=oldLabel')).toEqual( + expect(buildVisualQueryFromString('{app="frontend"} | label_format renameTo=original')).toEqual( noErrors({ labels: [ { @@ -465,13 +465,13 @@ describe('buildVisualQueryFromString', () => { label: 'app', }, ], - operations: [{ id: 'label_format', params: ['newLabel', 'oldLabel'] }], + operations: [{ id: 'label_format', params: ['original', 'renameTo'] }], }) ); }); it('parses query with multiple label format', () => { - expect(buildVisualQueryFromString('{app="frontend"} | label_format newLabel=oldLabel, bar="baz"')).toEqual( + expect(buildVisualQueryFromString('{app="frontend"} | label_format renameTo=original, bar=baz')).toEqual( noErrors({ labels: [ { @@ -481,8 +481,8 @@ describe('buildVisualQueryFromString', () => { }, ], operations: [ - { id: 'label_format', params: ['newLabel', 'oldLabel'] }, - { id: 'label_format', params: ['bar', 'baz'] }, + { id: 'label_format', params: ['original', 'renameTo'] }, + { id: 'label_format', params: ['baz', 'bar'] }, ], }) ); diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.ts index 3f9aea6340e..9c0e87dc441 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.ts @@ -295,15 +295,13 @@ function getLineFormat(expr: string, node: SyntaxNode): QueryBuilderOperation { function getLabelFormat(expr: string, node: SyntaxNode): QueryBuilderOperation { const id = 'label_format'; - const identifier = node.getChild('Identifier'); - const op = identifier!.nextSibling; - const value = op!.nextSibling; - - let valueString = handleQuotes(getString(expr, value)); + const renameTo = node.getChild('Identifier'); + const op = renameTo!.nextSibling; + const originalLabel = op!.nextSibling; return { id, - params: [getString(expr, identifier), valueString], + params: [getString(expr, originalLabel), handleQuotes(getString(expr, renameTo))], }; } diff --git a/public/app/plugins/datasource/loki/responseUtils.test.ts b/public/app/plugins/datasource/loki/responseUtils.test.ts index 992c05ea221..c65663d7ea7 100644 --- a/public/app/plugins/datasource/loki/responseUtils.test.ts +++ b/public/app/plugins/datasource/loki/responseUtils.test.ts @@ -2,7 +2,7 @@ import { cloneDeep } from 'lodash'; import { ArrayVector, DataFrame, FieldType } from '@grafana/data'; -import { dataFrameHasLokiError } from './responseUtils'; +import { dataFrameHasLevelLabel, dataFrameHasLokiError, extractLevelLikeLabelFromDataFrame } from './responseUtils'; const frame: DataFrame = { length: 1, @@ -28,7 +28,7 @@ const frame: DataFrame = { ], }; -describe('dataframeHasParsingError', () => { +describe('dataFrameHasParsingError', () => { it('handles frame with parsing error', () => { const input = cloneDeep(frame); input.fields[1].values = new ArrayVector([{ level: 'info', __error__: 'error' }]); @@ -39,3 +39,34 @@ describe('dataframeHasParsingError', () => { expect(dataFrameHasLokiError(input)).toBe(false); }); }); + +describe('dataFrameHasLevelLabel', () => { + it('returns true if level label is present', () => { + const input = cloneDeep(frame); + input.fields[1].values = new ArrayVector([{ level: 'info' }]); + expect(dataFrameHasLevelLabel(input)).toBe(true); + }); + it('returns false if level label is present', () => { + const input = cloneDeep(frame); + input.fields[1].values = new ArrayVector([{ foo: 'bar' }]); + expect(dataFrameHasLevelLabel(input)).toBe(false); + }); +}); + +describe('extractLevelLikeLabelFromDataFrame', () => { + it('returns label if lvl label is present', () => { + const input = cloneDeep(frame); + input.fields[1].values = new ArrayVector([{ lvl: 'info' }]); + expect(extractLevelLikeLabelFromDataFrame(input)).toBe('lvl'); + }); + it('returns label if level-like label is present', () => { + const input = cloneDeep(frame); + input.fields[1].values = new ArrayVector([{ error_level: 'info' }]); + expect(extractLevelLikeLabelFromDataFrame(input)).toBe('error_level'); + }); + it('returns undefined if no level-like label is present', () => { + const input = cloneDeep(frame); + input.fields[1].values = new ArrayVector([{ foo: 'info' }]); + expect(extractLevelLikeLabelFromDataFrame(input)).toBe(null); + }); +}); diff --git a/public/app/plugins/datasource/loki/responseUtils.ts b/public/app/plugins/datasource/loki/responseUtils.ts index fc2b7b904e0..ba0cf026295 100644 --- a/public/app/plugins/datasource/loki/responseUtils.ts +++ b/public/app/plugins/datasource/loki/responseUtils.ts @@ -4,6 +4,12 @@ export function dataFrameHasLokiError(frame: DataFrame): boolean { const labelSets: Labels[] = frame.fields.find((f) => f.name === 'labels')?.values.toArray() ?? []; return labelSets.some((labels) => labels.__error__ !== undefined); } + +export function dataFrameHasLevelLabel(frame: DataFrame): boolean { + const labelSets: Labels[] = frame.fields.find((f) => f.name === 'labels')?.values.toArray() ?? []; + return labelSets.some((labels) => labels.level !== undefined); +} + export function extractLogParserFromDataFrame(frame: DataFrame): { hasLogfmt: boolean; hasJSON: boolean } { const lineField = frame.fields.find((field) => field.type === FieldType.string); if (lineField == null) { @@ -37,3 +43,25 @@ export function extractHasErrorLabelFromDataFrame(frame: DataFrame): boolean { const labels: Array<{ [key: string]: string }> = labelField.values.toArray(); return labels.some((label) => label['__error__']); } + +export function extractLevelLikeLabelFromDataFrame(frame: DataFrame): string | null { + const labelField = frame.fields.find((field) => field.name === 'labels' && field.type === FieldType.other); + if (labelField == null) { + return null; + } + + // Depending on number of labels, this can be pretty heavy operation. + // Let's just look at first 2 lines If needed, we can introduce more later. + const labelsArray: Array<{ [key: string]: string }> = labelField.values.toArray().slice(0, 2); + let levelLikeLabel: string | null = null; + + // Find first level-like label + for (let labels of labelsArray) { + const label = Object.keys(labels).find((label) => label === 'lvl' || label.includes('level')); + if (label) { + levelLikeLabel = label; + break; + } + } + return levelLikeLabel; +}