From 2090534635f9fe4b14704a6c3da6d9f259fc65d4 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Tue, 12 Jul 2022 15:42:01 +0200 Subject: [PATCH] Loki: Support json parser with expressions in query builder (#51965) * Loki: Support json parser with expressions in query builder * Add explain docs for json parser --- package.json | 2 +- .../datasource/loki/query_utils.test.ts | 4 ++++ .../plugins/datasource/loki/query_utils.ts | 2 +- .../querybuilder/LokiQueryModeller.test.ts | 18 +++++++++++++++ .../loki/querybuilder/operations.ts | 17 ++++++++++++-- .../loki/querybuilder/parsing.test.ts | 22 +++++++++++-------- .../datasource/loki/querybuilder/parsing.ts | 18 ++++++++++----- yarn.lock | 10 ++++----- 8 files changed, 70 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index aad08ba9a43..de2d102b412 100644 --- a/package.json +++ b/package.json @@ -256,7 +256,7 @@ "@grafana/e2e-selectors": "workspace:*", "@grafana/experimental": "^0.0.2-canary.32", "@grafana/google-sdk": "0.0.3", - "@grafana/lezer-logql": "^0.0.13", + "@grafana/lezer-logql": "^0.0.14", "@grafana/runtime": "workspace:*", "@grafana/schema": "workspace:*", "@grafana/slate-react": "0.22.10-grafana", diff --git a/public/app/plugins/datasource/loki/query_utils.test.ts b/public/app/plugins/datasource/loki/query_utils.test.ts index 8668742b789..7d46779fd5e 100644 --- a/public/app/plugins/datasource/loki/query_utils.test.ts +++ b/public/app/plugins/datasource/loki/query_utils.test.ts @@ -146,4 +146,8 @@ describe('isQueryWithParser', () => { it('returns true if metric query with parser', () => { expect(isQueryWithParser('rate({job="grafana"} | json [5m])')).toBe(true); }); + + it('returns true if query with json parser with expressions', () => { + expect(isQueryWithParser('rate({job="grafana"} | json foo="bar", bar="baz" [5m])')).toBe(true); + }); }); diff --git a/public/app/plugins/datasource/loki/query_utils.ts b/public/app/plugins/datasource/loki/query_utils.ts index 5717053f391..c0972af4cb8 100644 --- a/public/app/plugins/datasource/loki/query_utils.ts +++ b/public/app/plugins/datasource/loki/query_utils.ts @@ -126,7 +126,7 @@ export function isQueryWithParser(query: string): boolean { const tree = parser.parse(query); tree.iterate({ enter: (type): false | void => { - if (type.name === 'LabelParser') { + if (type.name === 'LabelParser' || type.name === 'JsonExpression') { hasParser = true; } }, diff --git a/public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.test.ts b/public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.test.ts index 9135368d0cd..eef0ce74dca 100644 --- a/public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.test.ts +++ b/public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.test.ts @@ -22,6 +22,24 @@ describe('LokiQueryModeller', () => { ).toBe('{app="grafana"} | json'); }); + it('Can query with pipeline operation json and expression param', () => { + expect( + modeller.renderQuery({ + labels: [{ label: 'app', op: '=', value: 'grafana' }], + operations: [{ id: LokiOperationId.Json, params: ['foo="bar"'] }], + }) + ).toBe('{app="grafana"} | json foo="bar"'); + }); + + it('Can query with pipeline operation json and multiple expression params', () => { + expect( + modeller.renderQuery({ + labels: [{ label: 'app', op: '=', value: 'grafana' }], + operations: [{ id: LokiOperationId.Json, params: ['foo="bar", bar="baz"'] }], + }) + ).toBe('{app="grafana"} | json foo="bar", bar="baz"'); + }); + it('Can query with pipeline operation logfmt', () => { expect( modeller.renderQuery({ diff --git a/public/app/plugins/datasource/loki/querybuilder/operations.ts b/public/app/plugins/datasource/loki/querybuilder/operations.ts index e2483642b61..632c8c5884f 100644 --- a/public/app/plugins/datasource/loki/querybuilder/operations.ts +++ b/public/app/plugins/datasource/loki/querybuilder/operations.ts @@ -64,13 +64,26 @@ export function getOperationDefinitions(): QueryBuilderOperationDef[] { { id: LokiOperationId.Json, name: 'Json', - params: [], + params: [ + { + name: 'Expression', + type: 'string', + restParam: true, + optional: true, + minWidth: 18, + placeholder: 'server="servers[0]"', + description: + 'Using expressions with your json parser will extract only the specified json fields to labels. You can specify one or more expressions in this way. All expressions must be quoted.', + }, + ], defaultParams: [], alternativesKey: 'format', category: LokiVisualQueryOperationCategory.Formats, orderRank: LokiOperationOrder.LineFormats, - renderer: pipelineRenderer, + renderer: (model, def, innerExpr) => `${innerExpr} | json ${model.params.join(', ')}`.trim(), addOperationHandler: addLokiOperation, + explainHandler: () => + `This will extract keys and values from a [json](https://grafana.com/docs/loki/latest/logql/log_queries/#json) formatted log line as labels. The extracted labels can be used in label filter expressions and used as values for a range aggregation via the unwrap operation.`, }, { id: LokiOperationId.Logfmt, diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts index d12dcaa5d41..0c458a533f2 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts @@ -187,16 +187,20 @@ describe('buildVisualQueryFromString', () => { ); }); - it('returns error for query with JSON expression parser', () => { + it('parses query with JSON parser with expression', () => { const context = buildVisualQueryFromString('{app="frontend"} | json label="value" '); - expect(context.errors).toEqual([ - { - text: 'JsonExpressionParser not supported in visual query builder: json label="value"', - from: 19, - to: 37, - parentType: 'PipelineStage', - }, - ]); + expect(context.query).toEqual({ + labels: [{ label: 'app', op: '=', value: 'frontend' }], + operations: [{ id: 'json', params: ['label="value"'] }], + }); + }); + + it('parses query with JSON parser with multiple expressions', () => { + const context = buildVisualQueryFromString('{app="frontend"} | json label="value", bar="baz", foo="bar" '); + expect(context.query).toEqual({ + labels: [{ label: 'app', op: '=', value: 'frontend' }], + operations: [{ id: 'json', params: ['label="value"', 'bar="baz"', 'foo="bar"'] }], + }); }); it('parses query with with simple unwrap', () => { diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.ts index 469dcb4ccb7..3f9aea6340e 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.ts @@ -103,12 +103,9 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex } break; } - case 'JsonExpressionParser': { - // JsonExpressionParser is not supported in query builder - const error = 'JsonExpressionParser not supported in visual query builder'; - - context.errors.push(createNotSupportedError(expr, node, error)); + visQuery.operations.push(getJsonExpressionParser(expr, node)); + break; } case 'LineFormatExpr': { @@ -222,6 +219,17 @@ function getLabelParser(expr: string, node: SyntaxNode): QueryBuilderOperation { }; } +function getJsonExpressionParser(expr: string, node: SyntaxNode): QueryBuilderOperation { + const parserNode = node.getChild('Json'); + const parser = getString(expr, parserNode); + + const params = [...getAllByType(expr, node, 'JsonExpression')]; + return { + id: parser, + params, + }; +} + function getLabelFilter(expr: string, node: SyntaxNode): { operation?: QueryBuilderOperation; error?: string } { // Check for nodes not supported in visual builder and return error if (node.getChild('Or') || node.getChild('And') || node.getChild('Comma')) { diff --git a/yarn.lock b/yarn.lock index 835f747e93a..5027648df0d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4820,14 +4820,14 @@ __metadata: languageName: node linkType: hard -"@grafana/lezer-logql@npm:^0.0.13": - version: 0.0.13 - resolution: "@grafana/lezer-logql@npm:0.0.13" +"@grafana/lezer-logql@npm:^0.0.14": + version: 0.0.14 + resolution: "@grafana/lezer-logql@npm:0.0.14" dependencies: lezer: ^0.13.5 peerDependencies: "@lezer/lr": ^0.15.8 - checksum: 417ab91c4fa8317789435bafbadd0bc24a6b9d8494a87614f14dc8cbe5f05bf94f529eadc0b2643ff71a3a3fee45cbf588538d8224abe2f4134ff5cc2c2b0055 + checksum: 4e35f455b6b7c286dfca9f3770df0d4c325057352dc8241665b667d798db09de54e1c14f7bfd34d4ae5bbef782d28994ceaf5c517fc18cff46ae3a47527e1990 languageName: node linkType: hard @@ -20983,7 +20983,7 @@ __metadata: "@grafana/eslint-config": 4.0.0 "@grafana/experimental": ^0.0.2-canary.32 "@grafana/google-sdk": 0.0.3 - "@grafana/lezer-logql": ^0.0.13 + "@grafana/lezer-logql": ^0.0.14 "@grafana/runtime": "workspace:*" "@grafana/schema": "workspace:*" "@grafana/slate-react": 0.22.10-grafana