The open and composable observability and data visualization platform. Visualize metrics, logs, and traces from multiple sources like Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 
grafana/public/app/plugins/datasource/loki/queryUtils.test.ts

629 lines
23 KiB

import { String } from '@grafana/lezer-logql';
import {
getHighlighterExpressionsFromQuery,
getLokiQueryType,
isLogsQuery,
isQueryWithLabelFormat,
isQueryWithParser,
isQueryWithError,
parseToNodeNamesArray,
getParserFromQuery,
obfuscate,
requestSupportsSplitting,
isQueryWithRangeVariable,
isQueryPipelineErrorFiltering,
getLogQueryFromMetricsQuery,
getNormalizedLokiQuery,
getNodePositionsFromQuery,
getLogQueryFromMetricsQueryAtPosition,
interpolateShardingSelector,
requestSupportsSharding,
} from './queryUtils';
import { LokiQuery, LokiQueryDirection, LokiQueryType } from './types';
describe('getHighlighterExpressionsFromQuery', () => {
it('returns no expressions for empty query', () => {
expect(getHighlighterExpressionsFromQuery('')).toEqual([]);
});
it('returns no expression for query with empty filter ', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= ``')).toEqual([]);
});
it('returns no expression for query with empty filter and parser', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= `` | json count="counter" | __error__=``')).toEqual([]);
});
it('returns no expression for query with empty filter and chained filter', () => {
expect(
getHighlighterExpressionsFromQuery('{foo="bar"} |= `` |= `highlight` | json count="counter" | __error__=``')
).toEqual(['highlight']);
});
it('returns no expression for query with empty filter, chained and regex filter', () => {
expect(
getHighlighterExpressionsFromQuery(
'{foo="bar"} |= `` |= `highlight` |~ `high.ight` | json count="counter" | __error__=``'
)
).toEqual(['highlight', 'high.ight']);
});
it('returns no expression for query with empty filter, chained and regex quotes filter', () => {
expect(
getHighlighterExpressionsFromQuery(
'{foo="bar"} |= `` |= `highlight` |~ "highlight\\\\d" | json count="counter" | __error__=``'
)
).toEqual(['highlight', 'highlight\\d']);
});
it('returns an expression for query with filter using quotes', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x"')).toEqual(['x']);
});
it('returns an expression for query with filter using backticks', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= `x`')).toEqual(['x']);
});
it('returns expressions for query with filter chain', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" |~ "y"')).toEqual(['x', 'y']);
});
it('returns expressions for query with filter chain using both backticks and quotes', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" |~ `y`')).toEqual(['x', 'y']);
});
it('returns expression for query with log parser', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" | logfmt')).toEqual(['x']);
});
it('returns expressions for query with filter chain followed by log parser', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" |~ "y" | logfmt')).toEqual(['x', 'y']);
});
it('returns drops expressions for query with negative filter chain using quotes', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" != "y"')).toEqual(['x']);
});
it('returns expressions for query with filter chain using backticks', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= `x` |~ `y`')).toEqual(['x', 'y']);
});
it('returns expressions for query with filter chain using quotes and backticks', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" |~ `y`')).toEqual(['x', 'y']);
});
it('returns null if filter term is not wrapped in double quotes', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= x')).toEqual([]);
});
it('escapes filter term if regex filter operator is not used', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x[yz].w"')).toEqual(['x\\[yz\\]\\.w']);
});
it('does not escape filter term if regex filter operator is used', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |~ "x[yz].w" |~ "z.+"')).toEqual(['x[yz].w', 'z.+']);
});
it('removes extra backslash escaping if regex filter operator and quotes are used', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |~ "\\\\w+"')).toEqual(['\\w+']);
});
it('does not remove backslash escaping if regex filter operator and backticks are used', () => {
expect(getHighlighterExpressionsFromQuery('{foo="bar"} |~ `\\w+`')).toEqual(['\\w+']);
});
it.each`
input | expected
${'`"test"`'} | ${'"test"'}
${'"`test`"'} | ${'`test`'}
${'`"test"a`'} | ${'"test"a'}
`('should correctly identify the type of quote used in the term', ({ input, expected }) => {
expect(getHighlighterExpressionsFromQuery(`{foo="bar"} |= ${input}`)).toEqual([expected]);
});
it.each(['|=', '|~'])('returns multiple expressions when using or statements', (op: string) => {
expect(getHighlighterExpressionsFromQuery(`{app="frontend"} ${op} "line" or "text"`)).toEqual(['line', 'text']);
});
it.each(['|=', '|~'])('returns multiple expressions when using or statements and ip filters', (op: string) => {
expect(getHighlighterExpressionsFromQuery(`{app="frontend"} ${op} "line" or ip("10.0.0.1")`)).toEqual(['line']);
});
});
describe('getNormalizedLokiQuery', () => {
it('removes deprecated instant property', () => {
const input: LokiQuery = { refId: 'A', expr: 'test1', instant: true };
const output = getNormalizedLokiQuery(input);
expect(output).toStrictEqual({ refId: 'A', expr: 'test1', queryType: LokiQueryType.Range });
});
it('removes deprecated range property', () => {
const input: LokiQuery = { refId: 'A', expr: 'test1', range: true };
const output = getNormalizedLokiQuery(input);
expect(output).toStrictEqual({ refId: 'A', expr: 'test1', queryType: LokiQueryType.Range });
});
it('removes deprecated range and instant properties if query with queryType', () => {
const input: LokiQuery = { refId: 'A', expr: 'test1', range: true, instant: false, queryType: LokiQueryType.Range };
const output = getNormalizedLokiQuery(input);
expect(output).toStrictEqual({ refId: 'A', expr: 'test1', queryType: LokiQueryType.Range });
});
});
describe('getLokiQueryType', () => {
function expectCorrectQueryType(inputProps: Object, outputQueryType: LokiQueryType) {
const input: LokiQuery = { refId: 'A', expr: 'test1', ...inputProps };
const output = getLokiQueryType(input);
expect(output).toStrictEqual(outputQueryType);
}
it('handles no props case', () => {
expectCorrectQueryType({}, LokiQueryType.Range);
});
it('handles old-style instant case', () => {
expectCorrectQueryType({ instant: true, range: false }, LokiQueryType.Instant);
});
it('handles old-style range case', () => {
expectCorrectQueryType({ instant: false, range: true }, LokiQueryType.Range);
});
it('handles new+old style instant', () => {
expectCorrectQueryType({ instant: true, range: false, queryType: LokiQueryType.Range }, LokiQueryType.Range);
});
it('handles new+old style range', () => {
expectCorrectQueryType({ instant: false, range: true, queryType: LokiQueryType.Instant }, LokiQueryType.Instant);
});
it('handles new<>old conflict (new wins), range', () => {
expectCorrectQueryType({ instant: false, range: true, queryType: LokiQueryType.Range }, LokiQueryType.Range);
});
it('handles new<>old conflict (new wins), instant', () => {
expectCorrectQueryType({ instant: true, range: false, queryType: LokiQueryType.Instant }, LokiQueryType.Instant);
});
it('handles invalid new, range', () => {
expectCorrectQueryType({ queryType: 'invalid' }, LokiQueryType.Range);
});
it('handles invalid new, when old-range exists, use old', () => {
expectCorrectQueryType({ instant: false, range: true, queryType: 'invalid' }, LokiQueryType.Range);
});
it('handles invalid new, when old-instant exists, use old', () => {
expectCorrectQueryType({ instant: true, range: false, queryType: 'invalid' }, LokiQueryType.Instant);
});
});
describe('isQueryWithError', () => {
it('returns false if invalid query', () => {
expect(isQueryWithError('{job="grafana')).toBe(true);
});
it('returns true if valid query', () => {
expect(isQueryWithError('{job="grafana"}')).toBe(false);
});
});
describe('parseToNodeNamesArray', () => {
it('returns on empty query', () => {
expect(parseToNodeNamesArray('{}')).toEqual(['LogQL', 'Expr', 'LogExpr', 'Selector', '⚠']);
});
it('returns on invalid query', () => {
expect(parseToNodeNamesArray('{job="grafana"')).toEqual([
'LogQL',
'Expr',
'LogExpr',
'Selector',
'Matchers',
'Matcher',
'Identifier',
'Eq',
'String',
'⚠',
]);
});
it('returns on valid query', () => {
expect(parseToNodeNamesArray('{job="grafana"}')).toEqual([
'LogQL',
'Expr',
'LogExpr',
'Selector',
'Matchers',
'Matcher',
'Identifier',
'Eq',
'String',
]);
});
});
describe('obfuscate', () => {
it('obfuscates on invalid query', () => {
expect(obfuscate('{job="grafana"')).toEqual('{Identifier=String');
});
it('obfuscates on valid query', () => {
expect(
obfuscate('sum(sum_over_time({test="test"} |= `` | logfmt | __error__=`` | unwrap test | __error__=`` [10m]))')
).toEqual(
'sum(sum_over_time({Identifier=String} |= String | logfmt | __error__=String | unwrap Identifier | __error__=String [10m]))'
);
});
it('obfuscates on arithmetic operation', () => {
expect(obfuscate('2 + 3')).toEqual('Number + Number');
});
it('obfuscates a comment', () => {
expect(obfuscate('{job="grafana"} # test comment')).toEqual('{Identifier=String} LineComment');
});
it('does not obfuscate interval variables', () => {
expect(
obfuscate(
'sum(quantile_over_time(0.5, {label="$var"} | logfmt | __error__=`` | unwrap latency | __error__=`` [$__interval]))'
)
).toEqual(
'sum(quantile_over_time(Number, {Identifier=String} | logfmt | __error__=String | unwrap Identifier | __error__=String [$__interval]))'
);
});
});
describe('isLogsQuery', () => {
it('returns false if metrics query', () => {
expect(isLogsQuery('rate({job="grafana"}[5m])')).toBe(false);
});
it('returns true if valid query', () => {
expect(isLogsQuery('{job="grafana"}')).toBe(true);
});
});
describe('isQueryWithParser', () => {
it('returns false if query without parser', () => {
expect(isQueryWithParser('rate({job="grafana" |= "error" }[5m])')).toEqual({
parserCount: 0,
queryWithParser: false,
});
});
it('returns true if log query with parser', () => {
expect(isQueryWithParser('{job="grafana"} | json')).toEqual({ parserCount: 1, queryWithParser: true });
});
it('returns true if metric query with parser', () => {
expect(isQueryWithParser('rate({job="grafana"} | json [5m])')).toEqual({ parserCount: 1, queryWithParser: true });
});
it('returns true if query with json parser with expressions', () => {
expect(isQueryWithParser('rate({job="grafana"} | json foo="bar", bar="baz" [5m])')).toEqual({
parserCount: 1,
queryWithParser: true,
});
});
});
describe('isQueryWithLabelFormat', () => {
it('returns true if log query with label format', () => {
expect(isQueryWithLabelFormat('{job="grafana"} | label_format level=lvl')).toBe(true);
});
it('returns true if metrics query with label format', () => {
expect(isQueryWithLabelFormat('rate({job="grafana"} | label_format level=lvl [5m])')).toBe(true);
});
it('returns false if log query without label format', () => {
expect(isQueryWithLabelFormat('{job="grafana"} | json')).toBe(false);
});
it('returns false if metrics query without label format', () => {
expect(isQueryWithLabelFormat('rate({job="grafana"} [5m])')).toBe(false);
});
});
describe('isQueryWithRangeVariableDuration', () => {
it('identifies queries using $__range variable', () => {
expect(isQueryWithRangeVariable('rate({job="grafana"}[$__range])')).toBe(true);
});
it('identifies queries using $__range_s variable', () => {
expect(isQueryWithRangeVariable('rate({job="grafana"}[$__range_s])')).toBe(true);
});
it('identifies queries using $__range_ms variable', () => {
expect(isQueryWithRangeVariable('rate({job="grafana"}[$__range_ms])')).toBe(true);
});
it('does not return false positives', () => {
expect(isQueryWithRangeVariable('rate({job="grafana"} | logfmt | value="$__range" [5m])')).toBe(false);
expect(isQueryWithRangeVariable('rate({job="grafana"} | logfmt | value="[$__range]" [5m])')).toBe(false);
expect(isQueryWithRangeVariable('rate({job="grafana"} [$range])')).toBe(false);
expect(isQueryWithRangeVariable('rate({job="grafana"} [$_range])')).toBe(false);
expect(isQueryWithRangeVariable('rate({job="grafana"} [$_range_ms])')).toBe(false);
});
});
describe('getParserFromQuery', () => {
it('returns no parser', () => {
expect(getParserFromQuery('{job="grafana"}')).toBeUndefined();
});
it.each(['json', 'logfmt', 'pattern', 'regexp', 'unpack'])('detects %s parser', (parser: string) => {
expect(getParserFromQuery(`{job="grafana"} | ${parser}`)).toBe(parser);
expect(getParserFromQuery(`sum(count_over_time({place="luna"} | ${parser} | unwrap counter )) by (place)`)).toBe(
parser
);
});
it('supports json parser with arguments', () => {
// Redundant, but gives us a baseline
expect(getParserFromQuery('{job="grafana"} | json')).toBe('json');
expect(getParserFromQuery('{job="grafana"} | json field="otherField"')).toBe('json');
expect(getParserFromQuery('{job="grafana"} | json field="otherField", label="field2"')).toBe('json');
});
it('supports logfmt parser with arguments and flags', () => {
// Redundant, but gives us a baseline
expect(getParserFromQuery('{job="grafana"} | logfmt')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt --strict')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt --strict --keep-empty')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt field="otherField"')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt field="otherField", label')).toBe('logfmt');
expect(getParserFromQuery('{job="grafana"} | logfmt --strict field="otherField"')).toBe('logfmt');
expect(
getParserFromQuery('{job="grafana"} | logfmt --strict --keep-empty field="otherField", label="field2"')
).toBe('logfmt');
});
});
describe('requestSupportsSplitting', () => {
it('hidden requests are not partitioned', () => {
const requests: LokiQuery[] = [
{
expr: '{a="b"}',
refId: 'A',
hide: true,
},
];
expect(requestSupportsSplitting(requests)).toBe(false);
});
it('special requests are not partitioned', () => {
const requests: LokiQuery[] = [
{
expr: '{a="b"}',
refId: 'do-not-chunk',
},
];
expect(requestSupportsSplitting(requests)).toBe(false);
});
it('empty requests are not partitioned', () => {
const requests: LokiQuery[] = [
{
expr: '',
refId: 'A',
},
];
expect(requestSupportsSplitting(requests)).toBe(false);
});
it('all other requests are partitioned', () => {
const requests: LokiQuery[] = [
{
expr: '{a="b"}',
refId: 'A',
},
{
expr: 'count_over_time({a="b"}[1h])',
refId: 'B',
},
];
expect(requestSupportsSplitting(requests)).toBe(true);
});
});
describe('isQueryPipelineErrorFiltering', () => {
it('identifies pipeline error filters', () => {
expect(isQueryPipelineErrorFiltering('{job="grafana"} | logfmt | __error__=""')).toBe(true);
expect(isQueryPipelineErrorFiltering('{job="grafana"} | logfmt | error=""')).toBe(false);
});
});
describe('getLogQueryFromMetricsQuery', () => {
it('returns the log query from a metric query', () => {
expect(getLogQueryFromMetricsQuery('count_over_time({job="grafana"} | logfmt | label="value" [1m])')).toBe(
'{job="grafana"} | logfmt | label="value"'
);
expect(getLogQueryFromMetricsQuery('count_over_time({job="grafana"} [1m])')).toBe('{job="grafana"}');
expect(
getLogQueryFromMetricsQuery(
'sum(quantile_over_time(0.5, {label="$var"} | logfmt | __error__=`` | unwrap latency | __error__=`` [$__interval]))'
)
).toBe('{label="$var"} | logfmt | __error__=``');
});
it('does not return a query when there is no log query', () => {
expect(getLogQueryFromMetricsQuery('1+1')).toBe('');
expect(getLogQueryFromMetricsQuery('count_over_time([1s])')).toBe('');
});
});
describe('getLogQueryFromMetricsQueryAtPosition', () => {
it('works like getLogQueryFromMetricsQuery for simple queries', () => {
expect(
getLogQueryFromMetricsQueryAtPosition('count_over_time({job="grafana"} | logfmt | label="value" [1m])', 57)
).toBe('{job="grafana"} | logfmt | label="value"');
expect(getLogQueryFromMetricsQueryAtPosition('count_over_time({job="grafana"} [1m])', 37)).toBe('{job="grafana"}');
expect(
getLogQueryFromMetricsQueryAtPosition(
'sum(quantile_over_time(0.5, {label="$var"} | logfmt | __error__=`` | unwrap latency | __error__=`` [$__interval]))',
45
)
).toBe('{label="$var"} | logfmt | __error__=``');
});
it.each([
[
'count_over_time({place="moon"} | json test="test" [1m]) + avg_over_time({place="luna"} | logfmt test="test" [1m])',
'{place="moon"} | json test="test"',
49,
],
[
'count_over_time({place="moon"} | json test="test" [1m]) + avg_over_time({place="luna"} | logfmt test="test" [1m])',
'{place="luna"} | logfmt test="test"',
107,
],
])('gets the right query for complex queries', (metric: string, log: string, position: number) => {
expect(getLogQueryFromMetricsQueryAtPosition(metric, position)).toBe(log);
});
});
describe('getNodePositionsFromQuery', () => {
it('returns the right amount of positions without type', () => {
// LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
expect(getNodePositionsFromQuery('{job="grafana"}').length).toBe(9);
});
it('returns the right position of a string in a stream selector', () => {
// LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
const nodePositions = getNodePositionsFromQuery('{job="grafana"}', [String]);
expect(nodePositions.length).toBe(1);
expect(nodePositions[0].from).toBe(5);
expect(nodePositions[0].to).toBe(14);
});
it('returns an empty array with a wrong expr', () => {
// LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
const nodePositions = getNodePositionsFromQuery('not loql', [String]);
expect(nodePositions.length).toBe(0);
});
});
describe('interpolateShardingSelector', () => {
let queries: LokiQuery[] = [];
beforeEach(() => {
queries = [
{
refId: 'A',
expr: '{job="grafana"}',
},
];
});
it('returns the original query when there are no shards to interpolate', () => {
expect(interpolateShardingSelector(queries, [])).toEqual(queries);
});
it('adds the empty shard to the query when -1 is passed', () => {
expect(interpolateShardingSelector(queries, [-1])).toEqual([
{
refId: 'A',
expr: '{job="grafana", __stream_shard__=""}',
},
]);
});
it('uses an equality filter when a single shard is passed', () => {
expect(interpolateShardingSelector(queries, [13])).toEqual([
{
refId: 'A',
expr: '{job="grafana", __stream_shard__="13"}',
},
]);
});
it('supports multiple shard values', () => {
expect(interpolateShardingSelector(queries, [1, 13, 667])).toEqual([
{
refId: 'A',
expr: '{job="grafana", __stream_shard__=~"1|13|667"}',
},
]);
});
describe('For metric queries', () => {
let queries: LokiQuery[] = [];
beforeEach(() => {
queries = [
{
refId: 'A',
expr: 'count_over_time({job="grafana"} [5m])',
},
];
});
it('returns the original query when there are no shards to interpolate', () => {
expect(interpolateShardingSelector(queries, [])).toEqual(queries);
});
it('adds the empty shard to the query when -1 is passed', () => {
expect(interpolateShardingSelector(queries, [-1])).toEqual([
{
refId: 'A',
expr: 'count_over_time({job="grafana", __stream_shard__=""} | drop __stream_shard__ [5m])',
},
]);
});
it('uses an equality filter when a single shard is passed', () => {
expect(interpolateShardingSelector(queries, [13])).toEqual([
{
refId: 'A',
expr: 'count_over_time({job="grafana", __stream_shard__="13"} | drop __stream_shard__ [5m])',
},
]);
});
it('supports multiple shard values', () => {
expect(interpolateShardingSelector(queries, [1, 13, 667])).toEqual([
{
refId: 'A',
expr: 'count_over_time({job="grafana", __stream_shard__=~"1|13|667"} | drop __stream_shard__ [5m])',
},
]);
});
it('supports multiple metric queries values', () => {
const queries = [
{
refId: 'A',
expr: 'count_over_time({job="grafana"} [5m]) + count_over_time({job="grafana"} [5m])',
},
];
expect(interpolateShardingSelector(queries, [1, 13, 667])).toEqual([
{
refId: 'A',
expr: 'count_over_time({job="grafana", __stream_shard__=~"1|13|667"} | drop __stream_shard__ [5m]) + count_over_time({job="grafana", __stream_shard__=~"1|13|667"} | drop __stream_shard__ [5m])',
},
]);
});
});
});
describe('requestSupportsSharding', () => {
it('supports log queries with Scan direction', () => {
expect(requestSupportsSharding([{ refId: 'A', expr: '{place="luna"}', direction: LokiQueryDirection.Scan }])).toBe(
true
);
});
it('declines log queries without Scan direction', () => {
expect(
requestSupportsSharding([{ refId: 'A', expr: '{place="luna"}', direction: LokiQueryDirection.Backward }])
).toBe(false);
});
it.each([
'count_over_time({place="luna"}[1m])',
'sum_over_time({place="luna"}[1m])',
'sum by (level) (count_over_time({place="luna"}[1m]))',
'sum by (level) (rate({place="luna"}[1m]))',
'sum(sum by (level) (avg_over_time({place="luna"}[1m])))',
'sum(rate({place="luna"}[1m]))',
])('allows supported metric queries', (expr: string) => {
expect(requestSupportsSharding([{ refId: 'A', expr }])).toBe(true);
});
it.each([
'avg_over_time({place="luna"}[1m])',
'avg(sum_over_time({place="luna"}[1m]))',
'avg(rate({place="luna"}[1m]))',
'count_over_time({place="luna"}[1m]) / count_over_time({place="luna"}[1m])',
'avg(sum by (level) (avg_over_time({place="luna"}[1m])))',
'sum(rate({place="luna"}[1m])) / sum(rate({place="luna"}[1m]))',
])('declines supported metric queries', (expr: string) => {
expect(requestSupportsSharding([{ refId: 'A', expr }])).toBe(false);
});
});