diff --git a/public/app/plugins/datasource/loki/LanguageProvider.test.ts b/public/app/plugins/datasource/loki/LanguageProvider.test.ts index 39d845e45d8..a8bd0b9ed97 100644 --- a/public/app/plugins/datasource/loki/LanguageProvider.test.ts +++ b/public/app/plugins/datasource/loki/LanguageProvider.test.ts @@ -1,9 +1,6 @@ -import Plain from 'slate-plain-serializer'; - import { AbstractLabelOperator, DataFrame } from '@grafana/data'; -import { TypeaheadInput } from '@grafana/ui'; -import LanguageProvider, { LokiHistoryItem } from './LanguageProvider'; +import LanguageProvider from './LanguageProvider'; import { LokiDatasource } from './datasource'; import { createLokiDatasource, createMetadataRequest } from './mocks'; import { @@ -28,73 +25,6 @@ jest.mock('app/store/store', () => ({ })); describe('Language completion provider', () => { - const datasource = setup({}); - - describe('query suggestions', () => { - it('returns no suggestions on empty context', async () => { - const instance = new LanguageProvider(datasource); - const value = Plain.deserialize(''); - const result = await instance.provideCompletionItems({ text: '', prefix: '', value, wrapperClasses: [] }); - expect(result.context).toBeUndefined(); - - expect(result.suggestions.length).toEqual(0); - }); - - it('returns history on empty context when history was provided', async () => { - const instance = new LanguageProvider(datasource); - const value = Plain.deserialize(''); - const history: LokiHistoryItem[] = [ - { - query: { refId: '1', expr: '{app="foo"}' }, - ts: 1, - }, - ]; - const result = await instance.provideCompletionItems( - { text: '', prefix: '', value, wrapperClasses: [] }, - { history } - ); - expect(result.context).toBeUndefined(); - - expect(result.suggestions).toMatchObject([ - { - label: 'History', - items: [ - { - label: '{app="foo"}', - }, - ], - }, - ]); - }); - - it('returns function and history suggestions', async () => { - const instance = new LanguageProvider(datasource); - const input = createTypeaheadInput('m', 'm', undefined, 1, [], instance); - // Historic expressions don't have to match input, filtering is done in field - const history: LokiHistoryItem[] = [ - { - query: { refId: '1', expr: '{app="foo"}' }, - ts: 1, - }, - ]; - const result = await instance.provideCompletionItems(input, { history }); - expect(result.context).toBeUndefined(); - expect(result.suggestions.length).toEqual(2); - expect(result.suggestions[0].label).toEqual('History'); - expect(result.suggestions[1].label).toEqual('Functions'); - }); - - it('returns pipe operations on pipe context', async () => { - const instance = new LanguageProvider(datasource); - const input = createTypeaheadInput('{app="test"} | ', ' ', '', 15, ['context-pipe']); - const result = await instance.provideCompletionItems(input); - expect(result.context).toBeUndefined(); - expect(result.suggestions.length).toEqual(2); - expect(result.suggestions[0].label).toEqual('Operators'); - expect(result.suggestions[1].label).toEqual('Parsers'); - }); - }); - describe('fetchSeries', () => { it('should use match[] parameter', () => { const datasource = setup({}, { '{foo="bar"}': [{ label1: 'label_val1' }] }); @@ -131,102 +61,6 @@ describe('Language completion provider', () => { }); }); - describe('label key suggestions', () => { - it('returns all label suggestions on empty selector', async () => { - const datasource = setup({ label1: [], label2: [] }); - const provider = await getLanguageProvider(datasource); - const input = createTypeaheadInput('{}', '', '', 1); - const result = await provider.provideCompletionItems(input); - expect(result.context).toBe('context-labels'); - expect(result.suggestions).toEqual([ - { - items: [ - { label: 'label1', filterText: '"label1"' }, - { label: 'label2', filterText: '"label2"' }, - ], - label: 'Labels', - }, - ]); - }); - - it('returns all label suggestions on selector when starting to type', async () => { - const datasource = setup({ label1: [], label2: [] }); - const provider = await getLanguageProvider(datasource); - const input = createTypeaheadInput('{l}', '', '', 2); - const result = await provider.provideCompletionItems(input); - expect(result.context).toBe('context-labels'); - expect(result.suggestions).toEqual([ - { - items: [ - { label: 'label1', filterText: '"label1"' }, - { label: 'label2', filterText: '"label2"' }, - ], - label: 'Labels', - }, - ]); - }); - }); - - describe('label suggestions facetted', () => { - it('returns facetted label suggestions based on selector', async () => { - const datasource = setup({ label1: [], label2: [] }, { '{foo="bar"}': [{ label1: 'label_val1' }] }); - const provider = await getLanguageProvider(datasource); - const input = createTypeaheadInput('{foo="bar",}', '', '', 11); - const result = await provider.provideCompletionItems(input); - expect(result.context).toBe('context-labels'); - expect(result.suggestions).toEqual([{ items: [{ label: 'label1' }], label: 'Labels' }]); - }); - - it('returns facetted label suggestions for multipule selectors', async () => { - const datasource = setup({ label1: [], label2: [] }, { '{baz="42",foo="bar"}': [{ label2: 'label_val2' }] }); - const provider = await getLanguageProvider(datasource); - const input = createTypeaheadInput('{baz="42",foo="bar",}', '', '', 20); - const result = await provider.provideCompletionItems(input); - expect(result.context).toBe('context-labels'); - expect(result.suggestions).toEqual([{ items: [{ label: 'label2' }], label: 'Labels' }]); - }); - }); - - describe('label suggestions', () => { - it('returns label values suggestions from Loki', async () => { - const datasource = setup({ label1: ['label1_val1', 'label1_val2'], label2: [] }); - const provider = await getLanguageProvider(datasource); - const input = createTypeaheadInput('{label1=}', '=', 'label1'); - let result = await provider.provideCompletionItems(input); - - result = await provider.provideCompletionItems(input); - expect(result.context).toBe('context-label-values'); - expect(result.suggestions).toEqual([ - { - items: [ - { label: 'label1_val1', filterText: '"label1_val1"' }, - { label: 'label1_val2', filterText: '"label1_val2"' }, - ], - label: 'Label values for "label1"', - }, - ]); - }); - it('returns label values suggestions from Loki when re-editing', async () => { - const datasource = setup({ label1: ['label1_val1', 'label1_val2'], label2: [] }); - const provider = await getLanguageProvider(datasource); - const input = createTypeaheadInput('{label1="label1_v"}', 'label1_v', 'label1', 17, [ - 'attr-value', - 'context-labels', - ]); - let result = await provider.provideCompletionItems(input); - expect(result.context).toBe('context-label-values'); - expect(result.suggestions).toEqual([ - { - items: [ - { label: 'label1_val1', filterText: '"label1_val1"' }, - { label: 'label1_val2', filterText: '"label1_val2"' }, - ], - label: 'Label values for "label1"', - }, - ]); - }); - }); - describe('label values', () => { it('should fetch label values if not cached', async () => { const datasource = setup({ testkey: ['label1_val1', 'label1_val2'], label2: [] }); @@ -450,31 +284,6 @@ async function getLanguageProvider(datasource: LokiDatasource) { return instance; } -/** - * @param value Value of the full input - * @param text Last piece of text (not sure but in case of {label=} this would be just '=') - * @param labelKey Label by which to search for values. Cutting corners a bit here as this should be inferred from value - */ -function createTypeaheadInput( - value: string, - text: string, - labelKey?: string, - anchorOffset?: number, - wrapperClasses?: string[], - instance?: LanguageProvider -): TypeaheadInput { - const deserialized = Plain.deserialize(value); - const range = deserialized.selection.setAnchor(deserialized.selection.anchor.setOffset(anchorOffset || 1)); - const valueWithSelection = deserialized.setSelection(range); - return { - text, - prefix: instance ? instance.cleanText(text) : '', - wrapperClasses: wrapperClasses || ['context-labels'], - value: valueWithSelection, - labelKey, - }; -} - function setup( labelsAndValues: Record, series?: Record>> diff --git a/public/app/plugins/datasource/loki/LanguageProvider.ts b/public/app/plugins/datasource/loki/LanguageProvider.ts index 9121fe11b9e..f2753668ca9 100644 --- a/public/app/plugins/datasource/loki/LanguageProvider.ts +++ b/public/app/plugins/datasource/loki/LanguageProvider.ts @@ -1,15 +1,8 @@ -import { chain, difference } from 'lodash'; import { LRUCache } from 'lru-cache'; -import Prism, { Grammar } from 'prismjs'; +import Prism from 'prismjs'; -import { dateTime, AbsoluteTimeRange, LanguageProvider, HistoryItem, AbstractQuery, KeyValue } from '@grafana/data'; -import { CompletionItem, TypeaheadInput, TypeaheadOutput, CompletionItemGroup } from '@grafana/ui'; -import { - extractLabelMatchers, - parseSelector, - processLabels, - toPromLikeExpr, -} from 'app/plugins/datasource/prometheus/language_utils'; +import { LanguageProvider, AbstractQuery, KeyValue } from '@grafana/data'; +import { extractLabelMatchers, processLabels, toPromLikeExpr } from 'app/plugins/datasource/prometheus/language_utils'; import { LokiDatasource } from './datasource'; import { @@ -17,61 +10,15 @@ import { extractLogParserFromDataFrame, extractUnwrapLabelKeysFromDataFrame, } from './responseUtils'; -import syntax, { FUNCTIONS, PIPE_PARSERS, PIPE_OPERATORS } from './syntax'; +import syntax from './syntax'; import { ParserAndLabelKeysResult, LokiQuery, LokiQueryType } from './types'; -const DEFAULT_KEYS = ['job', 'namespace']; -const EMPTY_SELECTOR = '{}'; -const HISTORY_ITEM_COUNT = 10; -const HISTORY_COUNT_CUTOFF = 1000 * 60 * 60 * 24; // 24h const NS_IN_MS = 1000000; -// When changing RATE_RANGES, check if Prometheus/PromQL ranges should be changed too -// @see public/app/plugins/datasource/prometheus/promql.ts -const RATE_RANGES: CompletionItem[] = [ - { label: '$__auto', sortValue: '$__auto' }, - { label: '1m', sortValue: '00:01:00' }, - { label: '5m', sortValue: '00:05:00' }, - { label: '10m', sortValue: '00:10:00' }, - { label: '30m', sortValue: '00:30:00' }, - { label: '1h', sortValue: '01:00:00' }, - { label: '1d', sortValue: '24:00:00' }, -]; - -export const LABEL_REFRESH_INTERVAL = 1000 * 30; // 30sec - -const wrapLabel = (label: string) => ({ label, filterText: `\"${label}\"` }); - -export type LokiHistoryItem = HistoryItem; - -type TypeaheadContext = { - history?: LokiHistoryItem[]; - absoluteRange?: AbsoluteTimeRange; -}; - -export function addHistoryMetadata(item: CompletionItem, history: LokiHistoryItem[]): CompletionItem { - const cutoffTs = Date.now() - HISTORY_COUNT_CUTOFF; - const historyForItem = history.filter((h) => h.ts > cutoffTs && h.query.expr === item.label); - let hint = `Queried ${historyForItem.length} times in the last 24h.`; - const recent = historyForItem[0]; - - if (recent) { - const lastQueried = dateTime(recent.ts).fromNow(); - hint = `${hint} Last queried ${lastQueried}.`; - } - - return { - ...item, - documentation: hint, - }; -} - export default class LokiLanguageProvider extends LanguageProvider { labelKeys: string[]; - labelFetchTs: number; started = false; datasource: LokiDatasource; - lookupsDisabled = false; // Dynamically set to true for big/slow instances /** * Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does @@ -86,18 +33,10 @@ export default class LokiLanguageProvider extends LanguageProvider { this.datasource = datasource; this.labelKeys = []; - this.labelFetchTs = 0; Object.assign(this, initialValues); } - // Strip syntax chars - cleanText = (s: string) => s.replace(/[{}[\]="(),!~+\-*/^%\|]/g, '').trim(); - - getSyntax(): Grammar { - return syntax; - } - request = async (url: string, params?: any): Promise => { try { return await this.datasource.metadataRequest(url, params); @@ -109,8 +48,7 @@ export default class LokiLanguageProvider extends LanguageProvider { }; /** - * Initialise the language provider by fetching set of labels. Without this initialisation the provider would return - * just a set of hardcoded default labels on provideCompletionItems or a recent queries from history. + * Initialize the language provider by fetching set of labels. */ start = () => { if (!this.startTask) { @@ -123,212 +61,19 @@ export default class LokiLanguageProvider extends LanguageProvider { return this.startTask; }; - getLabelKeys(): string[] { - return this.labelKeys; - } - /** - * Return suggestions based on input that can be then plugged into a typeahead dropdown. - * Keep this DOM-free for testing - * @param input - * @param context Is optional in types but is required in case we are doing getLabelCompletionItems - * @param context.absoluteRange Required in case we are doing getLabelCompletionItems - * @param context.history Optional used only in getEmptyCompletionItems + * Returns the label keys that have been fetched. + * If labels have not been fetched yet, it will return an empty array. + * For updated labels (which should not happen often), use fetchLabels. + * @todo It is quite complicated to know when to use fetchLabels and when to use getLabelKeys. + * We should consider simplifying this and use caching in the same way as with seriesCache and labelsCache + * and just always use fetchLabels. + * Caching should be thought out properly, so we are not fetching this often, as labelKeys should not be changing often. + * + * @returns {string[]} An array of label keys or an empty array if labels have not been fetched. */ - async provideCompletionItems(input: TypeaheadInput, context?: TypeaheadContext): Promise { - const { wrapperClasses, value, prefix, text } = input; - const emptyResult: TypeaheadOutput = { suggestions: [] }; - - if (!value) { - return emptyResult; - } - - // Local text properties - const empty = value?.document.text.length === 0; - const selectedLines = value.document.getTextsAtRange(value.selection); - const currentLine = selectedLines.size === 1 ? selectedLines.first().getText() : null; - - const nextCharacter = currentLine ? currentLine[value.selection.anchor.offset] : null; - - // Syntax spans have 3 classes by default. More indicate a recognized token - const tokenRecognized = wrapperClasses.length > 3; - - // Non-empty prefix, but not inside known token - const prefixUnrecognized = prefix && !tokenRecognized; - - // Prevent suggestions in `function(|suffix)` - const noSuffix = !nextCharacter || nextCharacter === ')'; - - // Prefix is safe if it does not immediately follow a complete expression and has no text after it - const safePrefix = prefix && !text.match(/^['"~=\]})\s]+$/) && noSuffix; - - // About to type next operand if preceded by binary operator - const operatorsPattern = /[+\-*/^%]/; - const isNextOperand = text.match(operatorsPattern); - - // Determine candidates by CSS context - if (wrapperClasses.includes('context-range')) { - // Suggestions for metric[|] - return this.getRangeCompletionItems(); - } else if (wrapperClasses.includes('context-labels')) { - // Suggestions for {|} and {foo=|} - return await this.getLabelCompletionItems(input); - } else if (wrapperClasses.includes('context-pipe')) { - return this.getPipeCompletionItem(); - } else if (empty) { - // Suggestions for empty query field - return this.getEmptyCompletionItems(context); - } else if (prefixUnrecognized && noSuffix && !isNextOperand) { - // Show term suggestions in a couple of scenarios - return this.getBeginningCompletionItems(context); - } else if (prefixUnrecognized && safePrefix) { - // Show term suggestions in a couple of scenarios - return this.getTermCompletionItems(); - } - - return emptyResult; - } - - getBeginningCompletionItems = (context?: TypeaheadContext): TypeaheadOutput => { - return { - suggestions: [...this.getEmptyCompletionItems(context).suggestions, ...this.getTermCompletionItems().suggestions], - }; - }; - - getEmptyCompletionItems(context?: TypeaheadContext): TypeaheadOutput { - const history = context?.history; - const suggestions = []; - - if (history?.length) { - const historyItems = chain(history) - .map((h) => h.query.expr) - .filter() - .uniq() - .take(HISTORY_ITEM_COUNT) - .map(wrapLabel) - .map((item) => addHistoryMetadata(item, history)) - .value(); - - suggestions.push({ - prefixMatch: true, - skipSort: true, - label: 'History', - items: historyItems, - }); - } - - return { suggestions }; - } - - getTermCompletionItems = (): TypeaheadOutput => { - const suggestions = []; - - suggestions.push({ - prefixMatch: true, - label: 'Functions', - items: FUNCTIONS.map((suggestion) => ({ ...suggestion, kind: 'function' })), - }); - - return { suggestions }; - }; - - getPipeCompletionItem = (): TypeaheadOutput => { - const suggestions = []; - - suggestions.push({ - label: 'Operators', - items: PIPE_OPERATORS.map((suggestion) => ({ ...suggestion, kind: 'operators' })), - }); - - suggestions.push({ - label: 'Parsers', - items: PIPE_PARSERS.map((suggestion) => ({ ...suggestion, kind: 'parsers' })), - }); - - return { suggestions }; - }; - - getRangeCompletionItems(): TypeaheadOutput { - return { - context: 'context-range', - suggestions: [ - { - label: 'Range vector', - items: [...RATE_RANGES], - }, - ], - }; - } - - async getLabelCompletionItems({ text, wrapperClasses, labelKey, value }: TypeaheadInput): Promise { - let context = 'context-labels'; - const suggestions: CompletionItemGroup[] = []; - if (!value) { - return { context, suggestions: [] }; - } - const line = value.anchorBlock.getText(); - const cursorOffset = value.selection.anchor.offset; - const isValueStart = text.match(/^(=|=~|!=|!~)/); - - // Get normalized selector - let selector; - let parsedSelector; - try { - parsedSelector = parseSelector(line, cursorOffset); - selector = parsedSelector.selector; - } catch { - selector = EMPTY_SELECTOR; - } - - if (!labelKey && selector === EMPTY_SELECTOR) { - // start task gets all labels - await this.start(); - const allLabels = this.getLabelKeys(); - return { context, suggestions: [{ label: `Labels`, items: allLabels.map(wrapLabel) }] }; - } - - const existingKeys = parsedSelector ? parsedSelector.labelKeys : []; - - let labelValues; - // Query labels for selector - if (selector) { - if (selector === EMPTY_SELECTOR && labelKey) { - const labelValuesForKey = await this.getLabelValues(labelKey); - labelValues = { [labelKey]: labelValuesForKey }; - } else { - labelValues = await this.getSeriesLabels(selector); - } - } - - if (!labelValues) { - console.warn(`Server did not return any values for selector = ${selector}`); - return { context, suggestions }; - } - - if ((text && isValueStart) || wrapperClasses.includes('attr-value')) { - // Label values - if (labelKey && labelValues[labelKey]) { - context = 'context-label-values'; - suggestions.push({ - label: `Label values for "${labelKey}"`, - // Filter to prevent previously selected values from being repeatedly suggested - items: labelValues[labelKey].map(wrapLabel).filter(({ filterText }) => filterText !== text), - }); - } - } else { - // Label keys - const labelKeys = labelValues ? Object.keys(labelValues) : DEFAULT_KEYS; - if (labelKeys) { - const possibleKeys = difference(labelKeys, existingKeys); - if (possibleKeys.length) { - const newItems = possibleKeys.map((key) => ({ label: key })); - const newSuggestion: CompletionItemGroup = { label: `Labels`, items: newItems }; - suggestions.push(newSuggestion); - } - } - } - - return { context, suggestions }; + getLabelKeys(): string[] { + return this.labelKeys; } importFromAbstractQuery(labelBasedQuery: AbstractQuery): LokiQuery { @@ -351,10 +96,11 @@ export default class LokiLanguageProvider extends LanguageProvider { }; } + /** + * Wrapper method over fetchSeriesLabels to retrieve series labels and handle errors. + * @todo remove this in favor of fetchSeriesLabels as we already in this.request do the same thing + */ async getSeriesLabels(selector: string) { - if (this.lookupsDisabled) { - return undefined; - } try { return await this.fetchSeriesLabels(selector); } catch (error) { @@ -375,7 +121,6 @@ export default class LokiLanguageProvider extends LanguageProvider { async fetchLabels(): Promise { const url = 'labels'; const timeRange = this.datasource.getTimeRangeParams(); - this.labelFetchTs = Date.now().valueOf(); const res = await this.request(url, timeRange); if (Array.isArray(res)) { @@ -432,15 +177,18 @@ export default class LokiLanguageProvider extends LanguageProvider { // The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every // millisecond while still actually getting all the keys for the correct interval. This still can create problems // when user does not the newest values for a minute if already cached. - generateCacheKey(url: string, start: number, end: number, param: string): string { + private generateCacheKey(url: string, start: number, end: number, param: string): string { return [url, this.roundTime(start), this.roundTime(end), param].join(); } - // Round nanos epoch to nearest 5 minute interval - roundTime(nanos: number): number { - return nanos ? Math.floor(nanos / NS_IN_MS / 1000 / 60 / 5) : 0; + // Round nanoseconds epoch to nearest 5 minute interval + private roundTime(nanoseconds: number): number { + return nanoseconds ? Math.floor(nanoseconds / NS_IN_MS / 1000 / 60 / 5) : 0; } + /** + * @todo remove this in favor of fetchLabelValues as it is the same thing + */ async getLabelValues(key: string): Promise { return await this.fetchLabelValues(key); } diff --git a/public/app/plugins/datasource/loki/syntax.ts b/public/app/plugins/datasource/loki/syntax.ts index 9790da1ac19..ae939af997b 100644 --- a/public/app/plugins/datasource/loki/syntax.ts +++ b/public/app/plugins/datasource/loki/syntax.ts @@ -191,8 +191,8 @@ export const BUILT_IN_FUNCTIONS = [ ]; export const FUNCTIONS = [...AGGREGATION_OPERATORS, ...RANGE_VEC_FUNCTIONS, ...BUILT_IN_FUNCTIONS]; -export const LOKI_KEYWORDS = [...FUNCTIONS, ...PIPE_OPERATORS, ...PIPE_PARSERS].map((keyword) => keyword.label); +// Loki grammar is used for query highlight in query previews outside of code editor export const lokiGrammar: Grammar = { comment: { pattern: /#.*/,