apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Loki: Clean up Language provider and remove unused code (#77315)

Browse Source 
* Loki: Clean up Language provider and remove unused stuff

* Update

* Update comment
pull/76748/head
Ivana Huckova 2 years ago committed by GitHub
parent f8912517e6
commit 23c0054ecc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 29 additions and 472 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 193
      public/app/plugins/datasource/loki/LanguageProvider.test.ts
  2. 306
      public/app/plugins/datasource/loki/LanguageProvider.ts
  3. 2
      public/app/plugins/datasource/loki/syntax.ts

193
public/app/plugins/datasource/loki/LanguageProvider.test.ts
Unescape View File

@ -1,9 +1,6 @@
import Plain from 'slate-plain-serializer';
import { AbstractLabelOperator, DataFrame } from '@grafana/data';
import { TypeaheadInput } from '@grafana/ui';
import LanguageProvider, { LokiHistoryItem } from './LanguageProvider';
import LanguageProvider from './LanguageProvider';
import { LokiDatasource } from './datasource';
import { createLokiDatasource, createMetadataRequest } from './mocks';
import {
@ -28,73 +25,6 @@ jest.mock('app/store/store', () => ({
}));
describe('Language completion provider', () => {
const datasource = setup({});
describe('query suggestions', () => {
it('returns no suggestions on empty context', async () => {
const instance = new LanguageProvider(datasource);
const value = Plain.deserialize('');
const result = await instance.provideCompletionItems({ text: '', prefix: '', value, wrapperClasses: [] });
expect(result.context).toBeUndefined();
expect(result.suggestions.length).toEqual(0);
});
it('returns history on empty context when history was provided', async () => {
const instance = new LanguageProvider(datasource);
const value = Plain.deserialize('');
const history: LokiHistoryItem[] = [
{
query: { refId: '1', expr: '{app="foo"}' },
ts: 1,
},
];
const result = await instance.provideCompletionItems(
{ text: '', prefix: '', value, wrapperClasses: [] },
{ history }
);
expect(result.context).toBeUndefined();
expect(result.suggestions).toMatchObject([
{
label: 'History',
items: [
{
label: '{app="foo"}',
},
],
},
]);
});
it('returns function and history suggestions', async () => {
const instance = new LanguageProvider(datasource);
const input = createTypeaheadInput('m', 'm', undefined, 1, [], instance);
// Historic expressions don't have to match input, filtering is done in field
const history: LokiHistoryItem[] = [
{
query: { refId: '1', expr: '{app="foo"}' },
ts: 1,
},
];
const result = await instance.provideCompletionItems(input, { history });
expect(result.context).toBeUndefined();
expect(result.suggestions.length).toEqual(2);
expect(result.suggestions[0].label).toEqual('History');
expect(result.suggestions[1].label).toEqual('Functions');
});
it('returns pipe operations on pipe context', async () => {
const instance = new LanguageProvider(datasource);
const input = createTypeaheadInput('{app="test"} | ', ' ', '', 15, ['context-pipe']);
const result = await instance.provideCompletionItems(input);
expect(result.context).toBeUndefined();
expect(result.suggestions.length).toEqual(2);
expect(result.suggestions[0].label).toEqual('Operators');
expect(result.suggestions[1].label).toEqual('Parsers');
});
});
describe('fetchSeries', () => {
it('should use match[] parameter', () => {
const datasource = setup({}, { '{foo="bar"}': [{ label1: 'label_val1' }] });
@ -131,102 +61,6 @@ describe('Language completion provider', () => {
});
});
describe('label key suggestions', () => {
it('returns all label suggestions on empty selector', async () => {
const datasource = setup({ label1: [], label2: [] });
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{}', '', '', 1);
const result = await provider.provideCompletionItems(input);
expect(result.context).toBe('context-labels');
expect(result.suggestions).toEqual([
{
items: [
{ label: 'label1', filterText: '"label1"' },
{ label: 'label2', filterText: '"label2"' },
],
label: 'Labels',
},
]);
});
it('returns all label suggestions on selector when starting to type', async () => {
const datasource = setup({ label1: [], label2: [] });
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{l}', '', '', 2);
const result = await provider.provideCompletionItems(input);
expect(result.context).toBe('context-labels');
expect(result.suggestions).toEqual([
{
items: [
{ label: 'label1', filterText: '"label1"' },
{ label: 'label2', filterText: '"label2"' },
],
label: 'Labels',
},
]);
});
});
describe('label suggestions facetted', () => {
it('returns facetted label suggestions based on selector', async () => {
const datasource = setup({ label1: [], label2: [] }, { '{foo="bar"}': [{ label1: 'label_val1' }] });
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{foo="bar",}', '', '', 11);
const result = await provider.provideCompletionItems(input);
expect(result.context).toBe('context-labels');
expect(result.suggestions).toEqual([{ items: [{ label: 'label1' }], label: 'Labels' }]);
});
it('returns facetted label suggestions for multipule selectors', async () => {
const datasource = setup({ label1: [], label2: [] }, { '{baz="42",foo="bar"}': [{ label2: 'label_val2' }] });
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{baz="42",foo="bar",}', '', '', 20);
const result = await provider.provideCompletionItems(input);
expect(result.context).toBe('context-labels');
expect(result.suggestions).toEqual([{ items: [{ label: 'label2' }], label: 'Labels' }]);
});
});
describe('label suggestions', () => {
it('returns label values suggestions from Loki', async () => {
const datasource = setup({ label1: ['label1_val1', 'label1_val2'], label2: [] });
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{label1=}', '=', 'label1');
let result = await provider.provideCompletionItems(input);
result = await provider.provideCompletionItems(input);
expect(result.context).toBe('context-label-values');
expect(result.suggestions).toEqual([
{
items: [
{ label: 'label1_val1', filterText: '"label1_val1"' },
{ label: 'label1_val2', filterText: '"label1_val2"' },
],
label: 'Label values for "label1"',
},
]);
});
it('returns label values suggestions from Loki when re-editing', async () => {
const datasource = setup({ label1: ['label1_val1', 'label1_val2'], label2: [] });
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{label1="label1_v"}', 'label1_v', 'label1', 17, [
'attr-value',
'context-labels',
]);
let result = await provider.provideCompletionItems(input);
expect(result.context).toBe('context-label-values');
expect(result.suggestions).toEqual([
{
items: [
{ label: 'label1_val1', filterText: '"label1_val1"' },
{ label: 'label1_val2', filterText: '"label1_val2"' },
],
label: 'Label values for "label1"',
},
]);
});
});
describe('label values', () => {
it('should fetch label values if not cached', async () => {
const datasource = setup({ testkey: ['label1_val1', 'label1_val2'], label2: [] });
@ -450,31 +284,6 @@ async function getLanguageProvider(datasource: LokiDatasource) {
return instance;
}
/**
* @param value Value of the full input
* @param text Last piece of text (not sure but in case of {label=} this would be just '=')
* @param labelKey Label by which to search for values. Cutting corners a bit here as this should be inferred from value
*/
function createTypeaheadInput(
value: string,
text: string,
labelKey?: string,
anchorOffset?: number,
wrapperClasses?: string[],
instance?: LanguageProvider
): TypeaheadInput {
const deserialized = Plain.deserialize(value);
const range = deserialized.selection.setAnchor(deserialized.selection.anchor.setOffset(anchorOffset || 1));
const valueWithSelection = deserialized.setSelection(range);
return {
text,
prefix: instance ? instance.cleanText(text) : '',
wrapperClasses: wrapperClasses || ['context-labels'],
value: valueWithSelection,
labelKey,
};
}
function setup(
labelsAndValues: Record<string, string[]>,
series?: Record<string, Array<Record<string, string>>>

306
public/app/plugins/datasource/loki/LanguageProvider.ts
Unescape View File

@ -1,15 +1,8 @@
import { chain, difference } from 'lodash';
import { LRUCache } from 'lru-cache';
import Prism, { Grammar } from 'prismjs';
import Prism from 'prismjs';
import { dateTime, AbsoluteTimeRange, LanguageProvider, HistoryItem, AbstractQuery, KeyValue } from '@grafana/data';
import { CompletionItem, TypeaheadInput, TypeaheadOutput, CompletionItemGroup } from '@grafana/ui';
import {
extractLabelMatchers,
parseSelector,
processLabels,
toPromLikeExpr,
} from 'app/plugins/datasource/prometheus/language_utils';
import { LanguageProvider, AbstractQuery, KeyValue } from '@grafana/data';
import { extractLabelMatchers, processLabels, toPromLikeExpr } from 'app/plugins/datasource/prometheus/language_utils';
import { LokiDatasource } from './datasource';
import {
@ -17,61 +10,15 @@ import {
extractLogParserFromDataFrame,
extractUnwrapLabelKeysFromDataFrame,
} from './responseUtils';
import syntax, { FUNCTIONS, PIPE_PARSERS, PIPE_OPERATORS } from './syntax';
import syntax from './syntax';
import { ParserAndLabelKeysResult, LokiQuery, LokiQueryType } from './types';
const DEFAULT_KEYS = ['job', 'namespace'];
const EMPTY_SELECTOR = '{}';
const HISTORY_ITEM_COUNT = 10;
const HISTORY_COUNT_CUTOFF = 1000 * 60 * 60 * 24; // 24h
const NS_IN_MS = 1000000;
// When changing RATE_RANGES, check if Prometheus/PromQL ranges should be changed too
// @see public/app/plugins/datasource/prometheus/promql.ts
const RATE_RANGES: CompletionItem[] = [
{ label: '$__auto', sortValue: '$__auto' },
{ label: '1m', sortValue: '00:01:00' },
{ label: '5m', sortValue: '00:05:00' },
{ label: '10m', sortValue: '00:10:00' },
{ label: '30m', sortValue: '00:30:00' },
{ label: '1h', sortValue: '01:00:00' },
{ label: '1d', sortValue: '24:00:00' },
];
export const LABEL_REFRESH_INTERVAL = 1000 * 30; // 30sec
const wrapLabel = (label: string) => ({ label, filterText: `\"${label}\"` });
export type LokiHistoryItem = HistoryItem<LokiQuery>;
type TypeaheadContext = {
history?: LokiHistoryItem[];
absoluteRange?: AbsoluteTimeRange;
};
export function addHistoryMetadata(item: CompletionItem, history: LokiHistoryItem[]): CompletionItem {
const cutoffTs = Date.now() - HISTORY_COUNT_CUTOFF;
const historyForItem = history.filter((h) => h.ts > cutoffTs && h.query.expr === item.label);
let hint = `Queried ${historyForItem.length} times in the last 24h.`;
const recent = historyForItem[0];
if (recent) {
const lastQueried = dateTime(recent.ts).fromNow();
hint = `${hint} Last queried ${lastQueried}.`;
}
return {
...item,
documentation: hint,
};
}
export default class LokiLanguageProvider extends LanguageProvider {
labelKeys: string[];
labelFetchTs: number;
started = false;
datasource: LokiDatasource;
lookupsDisabled = false; // Dynamically set to true for big/slow instances
/**
* Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does
@ -86,18 +33,10 @@ export default class LokiLanguageProvider extends LanguageProvider {
this.datasource = datasource;
this.labelKeys = [];
this.labelFetchTs = 0;
Object.assign(this, initialValues);
}
// Strip syntax chars
cleanText = (s: string) => s.replace(/[{}[\]="(),!~+\-*/^%\|]/g, '').trim();
getSyntax(): Grammar {
return syntax;
}
request = async (url: string, params?: any): Promise<any> => {
try {
return await this.datasource.metadataRequest(url, params);
@ -109,8 +48,7 @@ export default class LokiLanguageProvider extends LanguageProvider {
};
/**
* Initialise the language provider by fetching set of labels. Without this initialisation the provider would return
* just a set of hardcoded default labels on provideCompletionItems or a recent queries from history.
* Initialize the language provider by fetching set of labels.
*/
start = () => {
if (!this.startTask) {
@ -123,212 +61,19 @@ export default class LokiLanguageProvider extends LanguageProvider {
return this.startTask;
};
getLabelKeys(): string[] {
return this.labelKeys;
}
/**
* Return suggestions based on input that can be then plugged into a typeahead dropdown.
* Keep this DOM-free for testing
* @param input
* @param context Is optional in types but is required in case we are doing getLabelCompletionItems
* @param context.absoluteRange Required in case we are doing getLabelCompletionItems
* @param context.history Optional used only in getEmptyCompletionItems
* Returns the label keys that have been fetched.
* If labels have not been fetched yet, it will return an empty array.
* For updated labels (which should not happen often), use fetchLabels.
* @todo It is quite complicated to know when to use fetchLabels and when to use getLabelKeys.
* We should consider simplifying this and use caching in the same way as with seriesCache and labelsCache
* and just always use fetchLabels.
* Caching should be thought out properly, so we are not fetching this often, as labelKeys should not be changing often.
*
* @returns {string[]} An array of label keys or an empty array if labels have not been fetched.
*/
async provideCompletionItems(input: TypeaheadInput, context?: TypeaheadContext): Promise<TypeaheadOutput> {
const { wrapperClasses, value, prefix, text } = input;
const emptyResult: TypeaheadOutput = { suggestions: [] };
if (!value) {
return emptyResult;
}
// Local text properties
const empty = value?.document.text.length === 0;
const selectedLines = value.document.getTextsAtRange(value.selection);
const currentLine = selectedLines.size === 1 ? selectedLines.first().getText() : null;
const nextCharacter = currentLine ? currentLine[value.selection.anchor.offset] : null;
// Syntax spans have 3 classes by default. More indicate a recognized token
const tokenRecognized = wrapperClasses.length > 3;
// Non-empty prefix, but not inside known token
const prefixUnrecognized = prefix && !tokenRecognized;
// Prevent suggestions in `function(|suffix)`
const noSuffix = !nextCharacter || nextCharacter === ')';
// Prefix is safe if it does not immediately follow a complete expression and has no text after it
const safePrefix = prefix && !text.match(/^['"~=\]})\s]+$/) && noSuffix;
// About to type next operand if preceded by binary operator
const operatorsPattern = /[+\-*/^%]/;
const isNextOperand = text.match(operatorsPattern);
// Determine candidates by CSS context
if (wrapperClasses.includes('context-range')) {
// Suggestions for metric[|]
return this.getRangeCompletionItems();
} else if (wrapperClasses.includes('context-labels')) {
// Suggestions for {|} and {foo=|}
return await this.getLabelCompletionItems(input);
} else if (wrapperClasses.includes('context-pipe')) {
return this.getPipeCompletionItem();
} else if (empty) {
// Suggestions for empty query field
return this.getEmptyCompletionItems(context);
} else if (prefixUnrecognized && noSuffix && !isNextOperand) {
// Show term suggestions in a couple of scenarios
return this.getBeginningCompletionItems(context);
} else if (prefixUnrecognized && safePrefix) {
// Show term suggestions in a couple of scenarios
return this.getTermCompletionItems();
}
return emptyResult;
}
getBeginningCompletionItems = (context?: TypeaheadContext): TypeaheadOutput => {
return {
suggestions: [...this.getEmptyCompletionItems(context).suggestions, ...this.getTermCompletionItems().suggestions],
};
};
getEmptyCompletionItems(context?: TypeaheadContext): TypeaheadOutput {
const history = context?.history;
const suggestions = [];
if (history?.length) {
const historyItems = chain(history)
.map((h) => h.query.expr)
.filter()
.uniq()
.take(HISTORY_ITEM_COUNT)
.map(wrapLabel)
.map((item) => addHistoryMetadata(item, history))
.value();
suggestions.push({
prefixMatch: true,
skipSort: true,
label: 'History',
items: historyItems,
});
}
return { suggestions };
}
getTermCompletionItems = (): TypeaheadOutput => {
const suggestions = [];
suggestions.push({
prefixMatch: true,
label: 'Functions',
items: FUNCTIONS.map((suggestion) => ({ ...suggestion, kind: 'function' })),
});
return { suggestions };
};
getPipeCompletionItem = (): TypeaheadOutput => {
const suggestions = [];
suggestions.push({
label: 'Operators',
items: PIPE_OPERATORS.map((suggestion) => ({ ...suggestion, kind: 'operators' })),
});
suggestions.push({
label: 'Parsers',
items: PIPE_PARSERS.map((suggestion) => ({ ...suggestion, kind: 'parsers' })),
});
return { suggestions };
};
getRangeCompletionItems(): TypeaheadOutput {
return {
context: 'context-range',
suggestions: [
{
label: 'Range vector',
items: [...RATE_RANGES],
},
],
};
}
async getLabelCompletionItems({ text, wrapperClasses, labelKey, value }: TypeaheadInput): Promise<TypeaheadOutput> {
let context = 'context-labels';
const suggestions: CompletionItemGroup[] = [];
if (!value) {
return { context, suggestions: [] };
}
const line = value.anchorBlock.getText();
const cursorOffset = value.selection.anchor.offset;
const isValueStart = text.match(/^(=|=~|!=|!~)/);
// Get normalized selector
let selector;
let parsedSelector;
try {
parsedSelector = parseSelector(line, cursorOffset);
selector = parsedSelector.selector;
} catch {
selector = EMPTY_SELECTOR;
}
if (!labelKey && selector === EMPTY_SELECTOR) {
// start task gets all labels
await this.start();
const allLabels = this.getLabelKeys();
return { context, suggestions: [{ label: `Labels`, items: allLabels.map(wrapLabel) }] };
}
const existingKeys = parsedSelector ? parsedSelector.labelKeys : [];
let labelValues;
// Query labels for selector
if (selector) {
if (selector === EMPTY_SELECTOR && labelKey) {
const labelValuesForKey = await this.getLabelValues(labelKey);
labelValues = { [labelKey]: labelValuesForKey };
} else {
labelValues = await this.getSeriesLabels(selector);
}
}
if (!labelValues) {
console.warn(`Server did not return any values for selector = ${selector}`);
return { context, suggestions };
}
if ((text && isValueStart) || wrapperClasses.includes('attr-value')) {
// Label values
if (labelKey && labelValues[labelKey]) {
context = 'context-label-values';
suggestions.push({
label: `Label values for "${labelKey}"`,
// Filter to prevent previously selected values from being repeatedly suggested
items: labelValues[labelKey].map(wrapLabel).filter(({ filterText }) => filterText !== text),
});
}
} else {
// Label keys
const labelKeys = labelValues ? Object.keys(labelValues) : DEFAULT_KEYS;
if (labelKeys) {
const possibleKeys = difference(labelKeys, existingKeys);
if (possibleKeys.length) {
const newItems = possibleKeys.map((key) => ({ label: key }));
const newSuggestion: CompletionItemGroup = { label: `Labels`, items: newItems };
suggestions.push(newSuggestion);
}
}
}
return { context, suggestions };
getLabelKeys(): string[] {
return this.labelKeys;
}
importFromAbstractQuery(labelBasedQuery: AbstractQuery): LokiQuery {
@ -351,10 +96,11 @@ export default class LokiLanguageProvider extends LanguageProvider {
};
}
/**
* Wrapper method over fetchSeriesLabels to retrieve series labels and handle errors.
* @todo remove this in favor of fetchSeriesLabels as we already in this.request do the same thing
*/
async getSeriesLabels(selector: string) {
if (this.lookupsDisabled) {
return undefined;
}
try {
return await this.fetchSeriesLabels(selector);
} catch (error) {
@ -375,7 +121,6 @@ export default class LokiLanguageProvider extends LanguageProvider {
async fetchLabels(): Promise<string[]> {
const url = 'labels';
const timeRange = this.datasource.getTimeRangeParams();
this.labelFetchTs = Date.now().valueOf();
const res = await this.request(url, timeRange);
if (Array.isArray(res)) {
@ -432,15 +177,18 @@ export default class LokiLanguageProvider extends LanguageProvider {
// The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every
// millisecond while still actually getting all the keys for the correct interval. This still can create problems
// when user does not the newest values for a minute if already cached.
generateCacheKey(url: string, start: number, end: number, param: string): string {
private generateCacheKey(url: string, start: number, end: number, param: string): string {
return [url, this.roundTime(start), this.roundTime(end), param].join();
}
// Round nanos epoch to nearest 5 minute interval
roundTime(nanos: number): number {
return nanos ? Math.floor(nanos / NS_IN_MS / 1000 / 60 / 5) : 0;
// Round nanoseconds epoch to nearest 5 minute interval
private roundTime(nanoseconds: number): number {
return nanoseconds ? Math.floor(nanoseconds / NS_IN_MS / 1000 / 60 / 5) : 0;
}
/**
* @todo remove this in favor of fetchLabelValues as it is the same thing
*/
async getLabelValues(key: string): Promise<string[]> {
return await this.fetchLabelValues(key);
}

2
public/app/plugins/datasource/loki/syntax.ts
Unescape View File

@ -191,8 +191,8 @@ export const BUILT_IN_FUNCTIONS = [
];
export const FUNCTIONS = [...AGGREGATION_OPERATORS, ...RANGE_VEC_FUNCTIONS, ...BUILT_IN_FUNCTIONS];
export const LOKI_KEYWORDS = [...FUNCTIONS, ...PIPE_OPERATORS, ...PIPE_PARSERS].map((keyword) => keyword.label);
// Loki grammar is used for query highlight in query previews outside of code editor
export const lokiGrammar: Grammar = {
comment: {
pattern: /#.*/,

Write Preview
Loading…
Cancel
Save