The open and composable observability and data visualization platform. Visualize metrics, logs, and traces from multiple sources like Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
grafana/public/app/plugins/datasource/loki/LogContextProvider.ts

410 lines
14 KiB

import { isEmpty } from 'lodash';
import { catchError, lastValueFrom, of, switchMap } from 'rxjs';
import {
CoreApp,
DataFrame,
DataQueryError,
DataQueryResponse,
FieldCache,
FieldType,
LogRowModel,
TimeRange,
toUtc,
LogRowContextQueryDirection,
LogRowContextOptions,
dateTime,
ScopedVars,
} from '@grafana/data';
Loki Query Editor: Add support for new logfmt features (#74619) * Loki autocomplete: add IN_LOGFMT situation for log queries * Loki autocomplete: add IN_LOGFMT situation for metric queries * Loki autocomplete: improve handling of trailing pipes and spaces * Loki autocomplete: add logfmt arguments completion * Loki autocomplete: add flags support to IN_LOGFMT * Loki autocomplete: extend IN_LOGFMT situation with labels and flag * Loki autocomplete: return logQuery in IN_LOGFMT situation * Loki autocomplete: offer label completions when IN_LOGFMT * Query utils: update parser detection method * Validation: update test * Loki autocomplete: improve IN_LOGFMT detection when in metric query * Loki autocomplete: improve logfmt suggestions * Loki autocomplete: improve logfmt suggestions in different scenarios * Loki autocomplete situation: refactor resolvers to support multiple paths * Situation: add test case * Loki autocomplete: allow user to use 2 flags * Situation: change flag to flags * Remove console log * Validation: import test parser * Completions: better handling of trailing comma scenario * Upgrade lezer-logql * Revert temporary imports * Loki Query Builder: Add support for new logfmt features (#74858) * Query builder: add params to logfmt definition * Logfmt operation: add default params * Query builder: update deprecated JsonExpression * Operation utils: update logfmt renderer * Query builder: parse LogfmtParser * Query builder: parse LogfmtExpressionParser * Remove console log * Remove unused variable * Remove extra character from render * Update unit tests * Fix unit tests * Operations: remove restParams from logfmt booleans * Parsing: group cases * Formatting * Formatting * Update modifyQuery * LogContextProvider: update with parser changes * LogContextProvider: remove unnecessary type castings It takes more energy to write `as unknow as LokiQuery` than to write a refId. * Formatting * Situation: use charAt instead of substring with endsWith * Situation: explain logfmt suggestions * Logfmt: improve flag suggestions * Remove console log * Completions: update test
2 years ago
import { LabelParser, LabelFilter, LineFilters, PipelineStage, Logfmt, Json } from '@grafana/lezer-logql';
import { LokiContextUi } from './components/LokiContextUi';
import { LokiDatasource, makeRequest, REF_ID_STARTER_LOG_ROW_CONTEXT } from './datasource';
import { escapeLabelValueInExactSelector, getLabelTypeFromFrame } from './languageUtils';
import { addLabelToQuery, addParserToQuery } from './modifyQuery';
import {
getNodePositionsFromQuery,
getParserFromQuery,
getStreamSelectorsFromQuery,
isQueryWithParser,
} from './queryUtils';
import { sortDataFrameByTime, SortDirection } from './sortDataFrame';
import { ContextFilter, LabelType, LokiQuery, LokiQueryDirection, LokiQueryType } from './types';
export const LOKI_LOG_CONTEXT_PRESERVED_LABELS = 'lokiLogContextPreservedLabels';
export const SHOULD_INCLUDE_PIPELINE_OPERATIONS = 'lokiLogContextShouldIncludePipelineOperations';
export type PreservedLabels = {
removedLabels: string[];
selectedExtractedLabels: string[];
};
export class LogContextProvider {
datasource: LokiDatasource;
cachedContextFilters: ContextFilter[];
onContextClose: (() => void) | undefined;
constructor(datasource: LokiDatasource) {
this.datasource = datasource;
this.cachedContextFilters = [];
}
private async getQueryAndRange(
row: LogRowModel,
options?: LogRowContextOptions,
origQuery?: LokiQuery,
cacheFilters = true
) {
const direction = (options && options.direction) || LogRowContextQueryDirection.Backward;
const limit = (options && options.limit) || this.datasource.maxLines;
// If the user doesn't have any filters applied already, or if we don't want
// to use the cached filters, we need to reinitialize them.
if (this.cachedContextFilters.length === 0 || !cacheFilters) {
const filters = (
await this.getInitContextFilters(row, origQuery, {
from: dateTime(row.timeEpochMs),
to: dateTime(row.timeEpochMs),
raw: { from: dateTime(row.timeEpochMs), to: dateTime(row.timeEpochMs) },
})
).contextFilters.filter((filter) => filter.enabled);
this.cachedContextFilters = filters;
}
return await this.prepareLogRowContextQueryTarget(row, limit, direction, origQuery);
}
getLogRowContextQuery = async (
row: LogRowModel,
options?: LogRowContextOptions,
origQuery?: LokiQuery,
cacheFilters = true
): Promise<LokiQuery> => {
if (origQuery && options?.scopedVars) {
origQuery = this.datasource.applyTemplateVariables(origQuery, options?.scopedVars);
}
const { query } = await this.getQueryAndRange(row, options, origQuery, cacheFilters);
if (!cacheFilters) {
// If the caller doesn't want to cache the filters, we need to reset them.
this.cachedContextFilters = [];
}
return query;
};
getLogRowContext = async (
row: LogRowModel,
options?: LogRowContextOptions,
origQuery?: LokiQuery
): Promise<{ data: DataFrame[] }> => {
if (origQuery && options?.scopedVars) {
origQuery = this.datasource.applyTemplateVariables(origQuery, options?.scopedVars);
}
const direction = (options && options.direction) || LogRowContextQueryDirection.Backward;
const { query, range } = await this.getQueryAndRange(row, options, origQuery);
const processResults = (result: DataQueryResponse): DataQueryResponse => {
const frames: DataFrame[] = result.data;
Logs: Redesign and improve LogContext (#65939) * Logs: Add new LogRowContext types to grafana/data * use right type for `RowContextOptions` * add missing renames * add show context modal * no need to call * removed unused css * sort properties * rename * use correct * use * add tests for * wip * remove add/minus buttons * add tests * disable processing of context results in Loki * moved into table to align properly * remove imports * add highlighting of opened logline * improve scrolling behavior * correct style for the table * use correct query direction * fix text * use LoadingBar * use overflow auto * rename `onToggleContext` to `onOpenContext` * add missing import * mock scrollIntoView * update unused props * remove unused import * no need to process context dataframes * only show `LogRowContextModal` if `getRowContext` is defined * remove unused param * use `userEvent` rather `fireEvent` * change to `TimeZone` * directly use style classes * revert change to public_dashboard_service_mock.go * improved styling * add missing await in test * fix lint * fix lint * remove LogRow scrolling when context is opened * remove references to `scrollElement` * Update public/app/features/logs/components/log-context/LogRowContextModal.tsx Co-authored-by: Matias Chomicki <matyax@gmail.com> * fix lint * add comment explaining `onCloseContext` * add comment about debounced onClose * add comments and remove `showRowMenu` * scroll twice to correctly center the element * revert double scrolling * remove unnecessary `processDataFrame` * trigger drone --------- Co-authored-by: Matias Chomicki <matyax@gmail.com>
2 years ago
const processedFrames = frames.map((frame) => sortDataFrameByTime(frame, SortDirection.Descending));
return {
...result,
data: processedFrames,
};
};
// this can only be called from explore currently
const app = CoreApp.Explore;
return lastValueFrom(
this.datasource.query(makeRequest(query, range, app, `${REF_ID_STARTER_LOG_ROW_CONTEXT}${direction}`)).pipe(
catchError((err) => {
const error: DataQueryError = {
message: 'Error during context query. Please check JS console logs.',
status: err.status,
statusText: err.statusText,
};
throw error;
}),
switchMap((res) => of(processResults(res)))
)
);
};
async prepareLogRowContextQueryTarget(
row: LogRowModel,
limit: number,
direction: LogRowContextQueryDirection,
origQuery?: LokiQuery
): Promise<{ query: LokiQuery; range: TimeRange }> {
const expr = this.prepareExpression(this.cachedContextFilters, origQuery);
const contextTimeBuffer = 2 * 60 * 60 * 1000; // 2h buffer
const queryDirection =
direction === LogRowContextQueryDirection.Forward ? LokiQueryDirection.Forward : LokiQueryDirection.Backward;
const query: LokiQuery = {
expr,
queryType: LokiQueryType.Range,
// refId has to be:
// - always different (temporarily, will be fixed later)
// - not increase in size
// because it may be called many times from logs-context
refId: `${REF_ID_STARTER_LOG_ROW_CONTEXT}_${Math.random().toString()}`,
maxLines: limit,
direction: queryDirection,
datasource: { uid: this.datasource.uid, type: this.datasource.type },
};
const fieldCache = new FieldCache(row.dataFrame);
const tsField = fieldCache.getFirstFieldOfType(FieldType.time);
if (tsField === undefined) {
throw new Error('loki: data frame missing time-field, should never happen');
}
const tsValue = tsField.values[row.rowIndex];
const timestamp = toUtc(tsValue);
const range =
queryDirection === LokiQueryDirection.Forward
? {
// start param in Loki API is inclusive so we'll have to filter out the row that this request is based from
// and any other that were logged in the same ns but before the row. Right now these rows will be lost
// because the are before but came it he response that should return only rows after.
from: timestamp,
// convert to ns, we lose some precision here but it is not that important at the far points of the context
to: toUtc(row.timeEpochMs + contextTimeBuffer),
}
: {
// convert to ns, we lose some precision here but it is not that important at the far points of the context
from: toUtc(row.timeEpochMs - contextTimeBuffer),
to: timestamp,
};
return {
query,
range: {
from: range.from,
to: range.to,
raw: range,
},
};
}
getLogRowContextUi(
row: LogRowModel,
runContextQuery?: () => void,
origQuery?: LokiQuery,
scopedVars?: ScopedVars
): React.ReactNode {
if (origQuery && scopedVars) {
origQuery = this.datasource.applyTemplateVariables(origQuery, scopedVars);
}
const updateFilter = (contextFilters: ContextFilter[]) => {
this.cachedContextFilters = contextFilters;
if (runContextQuery) {
runContextQuery();
}
};
// we need to cache this function so that it doesn't get recreated on every render
this.onContextClose =
this.onContextClose ??
(() => {
this.cachedContextFilters = [];
});
return LokiContextUi({
row,
origQuery,
updateFilter,
onClose: this.onContextClose,
logContextProvider: this,
runContextQuery,
});
}
prepareExpression(contextFilters: ContextFilter[], query: LokiQuery | undefined): string {
let preparedExpression = this.processContextFiltersToExpr(contextFilters, query);
if (window.localStorage.getItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS) === 'true') {
preparedExpression = this.processPipelineStagesToExpr(preparedExpression, query);
}
return preparedExpression;
}
processContextFiltersToExpr = (contextFilters: ContextFilter[], query: LokiQuery | undefined): string => {
const labelFilters = contextFilters
.map((filter) => {
if (!filter.nonIndexed && filter.enabled) {
// escape backslashes in label as users can't escape them by themselves
return `${filter.label}="${escapeLabelValueInExactSelector(filter.value)}"`;
}
return '';
})
// Filter empty strings
.filter((label) => !!label)
.join(',');
let expr = `{${labelFilters}}`;
// We need to have original query to get parser and include parsed labels
// We only add parser and parsed labels if there is only one parser in query
if (query) {
let hasParser = false;
if (isQueryWithParser(query.expr).parserCount === 1) {
hasParser = true;
const parser = getParserFromQuery(query.expr);
if (parser) {
expr = addParserToQuery(expr, parser);
}
}
const nonIndexedLabels = contextFilters.filter((filter) => filter.nonIndexed && filter.enabled);
for (const parsedLabel of nonIndexedLabels) {
if (parsedLabel.enabled) {
expr = addLabelToQuery(
expr,
parsedLabel.label,
'=',
parsedLabel.value,
hasParser ? LabelType.Parsed : LabelType.StructuredMetadata
);
}
}
}
return expr;
};
processPipelineStagesToExpr = (currentExpr: string, query: LokiQuery | undefined): string => {
let newExpr = currentExpr;
const origExpr = query?.expr ?? '';
if (isQueryWithParser(origExpr).parserCount > 1) {
return newExpr;
}
const allNodePositions = getNodePositionsFromQuery(origExpr, [
PipelineStage,
LabelParser,
Loki Query Editor: Add support for new logfmt features (#74619) * Loki autocomplete: add IN_LOGFMT situation for log queries * Loki autocomplete: add IN_LOGFMT situation for metric queries * Loki autocomplete: improve handling of trailing pipes and spaces * Loki autocomplete: add logfmt arguments completion * Loki autocomplete: add flags support to IN_LOGFMT * Loki autocomplete: extend IN_LOGFMT situation with labels and flag * Loki autocomplete: return logQuery in IN_LOGFMT situation * Loki autocomplete: offer label completions when IN_LOGFMT * Query utils: update parser detection method * Validation: update test * Loki autocomplete: improve IN_LOGFMT detection when in metric query * Loki autocomplete: improve logfmt suggestions * Loki autocomplete: improve logfmt suggestions in different scenarios * Loki autocomplete situation: refactor resolvers to support multiple paths * Situation: add test case * Loki autocomplete: allow user to use 2 flags * Situation: change flag to flags * Remove console log * Validation: import test parser * Completions: better handling of trailing comma scenario * Upgrade lezer-logql * Revert temporary imports * Loki Query Builder: Add support for new logfmt features (#74858) * Query builder: add params to logfmt definition * Logfmt operation: add default params * Query builder: update deprecated JsonExpression * Operation utils: update logfmt renderer * Query builder: parse LogfmtParser * Query builder: parse LogfmtExpressionParser * Remove console log * Remove unused variable * Remove extra character from render * Update unit tests * Fix unit tests * Operations: remove restParams from logfmt booleans * Parsing: group cases * Formatting * Formatting * Update modifyQuery * LogContextProvider: update with parser changes * LogContextProvider: remove unnecessary type castings It takes more energy to write `as unknow as LokiQuery` than to write a refId. * Formatting * Situation: use charAt instead of substring with endsWith * Situation: explain logfmt suggestions * Logfmt: improve flag suggestions * Remove console log * Completions: update test
2 years ago
Logfmt,
Json,
LineFilters,
LabelFilter,
]);
const pipelineStagePositions = allNodePositions.filter((position) => position.type?.id === PipelineStage);
const otherNodePositions = allNodePositions.filter((position) => position.type?.id !== PipelineStage);
for (const pipelineStagePosition of pipelineStagePositions) {
// we don't process pipeline stages that contain label parsers, line filters or label filters
if (otherNodePositions.some((position) => pipelineStagePosition.contains(position))) {
continue;
}
newExpr += ` ${pipelineStagePosition.getExpression(origExpr)}`;
}
return newExpr;
};
queryContainsValidPipelineStages = (query: LokiQuery | undefined): boolean => {
const origExpr = query?.expr ?? '';
const allNodePositions = getNodePositionsFromQuery(origExpr, [
PipelineStage,
LabelParser,
LineFilters,
LabelFilter,
]);
const pipelineStagePositions = allNodePositions.filter((position) => position.type?.id === PipelineStage);
const otherNodePositions = allNodePositions.filter((position) => position.type?.id !== PipelineStage);
return pipelineStagePositions.some((pipelineStagePosition) =>
otherNodePositions.every((position) => pipelineStagePosition.contains(position) === false)
);
};
getInitContextFilters = async (
row: LogRowModel,
query?: LokiQuery,
timeRange?: TimeRange
): Promise<{ contextFilters: ContextFilter[]; preservedFiltersApplied: boolean }> => {
let preservedFiltersApplied = false;
if (!query || isEmpty(row.labels)) {
return { contextFilters: [], preservedFiltersApplied };
}
const rowLabels = row.labels;
// 1. First we need to get all labels from the log row's label
// and correctly set parsed and not parsed labels
let allLabels: string[] = [];
if (!isQueryWithParser(query.expr).queryWithParser) {
// If there is no parser, we use getLabelKeys because it has better caching
// and all labels should already be fetched
await this.datasource.languageProvider.start(timeRange);
allLabels = this.datasource.languageProvider.getLabelKeys();
} else {
// If we have parser, we use fetchLabels to fetch actual labels for selected stream
const stream = getStreamSelectorsFromQuery(query.expr);
// We are using stream[0] as log query can always have just 1 stream selector
allLabels = await this.datasource.languageProvider.fetchLabels({ streamSelector: stream[0], timeRange });
}
const contextFilters: ContextFilter[] = [];
Object.entries(rowLabels).forEach(([label, value]) => {
const labelType = getLabelTypeFromFrame(label, row.dataFrame, row.rowIndex);
const filter: ContextFilter = {
label,
value: value,
enabled: allLabels.includes(label),
nonIndexed: labelType !== null && labelType !== LabelType.Indexed,
};
contextFilters.push(filter);
});
// Secondly we check for preserved labels and update enabled state of filters based on that
let preservedLabels: undefined | PreservedLabels = undefined;
const preservedLabelsString = window.localStorage.getItem(LOKI_LOG_CONTEXT_PRESERVED_LABELS);
if (preservedLabelsString) {
try {
preservedLabels = JSON.parse(preservedLabelsString);
// Do nothing when error occurs
} catch (e) {}
}
if (!preservedLabels) {
// If we don't have preservedLabels, we return contextFilters as they are
return { contextFilters, preservedFiltersApplied };
} else {
// Otherwise, we need to update filters based on preserved labels
let arePreservedLabelsUsed = false;
const newContextFilters = contextFilters.map((contextFilter) => {
// We checked for undefined above
if (preservedLabels!.removedLabels.includes(contextFilter.label)) {
arePreservedLabelsUsed = true;
return { ...contextFilter, enabled: false };
}
// We checked for undefined above
if (preservedLabels!.selectedExtractedLabels.includes(contextFilter.label)) {
arePreservedLabelsUsed = true;
return { ...contextFilter, enabled: true };
}
return { ...contextFilter };
});
const isAtLeastOneRealLabelEnabled = newContextFilters.some(({ enabled, nonIndexed }) => enabled && !nonIndexed);
if (!isAtLeastOneRealLabelEnabled) {
// If we end up with no real labels enabled, we need to reset the init filters
return { contextFilters, preservedFiltersApplied };
} else {
if (arePreservedLabelsUsed) {
preservedFiltersApplied = true;
}
return { contextFilters: newContextFilters, preservedFiltersApplied };
}
}
};
}