The open and composable observability and data visualization platform. Visualize metrics, logs, and traces from multiple sources like Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
grafana/public/app/plugins/datasource/loki/datasource.ts

784 lines
26 KiB

import { cloneDeep, map as lodashMap } from 'lodash';
import { lastValueFrom, merge, Observable, of, throwError } from 'rxjs';
import { catchError, map, switchMap } from 'rxjs/operators';
import {
AnnotationEvent,
AnnotationQueryRequest,
CoreApp,
DataFrame,
DataFrameView,
DataQueryError,
DataQueryRequest,
DataQueryResponse,
DataSourceInstanceSettings,
DataSourceWithLogsContextSupport,
Loki: Full range logs volume (#39327) * Basic implementation of getLogsVolumeQuery method * Add todos * Add a switcher to automatically load logs volume * De-scope dismissing logs volume panel * De-scope logs volume query cancellation * Remove todo * Aggregate logs volume components in single panel * Show logs volume only when it's available * Aggregate logs volume by level * Simplify aggregation * Handle no logs volume data * Add error handling * Do not show auto-load logs volume switcher when loading logs volume is not available * Remove old logs volume graph * Clean up * Make getting data provider more generic * Provide complete logs volume data (error, isLoading) * Display more specific error message * Add missing props to mocks * Remove setRequest method * Mark getQueryRelatedDataProviders as internal * Add missing dataQueryRequest and add a todo * Remove redundant loading state * Do not mutate existing queries * Apply fix for zooming-in from main * Post-merge fixes * Create collection for data provider results * Use more generic names * Move aggregation logic to Loki logs volume provider * Move LogsVolume to common types * Update tests * Post-merge fixes * Fix mapping related data values * Simplify prop mappings * Add docs * Fix property name * Clean-up * Mark new types as internal * Reduce number of providers to logs volume only * Simplify data structure to DataQueryResponse * Move Logs Volume panel to a separate component * Test logsVolumeProvider.ts * Add observable version of datasource mock * Test getLogsVolumeDataProvider method * Test LogsVolumePanel * Test logs volume reducer * Clean up * Clean up * Fix test * Use sum by to use level field directly * Fix strict type errors * Fix strict type errors * Use "logs" instead of "unknown" if only one level was detected * Add docs about logs volume * Rename histogramRequest to logsVolumeRequest * Use LogsVolumeContentWrapper all content types * Move `autoLoadLogsVolume` local storage handling * Fix strict error * Move getting autoLoadLogsVolume to initial state * Cancel current logs volume subscription * Test cancelling subscriptions * Update docs/sources/datasources/loki.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update packages/grafana-data/src/types/explore.ts Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Inline container styles * Ensure logs volume is aggregated per each subscription separately * Simplify logs volume provider * Type-guard support for logs volume provider * Simplify event handlers to avoid casting * Clean up and docs * Move auto-load switcher to logs volume panel * Fix test * Move DataSourceWithLogsVolumeSupport to avoid cross referencing * Simplify interface * Bring back old histogram and hide the new one behind a feature flag * Add missing props to logs histogram panel * Clean up the provider when it's not supported * Simplify storing autoLoadLogsVolume * Remove docs * Update packages/grafana-data/src/types/logsVolume.ts Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com> * Skip dataframes without fields (instant queries) * Revert styles changes * Revert styles changes * Add release tag Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com>
4 years ago
DataSourceWithLogsVolumeSupport,
DataSourceWithQueryExportSupport,
DataSourceWithQueryImportSupport,
dateMath,
DateTime,
FieldCache,
AbstractQuery,
FieldType,
getLogLevelFromKey,
Labels,
LoadingState,
LogLevel,
LogRowModel,
ScopedVars,
TimeRange,
rangeUtil,
toUtc,
QueryHint,
getDefaultTimeRange,
QueryFixAction,
} from '@grafana/data';
import { FetchError, config, DataSourceWithBackend } from '@grafana/runtime';
import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider';
import { queryLogsVolume } from 'app/core/logsModel';
import { convertToWebSocketUrl } from 'app/core/utils/explore';
import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
import { serializeParams } from '../../../core/utils/fetch';
import { renderLegendFormat } from '../prometheus/legend';
import { transformBackendResult } from './backendResultTransformer';
import { LokiAnnotationsQueryEditor } from './components/AnnotationsQueryEditor';
import LanguageProvider from './language_provider';
import { escapeLabelValueInSelector } from './language_utils';
import { LiveStreams, LokiLiveTarget } from './live_streams';
import {
addLabelFormatToQuery,
addLabelToQuery,
addNoPipelineErrorToQuery,
addParserToQuery,
removeCommentsFromQuery,
} from './modifyQuery';
import { getQueryHints } from './queryHints';
import { getNormalizedLokiQuery, isLogsQuery, isValidQuery } from './query_utils';
import { sortDataFrameByTime } from './sortDataFrame';
import { doLokiChannelStream } from './streaming';
import { LokiOptions, LokiQuery, LokiQueryDirection, LokiQueryType } from './types';
export type RangeQueryOptions = DataQueryRequest<LokiQuery> | AnnotationQueryRequest<LokiQuery>;
6 years ago
export const DEFAULT_MAX_LINES = 1000;
export const LOKI_ENDPOINT = '/loki/api/v1';
const NS_IN_MS = 1000000;
function makeRequest(query: LokiQuery, range: TimeRange, app: CoreApp, requestId: string): DataQueryRequest<LokiQuery> {
const intervalInfo = rangeUtil.calculateInterval(range, 1);
return {
targets: [query],
requestId,
interval: intervalInfo.interval,
intervalMs: intervalInfo.intervalMs,
range: range,
scopedVars: {},
timezone: 'UTC',
app,
startTime: Date.now(),
};
}
Loki: Full range logs volume (#39327) * Basic implementation of getLogsVolumeQuery method * Add todos * Add a switcher to automatically load logs volume * De-scope dismissing logs volume panel * De-scope logs volume query cancellation * Remove todo * Aggregate logs volume components in single panel * Show logs volume only when it's available * Aggregate logs volume by level * Simplify aggregation * Handle no logs volume data * Add error handling * Do not show auto-load logs volume switcher when loading logs volume is not available * Remove old logs volume graph * Clean up * Make getting data provider more generic * Provide complete logs volume data (error, isLoading) * Display more specific error message * Add missing props to mocks * Remove setRequest method * Mark getQueryRelatedDataProviders as internal * Add missing dataQueryRequest and add a todo * Remove redundant loading state * Do not mutate existing queries * Apply fix for zooming-in from main * Post-merge fixes * Create collection for data provider results * Use more generic names * Move aggregation logic to Loki logs volume provider * Move LogsVolume to common types * Update tests * Post-merge fixes * Fix mapping related data values * Simplify prop mappings * Add docs * Fix property name * Clean-up * Mark new types as internal * Reduce number of providers to logs volume only * Simplify data structure to DataQueryResponse * Move Logs Volume panel to a separate component * Test logsVolumeProvider.ts * Add observable version of datasource mock * Test getLogsVolumeDataProvider method * Test LogsVolumePanel * Test logs volume reducer * Clean up * Clean up * Fix test * Use sum by to use level field directly * Fix strict type errors * Fix strict type errors * Use "logs" instead of "unknown" if only one level was detected * Add docs about logs volume * Rename histogramRequest to logsVolumeRequest * Use LogsVolumeContentWrapper all content types * Move `autoLoadLogsVolume` local storage handling * Fix strict error * Move getting autoLoadLogsVolume to initial state * Cancel current logs volume subscription * Test cancelling subscriptions * Update docs/sources/datasources/loki.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update packages/grafana-data/src/types/explore.ts Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Inline container styles * Ensure logs volume is aggregated per each subscription separately * Simplify logs volume provider * Type-guard support for logs volume provider * Simplify event handlers to avoid casting * Clean up and docs * Move auto-load switcher to logs volume panel * Fix test * Move DataSourceWithLogsVolumeSupport to avoid cross referencing * Simplify interface * Bring back old histogram and hide the new one behind a feature flag * Add missing props to logs histogram panel * Clean up the provider when it's not supported * Simplify storing autoLoadLogsVolume * Remove docs * Update packages/grafana-data/src/types/logsVolume.ts Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com> * Skip dataframes without fields (instant queries) * Revert styles changes * Revert styles changes * Add release tag Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com>
4 years ago
export class LokiDatasource
extends DataSourceWithBackend<LokiQuery, LokiOptions>
implements
DataSourceWithLogsContextSupport,
DataSourceWithLogsVolumeSupport<LokiQuery>,
DataSourceWithQueryImportSupport<LokiQuery>,
DataSourceWithQueryExportSupport<LokiQuery>
{
private streams = new LiveStreams();
languageProvider: LanguageProvider;
6 years ago
maxLines: number;
constructor(
private instanceSettings: DataSourceInstanceSettings<LokiOptions>,
private readonly templateSrv: TemplateSrv = getTemplateSrv(),
private readonly timeSrv: TimeSrv = getTimeSrv()
) {
super(instanceSettings);
this.languageProvider = new LanguageProvider(this);
const settingsData = instanceSettings.jsonData || {};
this.maxLines = parseInt(settingsData.maxLines ?? '0', 10) || DEFAULT_MAX_LINES;
this.annotations = {
QueryEditor: LokiAnnotationsQueryEditor,
};
}
Loki: Full range logs volume (#39327) * Basic implementation of getLogsVolumeQuery method * Add todos * Add a switcher to automatically load logs volume * De-scope dismissing logs volume panel * De-scope logs volume query cancellation * Remove todo * Aggregate logs volume components in single panel * Show logs volume only when it's available * Aggregate logs volume by level * Simplify aggregation * Handle no logs volume data * Add error handling * Do not show auto-load logs volume switcher when loading logs volume is not available * Remove old logs volume graph * Clean up * Make getting data provider more generic * Provide complete logs volume data (error, isLoading) * Display more specific error message * Add missing props to mocks * Remove setRequest method * Mark getQueryRelatedDataProviders as internal * Add missing dataQueryRequest and add a todo * Remove redundant loading state * Do not mutate existing queries * Apply fix for zooming-in from main * Post-merge fixes * Create collection for data provider results * Use more generic names * Move aggregation logic to Loki logs volume provider * Move LogsVolume to common types * Update tests * Post-merge fixes * Fix mapping related data values * Simplify prop mappings * Add docs * Fix property name * Clean-up * Mark new types as internal * Reduce number of providers to logs volume only * Simplify data structure to DataQueryResponse * Move Logs Volume panel to a separate component * Test logsVolumeProvider.ts * Add observable version of datasource mock * Test getLogsVolumeDataProvider method * Test LogsVolumePanel * Test logs volume reducer * Clean up * Clean up * Fix test * Use sum by to use level field directly * Fix strict type errors * Fix strict type errors * Use "logs" instead of "unknown" if only one level was detected * Add docs about logs volume * Rename histogramRequest to logsVolumeRequest * Use LogsVolumeContentWrapper all content types * Move `autoLoadLogsVolume` local storage handling * Fix strict error * Move getting autoLoadLogsVolume to initial state * Cancel current logs volume subscription * Test cancelling subscriptions * Update docs/sources/datasources/loki.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update packages/grafana-data/src/types/explore.ts Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Inline container styles * Ensure logs volume is aggregated per each subscription separately * Simplify logs volume provider * Type-guard support for logs volume provider * Simplify event handlers to avoid casting * Clean up and docs * Move auto-load switcher to logs volume panel * Fix test * Move DataSourceWithLogsVolumeSupport to avoid cross referencing * Simplify interface * Bring back old histogram and hide the new one behind a feature flag * Add missing props to logs histogram panel * Clean up the provider when it's not supported * Simplify storing autoLoadLogsVolume * Remove docs * Update packages/grafana-data/src/types/logsVolume.ts Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com> * Skip dataframes without fields (instant queries) * Revert styles changes * Revert styles changes * Add release tag Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com>
4 years ago
getLogsVolumeDataProvider(request: DataQueryRequest<LokiQuery>): Observable<DataQueryResponse> | undefined {
const isQuerySuitable = (query: LokiQuery) => {
const normalized = getNormalizedLokiQuery(query);
const { expr } = normalized;
// it has to be a logs-producing range-query
return expr && isLogsQuery(expr) && normalized.queryType === LokiQueryType.Range;
};
const isLogsVolumeAvailable = request.targets.some(isQuerySuitable);
if (!isLogsVolumeAvailable) {
return undefined;
}
const logsVolumeRequest = cloneDeep(request);
logsVolumeRequest.targets = logsVolumeRequest.targets.filter(isQuerySuitable).map((target) => {
const query = removeCommentsFromQuery(target.expr);
return {
...target,
instant: false,
volumeQuery: true,
expr: `sum by (level) (count_over_time(${query}[$__interval]))`,
};
});
return queryLogsVolume(this, logsVolumeRequest, {
extractLevel,
range: request.range,
targets: request.targets,
});
Loki: Full range logs volume (#39327) * Basic implementation of getLogsVolumeQuery method * Add todos * Add a switcher to automatically load logs volume * De-scope dismissing logs volume panel * De-scope logs volume query cancellation * Remove todo * Aggregate logs volume components in single panel * Show logs volume only when it's available * Aggregate logs volume by level * Simplify aggregation * Handle no logs volume data * Add error handling * Do not show auto-load logs volume switcher when loading logs volume is not available * Remove old logs volume graph * Clean up * Make getting data provider more generic * Provide complete logs volume data (error, isLoading) * Display more specific error message * Add missing props to mocks * Remove setRequest method * Mark getQueryRelatedDataProviders as internal * Add missing dataQueryRequest and add a todo * Remove redundant loading state * Do not mutate existing queries * Apply fix for zooming-in from main * Post-merge fixes * Create collection for data provider results * Use more generic names * Move aggregation logic to Loki logs volume provider * Move LogsVolume to common types * Update tests * Post-merge fixes * Fix mapping related data values * Simplify prop mappings * Add docs * Fix property name * Clean-up * Mark new types as internal * Reduce number of providers to logs volume only * Simplify data structure to DataQueryResponse * Move Logs Volume panel to a separate component * Test logsVolumeProvider.ts * Add observable version of datasource mock * Test getLogsVolumeDataProvider method * Test LogsVolumePanel * Test logs volume reducer * Clean up * Clean up * Fix test * Use sum by to use level field directly * Fix strict type errors * Fix strict type errors * Use "logs" instead of "unknown" if only one level was detected * Add docs about logs volume * Rename histogramRequest to logsVolumeRequest * Use LogsVolumeContentWrapper all content types * Move `autoLoadLogsVolume` local storage handling * Fix strict error * Move getting autoLoadLogsVolume to initial state * Cancel current logs volume subscription * Test cancelling subscriptions * Update docs/sources/datasources/loki.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update packages/grafana-data/src/types/explore.ts Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Inline container styles * Ensure logs volume is aggregated per each subscription separately * Simplify logs volume provider * Type-guard support for logs volume provider * Simplify event handlers to avoid casting * Clean up and docs * Move auto-load switcher to logs volume panel * Fix test * Move DataSourceWithLogsVolumeSupport to avoid cross referencing * Simplify interface * Bring back old histogram and hide the new one behind a feature flag * Add missing props to logs histogram panel * Clean up the provider when it's not supported * Simplify storing autoLoadLogsVolume * Remove docs * Update packages/grafana-data/src/types/logsVolume.ts Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com> * Skip dataframes without fields (instant queries) * Revert styles changes * Revert styles changes * Add release tag Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com>
4 years ago
}
query(request: DataQueryRequest<LokiQuery>): Observable<DataQueryResponse> {
const queries = request.targets
.map(getNormalizedLokiQuery) // "fix" the `.queryType` prop
.map((q) => ({ ...q, maxLines: q.maxLines || this.maxLines })); // set maxLines if not set
const fixedRequest = {
...request,
targets: queries,
};
const streamQueries = fixedRequest.targets.filter((q) => q.queryType === LokiQueryType.Stream);
if (config.featureToggles.lokiLive && streamQueries.length > 0 && fixedRequest.rangeRaw?.to === 'now') {
// this is still an in-development feature,
// we do not support mixing stream-queries with normal-queries for now.
const streamRequest = {
...fixedRequest,
targets: streamQueries,
};
return merge(
...streamQueries.map((q) =>
doLokiChannelStream(
this.applyTemplateVariables(q, request.scopedVars),
this, // the datasource
streamRequest
)
)
);
}
if (fixedRequest.liveStreaming) {
return this.runLiveQueryThroughBackend(fixedRequest);
} else {
return super
.query(fixedRequest)
.pipe(
map((response) =>
transformBackendResult(response, fixedRequest.targets, this.instanceSettings.jsonData.derivedFields ?? [])
)
);
}
}
runLiveQueryThroughBackend(request: DataQueryRequest<LokiQuery>): Observable<DataQueryResponse> {
// this only works in explore-mode, so variables don't need to be handled,
// and only for logs-queries, not metric queries
const logsQueries = request.targets.filter((query) => query.expr !== '' && isLogsQuery(query.expr));
if (logsQueries.length === 0) {
return of({
data: [],
state: LoadingState.Done,
});
}
const subQueries = logsQueries.map((query) => {
const maxDataPoints = query.maxLines || this.maxLines;
// FIXME: currently we are running it through the frontend still.
return this.runLiveQuery(query, maxDataPoints);
});
return merge(...subQueries);
}
createLiveTarget(target: LokiQuery, maxDataPoints: number): LokiLiveTarget {
const query = target.expr;
const baseUrl = this.instanceSettings.url;
const params = serializeParams({ query });
return {
query,
url: convertToWebSocketUrl(`${baseUrl}/loki/api/v1/tail?${params}`),
refId: target.refId,
size: maxDataPoints,
};
}
/**
* Runs live queries which in this case means creating a websocket and listening on it for new logs.
* This returns a bit different dataFrame than runQueries as it returns single dataframe even if there are multiple
* Loki streams, sets only common labels on dataframe.labels and has additional dataframe.fields.labels for unique
* labels per row.
*/
runLiveQuery = (target: LokiQuery, maxDataPoints: number): Observable<DataQueryResponse> => {
const liveTarget = this.createLiveTarget(target, maxDataPoints);
return this.streams.getStream(liveTarget).pipe(
map((data) => ({
data: data || [],
key: `loki-${liveTarget.refId}`,
state: LoadingState.Streaming,
})),
catchError((err: any) => {
return throwError(() => `Live tailing was stopped due to following error: ${err.reason}`);
})
);
};
getRangeScopedVars(range: TimeRange = this.timeSrv.timeRange()) {
const msRange = range.to.diff(range.from);
const sRange = Math.round(msRange / 1000);
return {
__range_ms: { text: msRange, value: msRange },
__range_s: { text: sRange, value: sRange },
__range: { text: sRange + 's', value: sRange + 's' },
};
}
interpolateVariablesInQueries(queries: LokiQuery[], scopedVars: ScopedVars): LokiQuery[] {
let expandedQueries = queries;
if (queries && queries.length) {
expandedQueries = queries.map((query) => ({
...query,
datasource: this.getRef(),
expr: this.templateSrv.replace(query.expr, scopedVars, this.interpolateQueryExpr),
}));
}
return expandedQueries;
}
getQueryDisplayText(query: LokiQuery) {
return query.expr;
}
getTimeRangeParams() {
const timeRange = this.timeSrv.timeRange();
return { start: timeRange.from.valueOf() * NS_IN_MS, end: timeRange.to.valueOf() * NS_IN_MS };
}
async importFromAbstractQueries(abstractQueries: AbstractQuery[]): Promise<LokiQuery[]> {
await this.languageProvider.start();
const existingKeys = this.languageProvider.labelKeys;
if (existingKeys && existingKeys.length) {
abstractQueries = abstractQueries.map((abstractQuery) => {
abstractQuery.labelMatchers = abstractQuery.labelMatchers.filter((labelMatcher) => {
return existingKeys.includes(labelMatcher.name);
});
return abstractQuery;
});
}
return abstractQueries.map((abstractQuery) => this.languageProvider.importFromAbstractQuery(abstractQuery));
}
async exportToAbstractQueries(queries: LokiQuery[]): Promise<AbstractQuery[]> {
return queries.map((query) => this.languageProvider.exportToAbstractQuery(query));
}
async metadataRequest(url: string, params?: Record<string, string | number>) {
// url must not start with a `/`, otherwise the AJAX-request
// going from the browser will contain `//`, which can cause problems.
if (url.startsWith('/')) {
throw new Error(`invalid metadata request url: ${url}`);
}
const res = await this.getResource(url, params);
return res.data || [];
}
async metricFindQuery(query: string) {
if (!query) {
return Promise.resolve([]);
}
const interpolated = this.templateSrv.replace(query, {}, this.interpolateQueryExpr);
return await this.processMetricFindQuery(interpolated);
}
async processMetricFindQuery(query: string) {
const labelNamesRegex = /^label_names\(\)\s*$/;
const labelValuesRegex = /^label_values\((?:(.+),\s*)?([a-zA-Z_][a-zA-Z0-9_]*)\)\s*$/;
const labelNames = query.match(labelNamesRegex);
if (labelNames) {
return await this.labelNamesQuery();
}
const labelValues = query.match(labelValuesRegex);
if (labelValues) {
// If we have query expr, use /series endpoint
if (labelValues[1]) {
return await this.labelValuesSeriesQuery(labelValues[1], labelValues[2]);
}
return await this.labelValuesQuery(labelValues[2]);
}
return Promise.resolve([]);
}
async labelNamesQuery() {
const url = 'labels';
const params = this.getTimeRangeParams();
const result = await this.metadataRequest(url, params);
return result.map((value: string) => ({ text: value }));
}
async labelValuesQuery(label: string) {
const params = this.getTimeRangeParams();
const url = `label/${label}/values`;
const result = await this.metadataRequest(url, params);
return result.map((value: string) => ({ text: value }));
}
async labelValuesSeriesQuery(expr: string, label: string) {
const timeParams = this.getTimeRangeParams();
const params = {
...timeParams,
'match[]': expr,
};
const url = 'series';
const streams = new Set();
const result = await this.metadataRequest(url, params);
result.forEach((stream: { [key: string]: string }) => {
if (stream[label]) {
streams.add({ text: stream[label] });
}
});
return Array.from(streams);
}
async getDataSamples(query: LokiQuery): Promise<DataFrame[]> {
// Currently works only for log samples
if (!isValidQuery(query.expr) || !isLogsQuery(query.expr)) {
return [];
}
const lokiLogsQuery: LokiQuery = {
expr: query.expr,
queryType: LokiQueryType.Range,
refId: 'log-samples',
maxLines: 10,
};
// For samples, we use defaultTimeRange (now-6h/now) and limit od 10 lines so queries are small and fast
const timeRange = getDefaultTimeRange();
const request = makeRequest(lokiLogsQuery, timeRange, CoreApp.Explore, 'log-samples');
return await lastValueFrom(this.query(request).pipe(switchMap((res) => of(res.data))));
}
// By implementing getTagKeys and getTagValues we add ad-hoc filters functionality
async getTagKeys() {
return await this.labelNamesQuery();
}
async getTagValues(options: any = {}) {
return await this.labelValuesQuery(options.key);
}
interpolateQueryExpr(value: any, variable: any) {
// if no multi or include all do not regexEscape
if (!variable.multi && !variable.includeAll) {
return lokiRegularEscape(value);
}
if (typeof value === 'string') {
return lokiSpecialRegexEscape(value);
}
const escapedValues = lodashMap(value, lokiSpecialRegexEscape);
return escapedValues.join('|');
}
modifyQuery(query: LokiQuery, action: QueryFixAction): LokiQuery {
let expression = query.expr ?? '';
switch (action.type) {
case 'ADD_FILTER': {
if (action.options?.key && action.options?.value) {
expression = this.addLabelToQuery(expression, action.options.key, '=', action.options.value);
}
break;
}
case 'ADD_FILTER_OUT': {
if (action.options?.key && action.options?.value) {
expression = this.addLabelToQuery(expression, action.options.key, '!=', action.options.value);
}
break;
}
case 'ADD_LOGFMT_PARSER': {
expression = addParserToQuery(expression, 'logfmt');
break;
}
case 'ADD_JSON_PARSER': {
expression = addParserToQuery(expression, 'json');
break;
}
case 'ADD_NO_PIPELINE_ERROR': {
expression = addNoPipelineErrorToQuery(expression);
break;
}
case 'ADD_LEVEL_LABEL_FORMAT': {
if (action.options?.originalLabel && action.options?.renameTo) {
expression = addLabelFormatToQuery(expression, {
renameTo: action.options.renameTo,
originalLabel: action.options.originalLabel,
});
}
break;
}
default:
break;
}
return { ...query, expr: expression };
}
getTime(date: string | DateTime, roundUp: boolean) {
if (typeof date === 'string') {
date = dateMath.parse(date, roundUp)!;
}
return Math.ceil(date.valueOf() * 1e6);
}
getLogRowContext = async (row: LogRowModel, options?: RowContextOptions): Promise<{ data: DataFrame[] }> => {
const direction = (options && options.direction) || 'BACKWARD';
const limit = (options && options.limit) || 10;
const { query, range } = await this.prepareLogRowContextQueryTarget(row, limit, direction);
const processDataFrame = (frame: DataFrame): DataFrame => {
// log-row-context requires specific field-names to work, so we set them here: "ts", "line", "id"
const cache = new FieldCache(frame);
const timestampField = cache.getFirstFieldOfType(FieldType.time);
const lineField = cache.getFirstFieldOfType(FieldType.string);
const idField = cache.getFieldByName('id');
if (timestampField === undefined || lineField === undefined || idField === undefined) {
// this should never really happen, but i want to keep typescript happy
return { ...frame, fields: [] };
}
return {
...frame,
fields: [
{
...timestampField,
name: 'ts',
},
{
...lineField,
name: 'line',
},
{
...idField,
name: 'id',
},
],
};
};
const processResults = (result: DataQueryResponse): DataQueryResponse => {
const frames: DataFrame[] = result.data;
const processedFrames = frames
.map((frame) => sortDataFrameByTime(frame, 'DESCENDING'))
.map((frame) => processDataFrame(frame)); // rename fields if needed
return {
...result,
data: processedFrames,
};
};
// this can only be called from explore currently
const app = CoreApp.Explore;
return lastValueFrom(
this.query(makeRequest(query, range, app, `log-row-context-query-${direction}`)).pipe(
catchError((err) => {
const error: DataQueryError = {
message: 'Error during context query. Please check JS console logs.',
status: err.status,
statusText: err.statusText,
};
throw error;
}),
switchMap((res) => of(processResults(res)))
)
);
};
prepareLogRowContextQueryTarget = async (
row: LogRowModel,
limit: number,
direction: 'BACKWARD' | 'FORWARD'
): Promise<{ query: LokiQuery; range: TimeRange }> => {
// need to await the languageProvider to be started to have all labels. This call is not blocking after it has been called once.
await this.languageProvider.start();
const labels = this.languageProvider.getLabelKeys();
const expr = Object.keys(row.labels)
.map((label: string) => {
if (labels.includes(label)) {
// escape backslashes in label as users can't escape them by themselves
return `${label}="${row.labels[label].replace(/\\/g, '\\\\')}"`;
}
return '';
})
// Filter empty strings
.filter((label) => !!label)
.join(',');
const contextTimeBuffer = 2 * 60 * 60 * 1000; // 2h buffer
const queryDirection = direction === 'FORWARD' ? LokiQueryDirection.Forward : LokiQueryDirection.Backward;
const query: LokiQuery = {
expr: `{${expr}}`,
queryType: LokiQueryType.Range,
refId: '',
maxLines: limit,
direction: queryDirection,
};
const fieldCache = new FieldCache(row.dataFrame);
const tsField = fieldCache.getFirstFieldOfType(FieldType.time);
if (tsField === undefined) {
throw new Error('loki: dataframe missing time-field, should never happen');
}
const tsValue = tsField.values.get(row.rowIndex);
const timestamp = toUtc(tsValue);
const range =
queryDirection === LokiQueryDirection.Forward
? {
// start param in Loki API is inclusive so we'll have to filter out the row that this request is based from
// and any other that were logged in the same ns but before the row. Right now these rows will be lost
// because the are before but came it he response that should return only rows after.
from: timestamp,
// convert to ns, we loose some precision here but it is not that important at the far points of the context
to: toUtc(row.timeEpochMs + contextTimeBuffer),
}
: {
// convert to ns, we loose some precision here but it is not that important at the far points of the context
from: toUtc(row.timeEpochMs - contextTimeBuffer),
to: timestamp,
};
return {
query,
range: {
from: range.from,
to: range.to,
raw: range,
},
};
};
testDatasource(): Promise<{ status: string; message: string }> {
// Consider only last 10 minutes otherwise request takes too long
const nowMs = Date.now();
const params = {
start: (nowMs - 10 * 60 * 1000) * NS_IN_MS,
end: nowMs * NS_IN_MS,
};
return this.metadataRequest('labels', params).then(
(values) => {
return values.length > 0
? { status: 'success', message: 'Data source connected and labels found.' }
: {
status: 'error',
message:
'Data source connected, but no labels received. Verify that Loki and Promtail is configured properly.',
};
},
(err) => {
// we did a resource-call that failed.
// the only info we have, if exists, is err.data.message
// (when in development-mode, err.data.error exists too, but not in production-mode)
// things like err.status & err.statusText does not help,
// because those will only describe how the request between browser<>server failed
const info: string = err?.data?.message ?? '';
const infoInParentheses = info !== '' ? ` (${info})` : '';
const message = `Unable to fetch labels from Loki${infoInParentheses}, please check the server logs for more details`;
return { status: 'error', message: message };
}
);
}
async annotationQuery(options: any): Promise<AnnotationEvent[]> {
const { expr, maxLines, instant, tagKeys = '', titleFormat = '', textFormat = '' } = options.annotation;
if (!expr) {
return [];
}
const id = `annotation-${options.annotation.name}`;
const query: LokiQuery = {
refId: id,
expr,
maxLines,
instant,
queryType: instant ? LokiQueryType.Instant : LokiQueryType.Range,
};
const request = makeRequest(query, options.range, CoreApp.Dashboard, id);
const { data } = await lastValueFrom(this.query(request));
const annotations: AnnotationEvent[] = [];
const splitKeys: string[] = tagKeys.split(',').filter((v: string) => v !== '');
QueryProcessing: Observable query interface and RxJS for query & stream processing (#18899) * I needed to learn some rxjs and understand this more, so just playing around * Updated * Removed all the complete calls * Refactoring * StreamHandler -> observable start * progress * simple singal works * Handle update time range * added error handling * wrap old function * minor changes * handle data format in the subscribe function * Use replay subject to return last value to subscribers * Set loading state after no response in 50ms * added missing file * updated comment * Added cancelation of network requests * runRequest: Added unit test scenario framework * Progress on tests * minor refactor of unit tests * updated test * removed some old code * Shared queries work again, and also became so much simplier * unified query and observe methods * implict any fix * Fixed closed subject issue * removed comment * Use last returned data for loading state * WIP: Explore to runRequest makover step1 * Minor progress * Minor progress on explore and runRequest * minor progress * Things are starting to work in explore * Updated prometheus to use new observable query response, greatly simplified code * Revert refId change * Found better solution for key/refId/requestId problem * use observable with loki * tests compile * fix loki query prep * Explore: correct first response handling * Refactorings * Refactoring * Explore: Fixes LoadingState and GraphResults between runs (#18986) * Refactor: Adds state to DataQueryResponse * Fix: Fixes so we do not empty results before new data arrives Fixes: #17409 * Transformations work * observable test data * remove single() from loki promise * Fixed comment * Explore: Fixes failing Loki and Prometheus unit tests (#18995) * Tests: Makes datasource tests work again * Fix: Fixes loki datasource so highligthing works * Chore: Runs Prettier * Fixed query runner tests * Delay loading state indication to 200ms * Fixed test * fixed unit tests * Clear cached calcs * Fixed bug getProcesedDataFrames * Fix the correct test is a better idea * Fix: Fixes so queries in Explore are only run if Graph/Table is shown (#19000) * Fix: Fixes so queries in Explore are only run if Graph/Table is shown Fixes: #18618 * Refactor: Removes unnecessary condition * PanelData: provide legacy data only when needed (#19018) * no legacy * invert logic... now compiles * merge getQueryResponseData and getDataRaw * update comment about query editor * use single getData() function * only send legacy when it is used in explore * pre process rather than post process * pre process rather than post process * Minor refactoring * Add missing tags to test datasource response * MixedDatasource: Adds query observable pattern to MixedDatasource (#19037) * start mixed datasource * Refactor: Refactors into observable parttern * Tests: Fixes tests * Tests: Removes console.log * Refactor: Adds unique requestId
6 years ago
for (const frame of data) {
const view = new DataFrameView<{ Time: string; Line: string; labels: Labels }>(frame);
view.forEach((row) => {
const { labels } = row;
const maybeDuplicatedTags = Object.entries(labels)
.map(([key, val]) => [key, val.trim()]) // trim all label-values
.filter(([key, val]) => {
if (val === '') {
// remove empty
return false;
}
// if tags are specified, remove label if does not match tags
if (splitKeys.length && !splitKeys.includes(key)) {
return false;
}
return true;
})
.map(([key, val]) => val); // keep only the label-value
// remove duplicates
const tags = Array.from(new Set(maybeDuplicatedTags));
QueryProcessing: Observable query interface and RxJS for query & stream processing (#18899) * I needed to learn some rxjs and understand this more, so just playing around * Updated * Removed all the complete calls * Refactoring * StreamHandler -> observable start * progress * simple singal works * Handle update time range * added error handling * wrap old function * minor changes * handle data format in the subscribe function * Use replay subject to return last value to subscribers * Set loading state after no response in 50ms * added missing file * updated comment * Added cancelation of network requests * runRequest: Added unit test scenario framework * Progress on tests * minor refactor of unit tests * updated test * removed some old code * Shared queries work again, and also became so much simplier * unified query and observe methods * implict any fix * Fixed closed subject issue * removed comment * Use last returned data for loading state * WIP: Explore to runRequest makover step1 * Minor progress * Minor progress on explore and runRequest * minor progress * Things are starting to work in explore * Updated prometheus to use new observable query response, greatly simplified code * Revert refId change * Found better solution for key/refId/requestId problem * use observable with loki * tests compile * fix loki query prep * Explore: correct first response handling * Refactorings * Refactoring * Explore: Fixes LoadingState and GraphResults between runs (#18986) * Refactor: Adds state to DataQueryResponse * Fix: Fixes so we do not empty results before new data arrives Fixes: #17409 * Transformations work * observable test data * remove single() from loki promise * Fixed comment * Explore: Fixes failing Loki and Prometheus unit tests (#18995) * Tests: Makes datasource tests work again * Fix: Fixes loki datasource so highligthing works * Chore: Runs Prettier * Fixed query runner tests * Delay loading state indication to 200ms * Fixed test * fixed unit tests * Clear cached calcs * Fixed bug getProcesedDataFrames * Fix the correct test is a better idea * Fix: Fixes so queries in Explore are only run if Graph/Table is shown (#19000) * Fix: Fixes so queries in Explore are only run if Graph/Table is shown Fixes: #18618 * Refactor: Removes unnecessary condition * PanelData: provide legacy data only when needed (#19018) * no legacy * invert logic... now compiles * merge getQueryResponseData and getDataRaw * update comment about query editor * use single getData() function * only send legacy when it is used in explore * pre process rather than post process * pre process rather than post process * Minor refactoring * Add missing tags to test datasource response * MixedDatasource: Adds query observable pattern to MixedDatasource (#19037) * start mixed datasource * Refactor: Refactors into observable parttern * Tests: Fixes tests * Tests: Removes console.log * Refactor: Adds unique requestId
6 years ago
annotations.push({
time: new Date(row.Time).valueOf(),
title: renderLegendFormat(titleFormat, labels),
text: renderLegendFormat(textFormat, labels) || row.Line,
tags,
});
});
}
return annotations;
}
showContextToggle(row?: LogRowModel): boolean {
return (row && row.searchWords && row.searchWords.length > 0) === true;
}
processError(err: FetchError, target: LokiQuery) {
let error: DataQueryError = cloneDeep(err);
error.refId = target.refId;
if (error.data && err.data.message.includes('escape') && target.expr.includes('\\')) {
error.data.message = `Error: ${err.data.message}. Make sure that all special characters are escaped with \\. For more information on escaping of special characters visit LogQL documentation at https://grafana.com/docs/loki/latest/logql/.`;
}
return error;
}
addAdHocFilters(queryExpr: string) {
const adhocFilters = this.templateSrv.getAdhocFilters(this.name);
let expr = queryExpr;
expr = adhocFilters.reduce((acc: string, filter: { key: string; operator: string; value: string }) => {
const { key, operator, value } = filter;
return this.addLabelToQuery(acc, key, operator, value);
}, expr);
return expr;
}
addLabelToQuery(queryExpr: string, key: string, operator: string, value: string) {
const escapedValue = escapeLabelValueInSelector(value, operator);
return addLabelToQuery(queryExpr, key, operator, escapedValue);
}
// Used when running queries through backend
filterQuery(query: LokiQuery): boolean {
if (query.hide || query.expr === '') {
return false;
}
return true;
}
// Used when running queries through backend
applyTemplateVariables(target: LokiQuery, scopedVars: ScopedVars): LokiQuery {
// We want to interpolate these variables on backend
const { __interval, __interval_ms, ...rest } = scopedVars;
const exprWithAdHoc = this.addAdHocFilters(target.expr);
return {
...target,
legendFormat: this.templateSrv.replace(target.legendFormat, rest),
expr: this.templateSrv.replace(exprWithAdHoc, rest, this.interpolateQueryExpr),
};
}
interpolateString(string: string) {
return this.templateSrv.replace(string, undefined, this.interpolateQueryExpr);
}
getVariables(): string[] {
return this.templateSrv.getVariables().map((v) => `$${v.name}`);
}
getQueryHints(query: LokiQuery, result: DataFrame[]): QueryHint[] {
return getQueryHints(query.expr, result);
}
}
export function lokiRegularEscape(value: any) {
if (typeof value === 'string') {
return value.replace(/'/g, "\\\\'");
}
return value;
}
export function lokiSpecialRegexEscape(value: any) {
if (typeof value === 'string') {
return lokiRegularEscape(value.replace(/\\/g, '\\\\\\\\').replace(/[$^*{}\[\]+?.()|]/g, '\\\\$&'));
}
return value;
}
function extractLevel(dataFrame: DataFrame): LogLevel {
let valueField;
try {
valueField = new FieldCache(dataFrame).getFirstFieldOfType(FieldType.number);
} catch {}
return valueField?.labels ? getLogLevelFromLabels(valueField.labels) : LogLevel.unknown;
}
function getLogLevelFromLabels(labels: Labels): LogLevel {
const labelNames = ['level', 'lvl', 'loglevel'];
let levelLabel;
for (let labelName of labelNames) {
if (labelName in labels) {
levelLabel = labelName;
break;
}
}
return levelLabel ? getLogLevelFromKey(labels[levelLabel]) : LogLevel.unknown;
}