mirror of https://github.com/grafana/grafana
Loki: Full range logs volume (#39327)
* Basic implementation of getLogsVolumeQuery method * Add todos * Add a switcher to automatically load logs volume * De-scope dismissing logs volume panel * De-scope logs volume query cancellation * Remove todo * Aggregate logs volume components in single panel * Show logs volume only when it's available * Aggregate logs volume by level * Simplify aggregation * Handle no logs volume data * Add error handling * Do not show auto-load logs volume switcher when loading logs volume is not available * Remove old logs volume graph * Clean up * Make getting data provider more generic * Provide complete logs volume data (error, isLoading) * Display more specific error message * Add missing props to mocks * Remove setRequest method * Mark getQueryRelatedDataProviders as internal * Add missing dataQueryRequest and add a todo * Remove redundant loading state * Do not mutate existing queries * Apply fix for zooming-in from main * Post-merge fixes * Create collection for data provider results * Use more generic names * Move aggregation logic to Loki logs volume provider * Move LogsVolume to common types * Update tests * Post-merge fixes * Fix mapping related data values * Simplify prop mappings * Add docs * Fix property name * Clean-up * Mark new types as internal * Reduce number of providers to logs volume only * Simplify data structure to DataQueryResponse * Move Logs Volume panel to a separate component * Test logsVolumeProvider.ts * Add observable version of datasource mock * Test getLogsVolumeDataProvider method * Test LogsVolumePanel * Test logs volume reducer * Clean up * Clean up * Fix test * Use sum by to use level field directly * Fix strict type errors * Fix strict type errors * Use "logs" instead of "unknown" if only one level was detected * Add docs about logs volume * Rename histogramRequest to logsVolumeRequest * Use LogsVolumeContentWrapper all content types * Move `autoLoadLogsVolume` local storage handling * Fix strict error * Move getting autoLoadLogsVolume to initial state * Cancel current logs volume subscription * Test cancelling subscriptions * Update docs/sources/datasources/loki.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update packages/grafana-data/src/types/explore.ts Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Inline container styles * Ensure logs volume is aggregated per each subscription separately * Simplify logs volume provider * Type-guard support for logs volume provider * Simplify event handlers to avoid casting * Clean up and docs * Move auto-load switcher to logs volume panel * Fix test * Move DataSourceWithLogsVolumeSupport to avoid cross referencing * Simplify interface * Bring back old histogram and hide the new one behind a feature flag * Add missing props to logs histogram panel * Clean up the provider when it's not supported * Simplify storing autoLoadLogsVolume * Remove docs * Update packages/grafana-data/src/types/logsVolume.ts Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com> * Skip dataframes without fields (instant queries) * Revert styles changes * Revert styles changes * Add release tag Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com>pull/39862/head
Piotr Jamróz
4 years ago
committed by
GitHub
parent
b7a68a9516
commit
124e9daf26
@ -0,0 +1,22 @@ |
||||
import { DataQuery } from './query'; |
||||
import { DataQueryRequest, DataQueryResponse } from './datasource'; |
||||
import { Observable } from 'rxjs'; |
||||
|
||||
/** |
||||
* TODO: This should be added to ./logs.ts but because of cross reference between ./datasource.ts and ./logs.ts it can |
||||
* be done only after decoupling "logs" from "datasource" (https://github.com/grafana/grafana/pull/39536)
|
||||
* |
||||
* @internal |
||||
*/ |
||||
export interface DataSourceWithLogsVolumeSupport<TQuery extends DataQuery> { |
||||
getLogsVolumeDataProvider(request: DataQueryRequest<TQuery>): Observable<DataQueryResponse> | undefined; |
||||
} |
||||
|
||||
/** |
||||
* @internal |
||||
*/ |
||||
export const hasLogsVolumeSupport = <TQuery extends DataQuery>( |
||||
datasource: any |
||||
): datasource is DataSourceWithLogsVolumeSupport<TQuery> => { |
||||
return (datasource as DataSourceWithLogsVolumeSupport<TQuery>).getLogsVolumeDataProvider !== undefined; |
||||
}; |
||||
@ -0,0 +1,56 @@ |
||||
import React from 'react'; |
||||
import { render, screen } from '@testing-library/react'; |
||||
import { LogsVolumePanel } from './LogsVolumePanel'; |
||||
import { ExploreId } from '../../types'; |
||||
import { DataQueryResponse, LoadingState } from '@grafana/data'; |
||||
|
||||
jest.mock('./ExploreGraph', () => { |
||||
const ExploreGraph = () => <span>ExploreGraph</span>; |
||||
return { |
||||
ExploreGraph, |
||||
}; |
||||
}); |
||||
|
||||
function renderPanel(logsVolumeData?: DataQueryResponse) { |
||||
render( |
||||
<LogsVolumePanel |
||||
exploreId={ExploreId.left} |
||||
loadLogsVolumeData={() => {}} |
||||
absoluteRange={{ from: 0, to: 1 }} |
||||
timeZone="timeZone" |
||||
splitOpen={() => {}} |
||||
width={100} |
||||
onUpdateTimeRange={() => {}} |
||||
logsVolumeData={logsVolumeData} |
||||
autoLoadLogsVolume={false} |
||||
onChangeAutoLogsVolume={() => {}} |
||||
/> |
||||
); |
||||
} |
||||
|
||||
describe('LogsVolumePanel', () => { |
||||
it('shows loading message', () => { |
||||
renderPanel({ state: LoadingState.Loading, error: undefined, data: [] }); |
||||
expect(screen.getByText('Logs volume is loading...')).toBeInTheDocument(); |
||||
}); |
||||
|
||||
it('shows no volume data', () => { |
||||
renderPanel({ state: LoadingState.Done, error: undefined, data: [] }); |
||||
expect(screen.getByText('No volume data.')).toBeInTheDocument(); |
||||
}); |
||||
|
||||
it('renders logs volume histogram graph', () => { |
||||
renderPanel({ state: LoadingState.Done, error: undefined, data: [{}] }); |
||||
expect(screen.getByText('ExploreGraph')).toBeInTheDocument(); |
||||
}); |
||||
|
||||
it('shows error message', () => { |
||||
renderPanel({ state: LoadingState.Error, error: { data: { message: 'Error message' } }, data: [] }); |
||||
expect(screen.getByText('Failed to load volume logs for this query: Error message')).toBeInTheDocument(); |
||||
}); |
||||
|
||||
it('shows button to load logs volume', () => { |
||||
renderPanel(undefined); |
||||
expect(screen.getByText('Load logs volume')).toBeInTheDocument(); |
||||
}); |
||||
}); |
||||
@ -0,0 +1,114 @@ |
||||
import { AbsoluteTimeRange, DataQueryResponse, LoadingState, SplitOpen, TimeZone } from '@grafana/data'; |
||||
import { Button, Collapse, InlineField, InlineFieldRow, InlineSwitch, useTheme2 } from '@grafana/ui'; |
||||
import { ExploreGraph } from './ExploreGraph'; |
||||
import React, { useCallback } from 'react'; |
||||
import { ExploreId } from '../../types'; |
||||
import { css } from '@emotion/css'; |
||||
|
||||
type Props = { |
||||
exploreId: ExploreId; |
||||
loadLogsVolumeData: (exploreId: ExploreId) => void; |
||||
logsVolumeData?: DataQueryResponse; |
||||
absoluteRange: AbsoluteTimeRange; |
||||
timeZone: TimeZone; |
||||
splitOpen: SplitOpen; |
||||
width: number; |
||||
onUpdateTimeRange: (timeRange: AbsoluteTimeRange) => void; |
||||
autoLoadLogsVolume: boolean; |
||||
onChangeAutoLogsVolume: (value: boolean) => void; |
||||
}; |
||||
|
||||
export function LogsVolumePanel(props: Props) { |
||||
const { |
||||
width, |
||||
logsVolumeData, |
||||
exploreId, |
||||
loadLogsVolumeData, |
||||
absoluteRange, |
||||
timeZone, |
||||
splitOpen, |
||||
onUpdateTimeRange, |
||||
autoLoadLogsVolume, |
||||
onChangeAutoLogsVolume, |
||||
} = props; |
||||
const theme = useTheme2(); |
||||
const spacing = parseInt(theme.spacing(2).slice(0, -2), 10); |
||||
const height = 150; |
||||
|
||||
let LogsVolumePanelContent; |
||||
|
||||
if (!logsVolumeData) { |
||||
LogsVolumePanelContent = ( |
||||
<Button |
||||
onClick={() => { |
||||
loadLogsVolumeData(exploreId); |
||||
}} |
||||
> |
||||
Load logs volume |
||||
</Button> |
||||
); |
||||
} else if (logsVolumeData?.error) { |
||||
LogsVolumePanelContent = ( |
||||
<span> |
||||
Failed to load volume logs for this query:{' '} |
||||
{logsVolumeData.error.data?.message || logsVolumeData.error.statusText} |
||||
</span> |
||||
); |
||||
} else if (logsVolumeData?.state === LoadingState.Loading) { |
||||
LogsVolumePanelContent = <span>Logs volume is loading...</span>; |
||||
} else if (logsVolumeData?.data) { |
||||
if (logsVolumeData.data.length > 0) { |
||||
LogsVolumePanelContent = ( |
||||
<ExploreGraph |
||||
loadingState={LoadingState.Done} |
||||
data={logsVolumeData.data} |
||||
height={height} |
||||
width={width - spacing} |
||||
absoluteRange={absoluteRange} |
||||
onChangeTime={onUpdateTimeRange} |
||||
timeZone={timeZone} |
||||
splitOpenFn={splitOpen} |
||||
/> |
||||
); |
||||
} else { |
||||
LogsVolumePanelContent = <span>No volume data.</span>; |
||||
} |
||||
} |
||||
|
||||
const handleOnChangeAutoLogsVolume = useCallback( |
||||
(event: React.ChangeEvent<HTMLInputElement>) => { |
||||
const { target } = event; |
||||
if (target) { |
||||
onChangeAutoLogsVolume(target.checked); |
||||
} |
||||
}, |
||||
[onChangeAutoLogsVolume] |
||||
); |
||||
|
||||
return ( |
||||
<Collapse label="Logs volume" isOpen={true} loading={logsVolumeData?.state === LoadingState.Loading}> |
||||
<div |
||||
style={{ height }} |
||||
className={css({ |
||||
display: 'flex', |
||||
alignItems: 'center', |
||||
justifyContent: 'center', |
||||
})} |
||||
> |
||||
{LogsVolumePanelContent} |
||||
</div> |
||||
<div |
||||
className={css({ |
||||
display: 'flex', |
||||
justifyContent: 'end', |
||||
})} |
||||
> |
||||
<InlineFieldRow> |
||||
<InlineField label="Auto-load logs volume" transparent> |
||||
<InlineSwitch value={autoLoadLogsVolume} onChange={handleOnChangeAutoLogsVolume} transparent /> |
||||
</InlineField> |
||||
</InlineFieldRow> |
||||
</div> |
||||
</Collapse> |
||||
); |
||||
} |
||||
@ -0,0 +1,107 @@ |
||||
import { MockObservableDataSourceApi } from '../../../../../test/mocks/datasource_srv'; |
||||
import { createLokiLogsVolumeProvider } from './logsVolumeProvider'; |
||||
import LokiDatasource from '../datasource'; |
||||
import { DataQueryRequest, DataQueryResponse, FieldType, LoadingState, toDataFrame } from '@grafana/data'; |
||||
import { LokiQuery } from '../types'; |
||||
import { Observable } from 'rxjs'; |
||||
|
||||
function createFrame(labels: object, timestamps: number[], values: number[]) { |
||||
return toDataFrame({ |
||||
fields: [ |
||||
{ name: 'Time', type: FieldType.time, values: timestamps }, |
||||
{ |
||||
name: 'Number', |
||||
type: FieldType.number, |
||||
values, |
||||
labels, |
||||
}, |
||||
], |
||||
}); |
||||
} |
||||
|
||||
function createExpectedFields(levelName: string, timestamps: number[], values: number[]) { |
||||
return [ |
||||
{ name: 'Time', values: { buffer: timestamps } }, |
||||
{ |
||||
name: 'Value', |
||||
config: { displayNameFromDS: levelName }, |
||||
values: { buffer: values }, |
||||
}, |
||||
]; |
||||
} |
||||
|
||||
describe('LokiLogsVolumeProvider', () => { |
||||
let volumeProvider: Observable<DataQueryResponse>, |
||||
datasource: MockObservableDataSourceApi, |
||||
request: DataQueryRequest<LokiQuery>; |
||||
|
||||
function setup(datasourceSetup: () => void) { |
||||
datasourceSetup(); |
||||
request = ({ |
||||
targets: [{ expr: '{app="app01"}' }, { expr: '{app="app02"}' }], |
||||
} as unknown) as DataQueryRequest<LokiQuery>; |
||||
volumeProvider = createLokiLogsVolumeProvider((datasource as unknown) as LokiDatasource, request); |
||||
} |
||||
|
||||
function setupMultipleResults() { |
||||
// level=unknown
|
||||
const resultAFrame1 = createFrame({ app: 'app01' }, [100, 200, 300], [5, 5, 5]); |
||||
// level=error
|
||||
const resultAFrame2 = createFrame({ app: 'app01', level: 'error' }, [100, 200, 300], [0, 1, 0]); |
||||
// level=unknown
|
||||
const resultBFrame1 = createFrame({ app: 'app02' }, [100, 200, 300], [1, 2, 3]); |
||||
// level=error
|
||||
const resultBFrame2 = createFrame({ app: 'app02', level: 'error' }, [100, 200, 300], [1, 1, 1]); |
||||
|
||||
datasource = new MockObservableDataSourceApi('loki', [ |
||||
{ |
||||
data: [resultAFrame1, resultAFrame2], |
||||
}, |
||||
{ |
||||
data: [resultBFrame1, resultBFrame2], |
||||
}, |
||||
]); |
||||
} |
||||
|
||||
function setupErrorResponse() { |
||||
datasource = new MockObservableDataSourceApi('loki', [], undefined, 'Error message'); |
||||
} |
||||
|
||||
it('aggregates data frames by level', async () => { |
||||
setup(setupMultipleResults); |
||||
|
||||
await expect(volumeProvider).toEmitValuesWith((received) => { |
||||
expect(received).toMatchObject([ |
||||
{ state: LoadingState.Loading, error: undefined, data: [] }, |
||||
{ |
||||
state: LoadingState.Done, |
||||
error: undefined, |
||||
data: [ |
||||
{ |
||||
fields: createExpectedFields('unknown', [100, 200, 300], [6, 7, 8]), |
||||
}, |
||||
{ |
||||
fields: createExpectedFields('error', [100, 200, 300], [1, 2, 1]), |
||||
}, |
||||
], |
||||
}, |
||||
]); |
||||
}); |
||||
}); |
||||
|
||||
it('returns error', async () => { |
||||
setup(setupErrorResponse); |
||||
|
||||
await expect(volumeProvider).toEmitValuesWith((received) => { |
||||
expect(received).toMatchObject([ |
||||
{ state: LoadingState.Loading, error: undefined, data: [] }, |
||||
{ |
||||
state: LoadingState.Error, |
||||
error: 'Error message', |
||||
data: [], |
||||
}, |
||||
'Error message', |
||||
]); |
||||
}); |
||||
}); |
||||
}); |
||||
@ -0,0 +1,175 @@ |
||||
import { |
||||
DataFrame, |
||||
DataQueryRequest, |
||||
DataQueryResponse, |
||||
FieldCache, |
||||
FieldColorModeId, |
||||
FieldConfig, |
||||
FieldType, |
||||
getLogLevelFromKey, |
||||
Labels, |
||||
LoadingState, |
||||
LogLevel, |
||||
MutableDataFrame, |
||||
toDataFrame, |
||||
} from '@grafana/data'; |
||||
import { LokiQuery } from '../types'; |
||||
import { Observable } from 'rxjs'; |
||||
import { cloneDeep } from 'lodash'; |
||||
import LokiDatasource, { isMetricsQuery } from '../datasource'; |
||||
import { LogLevelColor } from '../../../../core/logs_model'; |
||||
import { BarAlignment, GraphDrawStyle, StackingMode } from '@grafana/schema'; |
||||
|
||||
export function createLokiLogsVolumeProvider( |
||||
datasource: LokiDatasource, |
||||
dataQueryRequest: DataQueryRequest<LokiQuery> |
||||
): Observable<DataQueryResponse> { |
||||
const logsVolumeRequest = cloneDeep(dataQueryRequest); |
||||
logsVolumeRequest.targets = logsVolumeRequest.targets |
||||
.filter((target) => target.expr && !isMetricsQuery(target.expr)) |
||||
.map((target) => { |
||||
return { |
||||
...target, |
||||
expr: `sum by (level) (count_over_time(${target.expr}[$__interval]))`, |
||||
}; |
||||
}); |
||||
|
||||
return new Observable((observer) => { |
||||
let rawLogsVolume: DataFrame[] = []; |
||||
observer.next({ |
||||
state: LoadingState.Loading, |
||||
error: undefined, |
||||
data: [], |
||||
}); |
||||
|
||||
const subscription = datasource.query(logsVolumeRequest).subscribe({ |
||||
complete: () => { |
||||
const aggregatedLogsVolume = aggregateRawLogsVolume(rawLogsVolume); |
||||
observer.next({ |
||||
state: LoadingState.Done, |
||||
error: undefined, |
||||
data: aggregatedLogsVolume, |
||||
}); |
||||
observer.complete(); |
||||
}, |
||||
next: (dataQueryResponse: DataQueryResponse) => { |
||||
rawLogsVolume = rawLogsVolume.concat(dataQueryResponse.data.map(toDataFrame)); |
||||
}, |
||||
error: (error) => { |
||||
observer.next({ |
||||
state: LoadingState.Error, |
||||
error: error, |
||||
data: [], |
||||
}); |
||||
observer.error(error); |
||||
}, |
||||
}); |
||||
return () => { |
||||
subscription?.unsubscribe(); |
||||
}; |
||||
}); |
||||
} |
||||
|
||||
/** |
||||
* Add up values for the same level and create a single data frame for each level |
||||
*/ |
||||
function aggregateRawLogsVolume(rawLogsVolume: DataFrame[]): DataFrame[] { |
||||
const logsVolumeByLevelMap: { [level in LogLevel]?: DataFrame[] } = {}; |
||||
let levels = 0; |
||||
rawLogsVolume.forEach((dataFrame) => { |
||||
let valueField; |
||||
try { |
||||
valueField = new FieldCache(dataFrame).getFirstFieldOfType(FieldType.number); |
||||
} catch {} |
||||
// If value field doesn't exist skip the frame (it may happen with instant queries)
|
||||
if (!valueField) { |
||||
return; |
||||
} |
||||
const level: LogLevel = valueField.labels ? getLogLevelFromLabels(valueField.labels) : LogLevel.unknown; |
||||
if (!logsVolumeByLevelMap[level]) { |
||||
logsVolumeByLevelMap[level] = []; |
||||
levels++; |
||||
} |
||||
logsVolumeByLevelMap[level]!.push(dataFrame); |
||||
}); |
||||
|
||||
return Object.keys(logsVolumeByLevelMap).map((level: string) => { |
||||
return aggregateFields(logsVolumeByLevelMap[level as LogLevel]!, getFieldConfig(level as LogLevel, levels)); |
||||
}); |
||||
} |
||||
|
||||
function getFieldConfig(level: LogLevel, levels: number) { |
||||
const name = levels === 1 && level === LogLevel.unknown ? 'logs' : level; |
||||
const color = LogLevelColor[level]; |
||||
return { |
||||
displayNameFromDS: name, |
||||
color: { |
||||
mode: FieldColorModeId.Fixed, |
||||
fixedColor: color, |
||||
}, |
||||
custom: { |
||||
drawStyle: GraphDrawStyle.Bars, |
||||
barAlignment: BarAlignment.Center, |
||||
barWidthFactor: 0.9, |
||||
barMaxWidth: 5, |
||||
lineColor: color, |
||||
pointColor: color, |
||||
fillColor: color, |
||||
lineWidth: 1, |
||||
fillOpacity: 100, |
||||
stacking: { |
||||
mode: StackingMode.Normal, |
||||
group: 'A', |
||||
}, |
||||
}, |
||||
}; |
||||
} |
||||
|
||||
/** |
||||
* Create a new data frame with a single field and values creating by adding field values |
||||
* from all provided data frames |
||||
*/ |
||||
function aggregateFields(dataFrames: DataFrame[], config: FieldConfig): DataFrame { |
||||
const aggregatedDataFrame = new MutableDataFrame(); |
||||
if (!dataFrames.length) { |
||||
return aggregatedDataFrame; |
||||
} |
||||
|
||||
const totalLength = dataFrames[0].length; |
||||
const timeField = new FieldCache(dataFrames[0]).getFirstFieldOfType(FieldType.time); |
||||
|
||||
if (!timeField) { |
||||
return aggregatedDataFrame; |
||||
} |
||||
|
||||
aggregatedDataFrame.addField({ name: 'Time', type: FieldType.time }, totalLength); |
||||
aggregatedDataFrame.addField({ name: 'Value', type: FieldType.number, config }, totalLength); |
||||
|
||||
dataFrames.forEach((dataFrame) => { |
||||
dataFrame.fields.forEach((field) => { |
||||
if (field.type === FieldType.number) { |
||||
for (let pointIndex = 0; pointIndex < totalLength; pointIndex++) { |
||||
const currentValue = aggregatedDataFrame.get(pointIndex).Value; |
||||
const valueToAdd = field.values.get(pointIndex); |
||||
const totalValue = |
||||
currentValue === null && valueToAdd === null ? null : (currentValue || 0) + (valueToAdd || 0); |
||||
aggregatedDataFrame.set(pointIndex, { Value: totalValue, Time: timeField.values.get(pointIndex) }); |
||||
} |
||||
} |
||||
}); |
||||
}); |
||||
|
||||
return aggregatedDataFrame; |
||||
} |
||||
|
||||
function getLogLevelFromLabels(labels: Labels): LogLevel { |
||||
const labelNames = ['level', 'lvl', 'loglevel']; |
||||
let levelLabel; |
||||
for (let labelName of labelNames) { |
||||
if (labelName in labels) { |
||||
levelLabel = labelName; |
||||
break; |
||||
} |
||||
} |
||||
return levelLabel ? getLogLevelFromKey(labels[levelLabel]) : LogLevel.unknown; |
||||
} |
||||
Loading…
Reference in new issue