From 4771eaba5bc26ea28fd8bcbdc31ca464c90dbc11 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Sun, 18 Nov 2018 09:38:06 +0000 Subject: [PATCH] Explore: POC dedup logging rows - added dedup switches to logs view - strategy 'exact' matches rows that are exact (except for dates) - strategy 'numbers' strips all numbers - strategy 'signature' strips all letters and numbers to that only whitespace and punctuation remains - added duplication indicator next to log level --- public/app/core/logs_model.ts | 48 ++++++++++ public/app/core/specs/logs_model.test.ts | 108 +++++++++++++++++++++++ public/app/features/explore/Logs.tsx | 66 ++++++++++++-- public/sass/pages/_explore.scss | 23 ++++- 4 files changed, 236 insertions(+), 9 deletions(-) create mode 100644 public/app/core/specs/logs_model.test.ts diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index ab0a3f26a88..21f518a682b 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -31,6 +31,7 @@ export interface LogSearchMatch { } export interface LogRow { + duplicates?: number; entry: string; key: string; // timestamp + labels labels: string; @@ -71,6 +72,53 @@ export interface LogsStreamLabels { [key: string]: string; } +export enum LogsDedupStrategy { + none = 'none', + exact = 'exact', + numbers = 'numbers', + signature = 'signature', +} + +const isoDateRegexp = /\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-6]\d[,\.]\d+([+-][0-2]\d:[0-5]\d|Z)/g; +function isDuplicateRow(row: LogRow, other: LogRow, strategy: LogsDedupStrategy): boolean { + switch (strategy) { + case LogsDedupStrategy.exact: + // Exact still strips dates + return row.entry.replace(isoDateRegexp, '') === other.entry.replace(isoDateRegexp, ''); + + case LogsDedupStrategy.numbers: + return row.entry.replace(/\d/g, '') === other.entry.replace(/\d/g, ''); + + case LogsDedupStrategy.signature: + return row.entry.replace(/\w/g, '') === other.entry.replace(/\w/g, ''); + + default: + return false; + } +} + +export function dedupLogRows(logs: LogsModel, strategy: LogsDedupStrategy): LogsModel { + if (strategy === LogsDedupStrategy.none) { + return logs; + } + + const dedupedRows = logs.rows.reduce((result: LogRow[], row: LogRow, index, list) => { + const previous = result[result.length - 1]; + if (index > 0 && isDuplicateRow(row, previous, strategy)) { + previous.duplicates++; + } else { + row.duplicates = 0; + result.push(row); + } + return result; + }, []); + + return { + ...logs, + rows: dedupedRows, + }; +} + export function makeSeriesForLogs(rows: LogRow[], intervalMs: number): TimeSeries[] { // Graph time series by log level const seriesByLevel = {}; diff --git a/public/app/core/specs/logs_model.test.ts b/public/app/core/specs/logs_model.test.ts new file mode 100644 index 00000000000..5e427468339 --- /dev/null +++ b/public/app/core/specs/logs_model.test.ts @@ -0,0 +1,108 @@ +import { dedupLogRows, LogsDedupStrategy, LogsModel } from '../logs_model'; + +describe('dedupLogRows()', () => { + test('should return rows as is when dedup is set to none', () => { + const logs = { + rows: [ + { + entry: 'WARN test 1.23 on [xxx]', + }, + { + entry: 'WARN test 1.23 on [xxx]', + }, + ], + }; + expect(dedupLogRows(logs as LogsModel, LogsDedupStrategy.none).rows).toMatchObject(logs.rows); + }); + + test('should dedup on exact matches', () => { + const logs = { + rows: [ + { + entry: 'WARN test 1.23 on [xxx]', + }, + { + entry: 'WARN test 1.23 on [xxx]', + }, + { + entry: 'INFO test 2.44 on [xxx]', + }, + { + entry: 'WARN test 1.23 on [xxx]', + }, + ], + }; + expect(dedupLogRows(logs as LogsModel, LogsDedupStrategy.exact).rows).toEqual([ + { + duplicates: 1, + entry: 'WARN test 1.23 on [xxx]', + }, + { + duplicates: 0, + entry: 'INFO test 2.44 on [xxx]', + }, + { + duplicates: 0, + entry: 'WARN test 1.23 on [xxx]', + }, + ]); + }); + + test('should dedup on number matches', () => { + const logs = { + rows: [ + { + entry: 'WARN test 1.2323423 on [xxx]', + }, + { + entry: 'WARN test 1.23 on [xxx]', + }, + { + entry: 'INFO test 2.44 on [xxx]', + }, + { + entry: 'WARN test 1.23 on [xxx]', + }, + ], + }; + expect(dedupLogRows(logs as LogsModel, LogsDedupStrategy.numbers).rows).toEqual([ + { + duplicates: 1, + entry: 'WARN test 1.2323423 on [xxx]', + }, + { + duplicates: 0, + entry: 'INFO test 2.44 on [xxx]', + }, + { + duplicates: 0, + entry: 'WARN test 1.23 on [xxx]', + }, + ]); + }); + + test('should dedup on signature matches', () => { + const logs = { + rows: [ + { + entry: 'WARN test 1.2323423 on [xxx]', + }, + { + entry: 'WARN test 1.23 on [xxx]', + }, + { + entry: 'INFO test 2.44 on [xxx]', + }, + { + entry: 'WARN test 1.23 on [xxx]', + }, + ], + }; + expect(dedupLogRows(logs as LogsModel, LogsDedupStrategy.signature).rows).toEqual([ + { + duplicates: 3, + entry: 'WARN test 1.2323423 on [xxx]', + }, + ]); + }); +}); diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index a00aaccb028..9eee5c31376 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -2,7 +2,7 @@ import React, { Fragment, PureComponent } from 'react'; import Highlighter from 'react-highlight-words'; import { RawTimeRange } from 'app/types/series'; -import { LogsModel } from 'app/core/logs_model'; +import { LogsDedupStrategy, LogsModel, dedupLogRows } from 'app/core/logs_model'; import { findHighlightChunksInText } from 'app/core/utils/text'; import { Switch } from 'app/core/components/Switch/Switch'; @@ -32,6 +32,7 @@ interface LogsProps { } interface LogsState { + dedup: LogsDedupStrategy; showLabels: boolean; showLocalTime: boolean; showUtc: boolean; @@ -39,11 +40,21 @@ interface LogsState { export default class Logs extends PureComponent { state = { + dedup: LogsDedupStrategy.none, showLabels: true, showLocalTime: true, showUtc: false, }; + onChangeDedup = (dedup: LogsDedupStrategy) => { + this.setState(prevState => { + if (prevState.dedup === dedup) { + return { dedup: LogsDedupStrategy.none }; + } + return { dedup }; + }); + }; + onChangeLabels = (event: React.SyntheticEvent) => { const target = event.target as HTMLInputElement; this.setState({ @@ -67,9 +78,18 @@ export default class Logs extends PureComponent { render() { const { className = '', data, loading = false, position, range } = this.props; - const { showLabels, showLocalTime, showUtc } = this.state; + const { dedup, showLabels, showLocalTime, showUtc } = this.state; const hasData = data && data.rows && data.rows.length > 0; - const cssColumnSizes = ['4px']; + const dedupedData = dedupLogRows(data, dedup); + const dedupCount = dedupedData.rows.reduce((sum, row) => sum + row.duplicates, 0); + const meta = [...data.meta]; + if (dedup !== LogsDedupStrategy.none) { + meta.push({ + label: 'Dedup count', + value: String(dedupCount), + }); + } + const cssColumnSizes = ['3px']; // Log-level indicator line if (showUtc) { cssColumnSizes.push('minmax(100px, max-content)'); } @@ -102,10 +122,34 @@ export default class Logs extends PureComponent { + this.onChangeDedup(LogsDedupStrategy.none)} + small + /> + this.onChangeDedup(LogsDedupStrategy.exact)} + small + /> + this.onChangeDedup(LogsDedupStrategy.numbers)} + small + /> + this.onChangeDedup(LogsDedupStrategy.signature)} + small + /> {hasData && - data.meta && ( + meta && (
- {data.meta.map(item => ( + {meta.map(item => (
{item.label}: {item.value} @@ -118,9 +162,17 @@ export default class Logs extends PureComponent {
{hasData && - data.rows.map(row => ( + dedupedData.rows.map(row => ( -
+
+ {row.duplicates > 0 && ( +
+ {Array.apply(null, { length: row.duplicates }).map(index => ( +
+ ))} +
+ )} +
{showUtc &&
{row.timestamp}
} {showLocalTime &&
{row.timeLocal}
} {showLabels && ( diff --git a/public/sass/pages/_explore.scss b/public/sass/pages/_explore.scss index 210920e848d..edb637e5e22 100644 --- a/public/sass/pages/_explore.scss +++ b/public/sass/pages/_explore.scss @@ -300,8 +300,8 @@ .logs-row-level { background-color: transparent; - margin: 6px 0; - border-radius: 2px; + margin: 2px 0; + position: relative; opacity: 0.8; } @@ -326,6 +326,25 @@ .logs-row-level-debug { background-color: #1f78c1; } + + .logs-row-level__duplicates { + position: absolute; + width: 9px; + height: 100%; + top: 0; + left: 5px; + display: flex; + flex-wrap: wrap; + align-items: flex-start; + align-content: flex-start; + } + + .logs-row-level__duplicate { + width: 2px; + height: 3px; + background-color: #1f78c1; + margin: 0 1px 1px 0; + } } }