apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Loki: Remove unnecessary deduplication (#29421)

Browse Source 
* Remove unnecessary deduplication

* Remove dedup test as we are not doing dedup on all logs  anymore

* Create unique ids in Loki

* Fix comment

* Fix comment

* Store prev response uids in usedUids

* Revert "Store prev response uids in usedUids"

This reverts commit 05c496e2a8.

* Add comment
pull/29832/head^2
Ivana Huckova 5 years ago committed by GitHub
parent dd326d296e
commit 2a76195aed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 101 additions and 112 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 94
      public/app/core/logs_model.test.ts
  2. 7
      public/app/core/logs_model.ts
  3. 5
      public/app/core/utils/explore.ts
  4. 6
      public/app/features/explore/Explore.tsx
  5. 2
      public/app/features/explore/LiveLogs.tsx
  6. 68
      public/app/plugins/datasource/loki/result_transformer.test.ts
  7. 31
      public/app/plugins/datasource/loki/result_transformer.ts

94
public/app/core/logs_model.test.ts
Unescape View File

@ -575,100 +575,6 @@ describe('dataFrameToLogsModel', () => {
const logsModel = dataFrameToLogsModel(series, 1, 'utc');
expect(logsModel.rows[0].uid).toBe('0');
});
it('given multiple series with equal ids should return expected logs model', () => {
const series: DataFrame[] = [
toDataFrame({
fields: [
{
name: 'ts',
type: FieldType.time,
values: ['1970-01-01T00:00:00Z'],
},
{
name: 'line',
type: FieldType.string,
values: ['WARN boooo 1'],
labels: {
foo: 'bar',
baz: '1',
level: 'dbug',
},
},
{
name: 'id',
type: FieldType.string,
values: ['0'],
},
],
}),
toDataFrame({
fields: [
{
name: 'ts',
type: FieldType.time,
values: ['1970-01-01T00:00:01Z'],
},
{
name: 'line',
type: FieldType.string,
values: ['WARN boooo 2'],
labels: {
foo: 'bar',
baz: '2',
level: 'dbug',
},
},
{
name: 'id',
type: FieldType.string,
values: ['1'],
},
],
}),
toDataFrame({
fields: [
{
name: 'ts',
type: FieldType.time,
values: ['1970-01-01T00:00:01Z'],
},
{
name: 'line',
type: FieldType.string,
values: ['WARN boooo 2'],
labels: {
foo: 'bar',
baz: '2',
level: 'dbug',
},
},
{
name: 'id',
type: FieldType.string,
values: ['1'],
},
],
}),
];
const logsModel = dataFrameToLogsModel(series, 0, 'utc');
expect(logsModel.hasUniqueLabels).toBeTruthy();
expect(logsModel.rows).toHaveLength(2);
expect(logsModel.rows).toMatchObject([
{
entry: 'WARN boooo 1',
labels: { foo: 'bar' },
logLevel: LogLevel.debug,
uniqueLabels: { baz: '1' },
},
{
entry: 'WARN boooo 2',
labels: { foo: 'bar' },
logLevel: LogLevel.debug,
uniqueLabels: { baz: '2' },
},
]);
});
});
describe('logSeriesToLogsModel', () => {

7
public/app/core/logs_model.ts
Unescape View File

@ -32,7 +32,6 @@ import {
} from '@grafana/data';
import { getThemeColor } from 'app/core/utils/colors';
import { deduplicateLogRowsById } from 'app/core/utils/explore';
import { SIPrefix } from '@grafana/data/src/valueFormats/symbolFormatters';
export const LogLevelColor = {
@ -393,8 +392,6 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
}
}
const deduplicatedLogRows = deduplicateLogRowsById(rows);
// Meta data to display in status
const meta: LogsMetaItem[] = [];
if (_.size(commonLabels) > 0) {
@ -416,7 +413,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
if (limits.length > 0) {
meta.push({
label: 'Limit',
value: `${limitValue} (${deduplicatedLogRows.length} returned)`,
value: `${limitValue} (${rows.length} returned)`,
kind: LogsMetaKind.String,
});
}
@ -464,7 +461,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
return {
hasUniqueLabels,
meta,
rows: deduplicatedLogRows,
rows,
};
}

5
public/app/core/utils/explore.ts
Unescape View File

@ -12,7 +12,6 @@ import {
DefaultTimeZone,
HistoryItem,
IntervalValues,
LogRowModel,
LogsDedupStrategy,
LogsSortOrder,
RawTimeRange,
@ -486,10 +485,6 @@ export function getIntervals(range: TimeRange, lowLimit?: string, resolution?: n
return rangeUtil.calculateInterval(range, resolution, lowLimit);
}
export function deduplicateLogRowsById(rows: LogRowModel[]) {
return _.uniqBy(rows, 'uid');
}
export const getFirstNonQueryRowSpecificError = (queryErrors?: DataQueryError[]): DataQueryError | undefined => {
const refId = getValueWithRefId(queryErrors);
return refId ? undefined : getFirstQueryErrorWithoutRefId(queryErrors);

6
public/app/features/explore/Explore.tsx
Unescape View File

@ -57,8 +57,8 @@ import { FILTER_FOR_OPERATOR, FILTER_OUT_OPERATOR, FilterItem } from '@grafana/u
const getStyles = stylesFactory((theme: GrafanaTheme) => {
return {
logsMain: css`
label: logsMain;
exploreMain: css`
label: exploreMain;
// Is needed for some transition animations to work.
position: relative;
margin-top: 21px;
@ -347,7 +347,7 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
}
return (
<main className={cx(styles.logsMain)} style={{ width }}>
<main className={cx(styles.exploreMain)} style={{ width }}>
<ErrorBoundaryAlert>
{showStartPage && StartPage && (
<div className={'grafana-info-box grafana-info-box--max-lg'}>

2
public/app/features/explore/LiveLogs.tsx
Unescape View File

@ -14,7 +14,7 @@ const getStyles = (theme: GrafanaTheme) => ({
font-size: ${theme.typography.size.sm};
display: flex;
flex-flow: column nowrap;
height: 65vh;
height: 60vh;
overflow-y: auto;
:first-child {
margin-top: auto !important;

68
public/app/plugins/datasource/loki/result_transformer.test.ts
Unescape View File

@ -71,6 +71,37 @@ describe('loki result transformer', () => {
expect(data[1].fields[1].values.get(0)).toEqual(streamResult[1].values[0][1]);
expect(data[1].fields[2].values.get(0)).toEqual('75d73d66cff40f9d1a1f2d5a0bf295d0');
});
it('should always generate unique ids for logs', () => {
const streamResultWithDuplicateLogs: LokiStreamResult[] = [
{
stream: {
foo: 'bar',
},
values: [
['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"'],
['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"'],
['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Non-duplicated"'],
['1579857562021616000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Duplicated"'],
],
},
{
stream: {
bar: 'foo',
},
values: [['1579857562021617000', 't=2020-02-12T15:04:51+0000 lvl=info msg="Non-dupliicated"']],
},
];
const data = streamResultWithDuplicateLogs.map(stream => ResultTransformer.lokiStreamResultToDataFrame(stream));
expect(data[0].fields[2].values.get(0)).toEqual('65cee200875f58ee1430d8bd2e8b74e7');
expect(data[0].fields[2].values.get(1)).toEqual('65cee200875f58ee1430d8bd2e8b74e7_1');
expect(data[0].fields[2].values.get(2)).not.toEqual('65cee200875f58ee1430d8bd2e8b74e7_2');
expect(data[0].fields[2].values.get(3)).toEqual('65cee200875f58ee1430d8bd2e8b74e7_2');
expect(data[1].fields[2].values.get(0)).not.toEqual('65cee200875f58ee1430d8bd2e8b74e7_3');
});
});
describe('lokiStreamsToDataFrames', () => {
@ -131,7 +162,44 @@ describe('loki result transformer', () => {
id: '19e8e093d70122b3b53cb6e24efd6e2d',
});
});
it('should always generate unique ids for logs', () => {
const tailResponse: LokiTailResponse = {
streams: [
{
stream: {
filename: '/var/log/grafana/grafana.log',
job: 'grafana',
},
values: [
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 1"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 1"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 2"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Not dupplicated"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 1"'],
['1581519914265798400', 't=2020-02-12T15:04:51+0000 lvl=info msg="Dupplicated 2"'],
],
},
],
};
const data = new CircularDataFrame({ capacity: 6 });
data.addField({ name: 'ts', type: FieldType.time, config: { displayName: 'Time' } });
data.addField({ name: 'tsNs', type: FieldType.time, config: { displayName: 'Time ns' } });
data.addField({ name: 'line', type: FieldType.string }).labels = { job: 'grafana' };
data.addField({ name: 'labels', type: FieldType.other });
data.addField({ name: 'id', type: FieldType.string });
ResultTransformer.appendResponseToBufferedData(tailResponse, data);
expect(data.get(0).id).toEqual('870e4d105741bdfc2c67904ee480d4f3');
expect(data.get(1).id).toEqual('870e4d105741bdfc2c67904ee480d4f3_1');
expect(data.get(2).id).toEqual('707e4ec2b842f389dbb993438505856d');
expect(data.get(3).id).toEqual('78f044015a58fad3e257a855b167d85e');
expect(data.get(4).id).toEqual('870e4d105741bdfc2c67904ee480d4f3_2');
expect(data.get(5).id).toEqual('707e4ec2b842f389dbb993438505856d_1');
});
});
describe('createMetricLabel', () => {
it('should create correct label based on passed variables', () => {
const label = ResultTransformer.createMetricLabel({}, ({

31
public/app/plugins/datasource/loki/result_transformer.ts
Unescape View File

@ -53,12 +53,15 @@ export function lokiStreamResultToDataFrame(stream: LokiStreamResult, reverse?:
const lines = new ArrayVector<string>([]);
const uids = new ArrayVector<string>([]);
// We need to store and track all used uids to ensure that uids are unique
const usedUids: { string?: number } = {};
for (const [ts, line] of stream.values) {
// num ns epoch in string, we convert it to iso string here so it matches old format
times.add(new Date(parseInt(ts.substr(0, ts.length - 6), 10)).toISOString());
timesNs.add(ts);
lines.add(line);
uids.add(createUid(ts, labelsString, line));
uids.add(createUid(ts, labelsString, line, usedUids));
}
return constructDataFrame(times, timesNs, lines, uids, labels, reverse, refId);
@ -127,6 +130,10 @@ export function appendResponseToBufferedData(response: LokiTailResponse, data: M
const labelsField = data.fields[3];
const idField = data.fields[4];
// We are comparing used ids only within the received stream. This could be a problem if the same line + labels + nanosecond timestamp came in 2 separate batches.
// As this is very unlikely, and the result would only affect live-tailing css animation we have decided to not compare all received uids from data param as this would slow down processing.
const usedUids: { string?: number } = {};
for (const stream of streams) {
// Find unique labels
const unique = findUniqueLabels(stream.stream, baseLabels);
@ -141,13 +148,29 @@ export function appendResponseToBufferedData(response: LokiTailResponse, data: M
tsNsField.values.add(ts);
lineField.values.add(line);
labelsField.values.add(unique);
idField.values.add(createUid(ts, allLabelsString, line));
idField.values.add(createUid(ts, allLabelsString, line, usedUids));
}
}
}
function createUid(ts: string, labelsString: string, line: string): string {
return md5(`${ts}_${labelsString}_${line}`);
function createUid(ts: string, labelsString: string, line: string, usedUids: any): string {
// Generate id as hashed nanosecond timestamp, labels and line (this does not have to be unique)
let id = md5(`${ts}_${labelsString}_${line}`);
// Check if generated id is unique
// If not and we've already used it, append it's count after it
if (id in usedUids) {
// Increase the count
const newCount = usedUids[id] + 1;
usedUids[id] = newCount;
// Append count to generated id to make it unique
id = `${id}_${newCount}`;
} else {
// If id is unique and wasn't used, add it to usedUids and start count at 0
usedUids[id] = 0;
}
// Return unique id
return id;
}
function lokiMatrixToTimeSeries(matrixResult: LokiMatrixResult, options: TransformerOptions): TimeSeries {

Write Preview
Loading…
Cancel
Save