diff --git a/packages/grafana-data/src/types/dataFrame.ts b/packages/grafana-data/src/types/dataFrame.ts index bb4e9784e34..36c27d1b845 100644 --- a/packages/grafana-data/src/types/dataFrame.ts +++ b/packages/grafana-data/src/types/dataFrame.ts @@ -67,6 +67,12 @@ export interface FieldConfig { min?: number | null; max?: number | null; + // Interval indicates the expected regular step between values in the series. + // When an interval exists, consumers can identify "missing" values when the expected value is not present. + // The grafana timeseries visualization will render disconnected values when missing values are found it the time field. + // The interval uses the same units as the values. For time.Time, this is defined in milliseconds. + interval?: number | null; + // Convert input values into a display string mappings?: ValueMapping[]; diff --git a/packages/grafana-ui/src/components/GraphNG/nullInsertThreshold.test.ts b/packages/grafana-ui/src/components/GraphNG/nullInsertThreshold.test.ts new file mode 100644 index 00000000000..c74fb32dbf5 --- /dev/null +++ b/packages/grafana-ui/src/components/GraphNG/nullInsertThreshold.test.ts @@ -0,0 +1,215 @@ +import { ArrayVector, FieldType, MutableDataFrame } from '@grafana/data'; +import { applyNullInsertThreshold } from './nullInsertThreshold'; + +function randInt(min: number, max: number) { + return Math.floor(Math.random() * (max - min + 1) + min); +} + +function genFrame() { + let fieldCount = 10; + let valueCount = 3000; + let step = 1000; + let skipProb = 0.5; + let skipSteps = [1, 5]; // min, max + + let allValues = Array(fieldCount); + + allValues[0] = Array(valueCount); + + for (let i = 0, curStep = Date.now(); i < valueCount; i++) { + curStep = allValues[0][i] = curStep + step * (Math.random() < skipProb ? randInt(skipSteps[0], skipSteps[1]) : 1); + } + + for (let fi = 1; fi < fieldCount; fi++) { + let values = Array(valueCount); + + for (let i = 0; i < valueCount; i++) { + values[i] = Math.random() * 100; + } + + allValues[fi] = values; + } + + return { + length: valueCount, + fields: allValues.map((values, i) => { + return { + name: 'A-' + i, + type: i === 0 ? FieldType.time : FieldType.number, + config: { + interval: i === 0 ? step : null, + }, + values: new ArrayVector(values), + }; + }), + }; +} + +describe('nullInsertThreshold Transformer', () => { + test('should insert nulls at +threshold between adjacent > threshold: 1', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [1, 3, 10] }, + { name: 'One', type: FieldType.number, config: { custom: { insertNulls: 1 } }, values: [4, 6, 8] }, + { name: 'Two', type: FieldType.string, config: { custom: { insertNulls: 1 } }, values: ['a', 'b', 'c'] }, + ], + }); + + const result = applyNullInsertThreshold(df); + + expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 10]); + expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, 8]); + expect(result.fields[2].values.toArray()).toStrictEqual(['a', null, 'b', null, 'c']); + }); + + test('should insert nulls at +threshold between adjacent > threshold: 2', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [5, 7, 11] }, + { name: 'One', type: FieldType.number, config: { custom: { insertNulls: 2 } }, values: [4, 6, 8] }, + { name: 'Two', type: FieldType.string, config: { custom: { insertNulls: 2 } }, values: ['a', 'b', 'c'] }, + ], + }); + + const result = applyNullInsertThreshold(df); + + expect(result.fields[0].values.toArray()).toStrictEqual([5, 7, 9, 11]); + expect(result.fields[1].values.toArray()).toStrictEqual([4, 6, null, 8]); + expect(result.fields[2].values.toArray()).toStrictEqual(['a', 'b', null, 'c']); + }); + + test('should insert nulls at +interval between adjacent > interval: 1', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, config: { interval: 1 }, values: [1, 3, 10] }, + { name: 'One', type: FieldType.number, values: [4, 6, 8] }, + { name: 'Two', type: FieldType.string, values: ['a', 'b', 'c'] }, + ], + }); + + const result = applyNullInsertThreshold(df); + + expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 10]); + expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, 8]); + expect(result.fields[2].values.toArray()).toStrictEqual(['a', null, 'b', null, 'c']); + }); + + // TODO: make this work + test.skip('should insert nulls at +threshold (when defined) instead of +interval', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, config: { interval: 2 }, values: [5, 7, 11] }, + { name: 'One', type: FieldType.number, config: { custom: { insertNulls: 1 } }, values: [4, 6, 8] }, + { name: 'Two', type: FieldType.string, config: { custom: { insertNulls: 1 } }, values: ['a', 'b', 'c'] }, + ], + }); + + const result = applyNullInsertThreshold(df); + + expect(result.fields[0].values.toArray()).toStrictEqual([5, 6, 7, 8, 11]); + expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, 8]); + expect(result.fields[2].values.toArray()).toStrictEqual(['a', null, 'b', null, 'c']); + }); + + test('should insert nulls at midpoints between adjacent > interval: 2', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, config: { interval: 2 }, values: [5, 7, 11] }, + { name: 'One', type: FieldType.number, values: [4, 6, 8] }, + { name: 'Two', type: FieldType.string, values: ['a', 'b', 'c'] }, + ], + }); + + const result = applyNullInsertThreshold(df); + + expect(result.fields[0].values.toArray()).toStrictEqual([5, 7, 9, 11]); + expect(result.fields[1].values.toArray()).toStrictEqual([4, 6, null, 8]); + expect(result.fields[2].values.toArray()).toStrictEqual(['a', 'b', null, 'c']); + }); + + test('should noop on fewer than two values', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, config: { interval: 1 }, values: [1] }, + { name: 'Value', type: FieldType.number, values: [1] }, + ], + }); + + const result = applyNullInsertThreshold(df); + + expect(result).toBe(df); + }); + + test('should noop on invalid threshold', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [1, 2, 4] }, + { name: 'Value', type: FieldType.number, config: { custom: { insertNulls: -1 } }, values: [1, 1, 1] }, + ], + }); + + const result = applyNullInsertThreshold(df); + + expect(result).toBe(df); + }); + + test('should noop on invalid interval', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, config: { interval: -1 }, values: [1, 2, 4] }, + { name: 'Value', type: FieldType.number, values: [1, 1, 1] }, + ], + }); + + const result = applyNullInsertThreshold(df); + + expect(result).toBe(df); + }); + + test('should noop when no missing steps', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, config: { interval: 1 }, values: [1, 2, 3] }, + { name: 'Value', type: FieldType.number, values: [1, 1, 1] }, + ], + }); + + const result = applyNullInsertThreshold(df); + + expect(result).toBe(df); + }); + + test('should noop when refFieldName not found', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'Time', type: FieldType.time, config: { interval: 1 }, values: [1, 2, 5] }, + { name: 'Value', type: FieldType.number, values: [1, 1, 1] }, + ], + }); + + const result = applyNullInsertThreshold(df, 'Time2'); + + expect(result).toBe(df); + }); + + test('perf stress test should be <= 10ms', () => { + // 10 fields x 3,000 values with 50% skip (output = 10 fields x 6,000 values) + let bigFrameA = genFrame(); + + // eslint-disable-next-line no-console + console.time('insertValues-10x3k'); + applyNullInsertThreshold(bigFrameA); + // eslint-disable-next-line no-console + console.timeEnd('insertValues-10x3k'); + }); +}); diff --git a/packages/grafana-ui/src/components/GraphNG/nullInsertThreshold.ts b/packages/grafana-ui/src/components/GraphNG/nullInsertThreshold.ts new file mode 100644 index 00000000000..4e22d0af2b1 --- /dev/null +++ b/packages/grafana-ui/src/components/GraphNG/nullInsertThreshold.ts @@ -0,0 +1,115 @@ +import { ArrayVector, DataFrame, FieldType } from '@grafana/data'; + +type InsertMode = (prev: number, next: number, threshold: number) => number; + +const INSERT_MODES = { + threshold: (prev: number, next: number, threshold: number) => prev + threshold, + midpoint: (prev: number, next: number, threshold: number) => (prev + next) / 2, + // previous time + 1ms to prevent StateTimeline from forward-interpolating prior state + plusone: (prev: number, next: number, threshold: number) => prev + 1, +}; + +export function applyNullInsertThreshold( + frame: DataFrame, + refFieldName?: string | null, + insertMode: InsertMode = INSERT_MODES.threshold +): DataFrame { + if (frame.length < 2) { + return frame; + } + + const refField = frame.fields.find((field) => { + // note: getFieldDisplayName() would require full DF[] + return refFieldName != null ? field.name === refFieldName : field.type === FieldType.time; + }); + + if (refField == null) { + return frame; + } + + const thresholds = frame.fields.map((field) => field.config.custom?.insertNulls ?? refField.config.interval ?? null); + + const uniqueThresholds = new Set(thresholds); + + uniqueThresholds.delete(null as any); + + if (uniqueThresholds.size === 0) { + return frame; + } + + if (uniqueThresholds.size === 1) { + const threshold = uniqueThresholds.values().next().value; + + if (threshold <= 0) { + return frame; + } + + const refValues = refField.values.toArray(); + + const frameValues = frame.fields.map((field) => field.values.toArray()); + + const filledFieldValues = nullInsertThreshold(refValues, frameValues, threshold, insertMode); + + if (filledFieldValues === frameValues) { + return frame; + } + + return { + ...frame, + length: filledFieldValues[0].length, + fields: frame.fields.map((field, i) => ({ + ...field, + values: new ArrayVector(filledFieldValues[i]), + })), + }; + } + + // TODO: unique threshold-per-field (via overrides) is unimplemented + // should be done by processing each (refField + thresholdA-field1 + thresholdA-field2...) + // as a separate nullInsertThreshold() dataset, then re-join into single dataset via join() + return frame; +} + +function nullInsertThreshold(refValues: number[], frameValues: any[][], threshold: number, getInsertValue: InsertMode) { + const len = refValues.length; + let prevValue: number = refValues[0]; + const refValuesNew: number[] = [prevValue]; + + for (let i = 1; i < len; i++) { + const curValue = refValues[i]; + + if (curValue - prevValue > threshold) { + refValuesNew.push(getInsertValue(prevValue, curValue, threshold)); + } + + refValuesNew.push(curValue); + + prevValue = curValue; + } + + const filledLen = refValuesNew.length; + + if (filledLen === len) { + return frameValues; + } + + const filledFieldValues: any[][] = []; + + for (let fieldValues of frameValues) { + let filledValues; + + if (fieldValues !== refValues) { + filledValues = Array(filledLen); + + for (let i = 0, j = 0; i < filledLen; i++) { + filledValues[i] = refValues[j] === refValuesNew[i] ? fieldValues[j++] : null; + } + } else { + filledValues = refValuesNew; + } + + filledFieldValues.push(filledValues); + } + + return filledFieldValues; +} diff --git a/packages/grafana-ui/src/components/GraphNG/utils.ts b/packages/grafana-ui/src/components/GraphNG/utils.ts index fe733543086..427408064a0 100644 --- a/packages/grafana-ui/src/components/GraphNG/utils.ts +++ b/packages/grafana-ui/src/components/GraphNG/utils.ts @@ -1,6 +1,7 @@ import { XYFieldMatchers } from './types'; import { ArrayVector, DataFrame, FieldConfig, FieldType, outerJoinDataFrames } from '@grafana/data'; import { nullToUndefThreshold } from './nullToUndefThreshold'; +import { applyNullInsertThreshold } from './nullInsertThreshold'; import { AxisPlacement, GraphFieldConfig, ScaleDistribution, ScaleDistributionConfig } from '@grafana/schema'; import { FIXED_UNIT } from './GraphNG'; @@ -32,7 +33,7 @@ function applySpanNullsThresholds(frame: DataFrame) { export function preparePlotFrame(frames: DataFrame[], dimFields: XYFieldMatchers) { let alignedFrame = outerJoinDataFrames({ - frames: frames, + frames: frames.map((frame) => applyNullInsertThreshold(frame)), joinBy: dimFields.x, keep: dimFields.y, keepOriginIndices: true, diff --git a/packages/grafana-ui/src/components/Sparkline/utils.ts b/packages/grafana-ui/src/components/Sparkline/utils.ts index d9ec460f3b3..9f57db07152 100644 --- a/packages/grafana-ui/src/components/Sparkline/utils.ts +++ b/packages/grafana-ui/src/components/Sparkline/utils.ts @@ -1,5 +1,6 @@ import { DataFrame, FieldConfig, FieldSparkline, IndexVector } from '@grafana/data'; import { GraphFieldConfig } from '@grafana/schema'; +import { applyNullInsertThreshold } from '../GraphNG/nullInsertThreshold'; /** @internal * Given a sparkline config returns a DataFrame ready to be turned into Plot data set @@ -11,7 +12,7 @@ export function preparePlotFrame(sparkline: FieldSparkline, config?: FieldConfig ...config, }; - return { + return applyNullInsertThreshold({ refId: 'sparkline', fields: [ sparkline.x ?? IndexVector.newField(length), @@ -21,5 +22,5 @@ export function preparePlotFrame(sparkline: FieldSparkline, config?: FieldConfig }, ], length, - }; + }); } diff --git a/public/app/plugins/panel/state-timeline/timeline.ts b/public/app/plugins/panel/state-timeline/timeline.ts index b82ca170a22..d81d2384db8 100644 --- a/public/app/plugins/panel/state-timeline/timeline.ts +++ b/public/app/plugins/panel/state-timeline/timeline.ts @@ -41,6 +41,7 @@ export interface TimelineCoreOptions { colWidth?: number; theme: GrafanaTheme2; showValue: VisibilityMode; + mergeValues?: boolean; isDiscrete: (seriesIdx: number) => boolean; getValueColor: (seriesIdx: number, value: any) => string; label: (seriesIdx: number) => string; @@ -62,6 +63,7 @@ export function getConfig(opts: TimelineCoreOptions) { rowHeight = 0, colWidth = 0, showValue, + mergeValues = false, theme, label, formatValue, @@ -212,11 +214,16 @@ export function getConfig(opts: TimelineCoreOptions) { walk(rowHeight, sidx - 1, numSeries, yDim, (iy, y0, height) => { if (mode === TimelineMode.Changes) { for (let ix = 0; ix < dataY.length; ix++) { - if (dataY[ix] != null) { + let yVal = dataY[ix]; + + if (yVal != null) { let left = Math.round(valToPosX(dataX[ix], scaleX, xDim, xOff)); let nextIx = ix; - while (dataY[++nextIx] === undefined && nextIx < dataY.length) {} + while ( + ++nextIx < dataY.length && + (dataY[nextIx] === undefined || (mergeValues && dataY[nextIx] === yVal)) + ) {} // to now (not to end of chart) let right = @@ -236,7 +243,7 @@ export function getConfig(opts: TimelineCoreOptions) { strokeWidth, iy, ix, - dataY[ix], + yVal, discrete ); diff --git a/public/app/plugins/panel/state-timeline/utils.test.ts b/public/app/plugins/panel/state-timeline/utils.test.ts index 8e7fb9f4a90..0382bf21bd0 100644 --- a/public/app/plugins/panel/state-timeline/utils.test.ts +++ b/public/app/plugins/panel/state-timeline/utils.test.ts @@ -54,12 +54,12 @@ describe('prepare timeline graph', () => { const field = out.fields.find((f) => f.name === 'b'); expect(field?.values.toArray()).toMatchInlineSnapshot(` Array [ + 1, 1, undefined, - undefined, - undefined, + 1, + 2, 2, - undefined, null, 2, 3, diff --git a/public/app/plugins/panel/state-timeline/utils.ts b/public/app/plugins/panel/state-timeline/utils.ts index f5cbefd4e67..74209a04412 100644 --- a/public/app/plugins/panel/state-timeline/utils.ts +++ b/public/app/plugins/panel/state-timeline/utils.ts @@ -69,6 +69,7 @@ export const preparePlotConfigBuilder: UPlotConfigPrepFn = ({ colWidth, showValue, alignValue, + mergeValues, }) => { const builder = new UPlotConfigBuilder(timeZone); @@ -98,6 +99,7 @@ export const preparePlotConfigBuilder: UPlotConfigPrepFn = ({ mode: mode!, numSeries: frame.fields.length - 1, isDiscrete: (seriesIdx) => isDiscrete(frame.fields[seriesIdx]), + mergeValues, rowHeight: rowHeight!, colWidth: colWidth, showValue: showValue!, @@ -329,7 +331,6 @@ export function mergeThresholdValues(field: Field, theme: GrafanaTheme2): Field textToColor.set(items[i].label, items[i].color!); } - let prev: Threshold | undefined = undefined; let input = field.values.toArray(); const vals = new Array(field.values.length); if (thresholds.mode === ThresholdsMode.Percentage) { @@ -347,19 +348,21 @@ export function mergeThresholdValues(field: Field, theme: GrafanaTheme2): Field const v = input[i]; if (v == null) { vals[i] = v; - prev = undefined; - } - const active = getActiveThreshold(v, thresholds.steps); - if (active === prev) { - vals[i] = undefined; } else { - vals[i] = thresholdToText.get(active); + vals[i] = thresholdToText.get(getActiveThreshold(v, thresholds.steps)); } - prev = active; } return { ...field, + config: { + ...field.config, + custom: { + ...field.config.custom, + // magic value for join() to leave nulls alone + spanNulls: -1, + }, + }, type: FieldType.string, values: new ArrayVector(vals), display: (value: string) => ({ @@ -415,18 +418,6 @@ export function prepareTimelineFields( }, }, }; - - if (mergeValues) { - let merged = unsetSameFutureValues(field.values.toArray()); - if (merged) { - fields.push({ - ...field, - values: new ArrayVector(merged), - }); - changed = true; - continue; - } - } fields.push(field); break; default: