mirror of https://github.com/grafana/grafana
Transformations: Add regression analysis transformation (#78457)
* regression analysis first dragt * Swap to better regression libraries * fix name * Interpolate x points instead of using source x points * clean up ui and add feature toggle * fix merge error * change to loop for finding min max, rename resolution * Add docs * add docs and tests * change name to regression analysis * update docs * Fix editor labels * add regression images * fix docspull/78614/head
Oscar Kilhed
2 years ago
committed by
GitHub
parent
7fa73d2b21
commit
ab982e7bd3
|
@ -0,0 +1,142 @@ |
||||
import { |
||||
DataFrame, |
||||
DataFrameDTO, |
||||
DataTransformContext, |
||||
Field, |
||||
FieldType, |
||||
toDataFrame, |
||||
toDataFrameDTO, |
||||
} from '@grafana/data'; |
||||
|
||||
import { ModelType, RegressionTransformer, RegressionTransformerOptions } from './regression'; |
||||
|
||||
describe('Regression transformation', () => { |
||||
it('it should predict a linear regression to exactly fit the data when the data is f(x) = x', () => { |
||||
const source = [ |
||||
toDataFrame({ |
||||
name: 'data', |
||||
refId: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'value', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
], |
||||
}), |
||||
]; |
||||
|
||||
const config: RegressionTransformerOptions = { |
||||
modelType: ModelType.linear, |
||||
predictionCount: 6, |
||||
xFieldName: 'time', |
||||
yFieldName: 'value', |
||||
}; |
||||
|
||||
expect(toEquableDataFrames(RegressionTransformer.transformer(config, {} as DataTransformContext)(source))).toEqual( |
||||
toEquableDataFrames([ |
||||
toEquableDataFrame({ |
||||
name: 'data', |
||||
refId: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5], config: {} }, |
||||
{ name: 'value', type: FieldType.number, values: [0, 1, 2, 3, 4, 5], config: {} }, |
||||
], |
||||
length: 6, |
||||
}), |
||||
toEquableDataFrame({ |
||||
name: 'linear regression', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5], config: {} }, |
||||
{ name: 'value predicted', type: FieldType.number, values: [0, 1, 2, 3, 4, 5], config: {} }, |
||||
], |
||||
length: 6, |
||||
}), |
||||
]) |
||||
); |
||||
}); |
||||
it('it should predict a linear regression where f(x) = 1', () => { |
||||
const source = [ |
||||
toDataFrame({ |
||||
name: 'data', |
||||
refId: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'value', type: FieldType.number, values: [0, 1, 2, 2, 1, 0] }, |
||||
], |
||||
}), |
||||
]; |
||||
|
||||
const config: RegressionTransformerOptions = { |
||||
modelType: ModelType.linear, |
||||
predictionCount: 6, |
||||
xFieldName: 'time', |
||||
yFieldName: 'value', |
||||
}; |
||||
|
||||
expect(toEquableDataFrames(RegressionTransformer.transformer(config, {} as DataTransformContext)(source))).toEqual( |
||||
toEquableDataFrames([ |
||||
toEquableDataFrame({ |
||||
name: 'data', |
||||
refId: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5], config: {} }, |
||||
{ name: 'value', type: FieldType.number, values: [0, 1, 2, 2, 1, 0], config: {} }, |
||||
], |
||||
length: 6, |
||||
}), |
||||
toEquableDataFrame({ |
||||
name: 'linear regression', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5], config: {} }, |
||||
{ name: 'value predicted', type: FieldType.number, values: [1, 1, 1, 1, 1, 1], config: {} }, |
||||
], |
||||
length: 6, |
||||
}), |
||||
]) |
||||
); |
||||
}); |
||||
|
||||
it('it should predict a quadratic function', () => { |
||||
const source = [ |
||||
toDataFrame({ |
||||
name: 'data', |
||||
refId: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'value', type: FieldType.number, values: [0, 1, 2, 2, 1, 0] }, |
||||
], |
||||
}), |
||||
]; |
||||
|
||||
const config: RegressionTransformerOptions = { |
||||
modelType: ModelType.polynomial, |
||||
degree: 2, |
||||
predictionCount: 6, |
||||
xFieldName: 'time', |
||||
yFieldName: 'value', |
||||
}; |
||||
|
||||
const result = RegressionTransformer.transformer(config, {} as DataTransformContext)(source); |
||||
|
||||
expect(result[1].fields[1].values[0]).toBeCloseTo(-0.1, 1); |
||||
expect(result[1].fields[1].values[1]).toBeCloseTo(1.2, 1); |
||||
expect(result[1].fields[1].values[2]).toBeCloseTo(1.86, 1); |
||||
expect(result[1].fields[1].values[3]).toBeCloseTo(1.86, 1); |
||||
expect(result[1].fields[1].values[4]).toBeCloseTo(1.2, 1); |
||||
expect(result[1].fields[1].values[5]).toBeCloseTo(-0.1, 1); |
||||
}); |
||||
}); |
||||
|
||||
function toEquableDataFrame(source: DataFrame): DataFrame { |
||||
return toDataFrame({ |
||||
...source, |
||||
fields: source.fields.map((field: Field) => { |
||||
return { |
||||
...field, |
||||
config: {}, |
||||
}; |
||||
}), |
||||
}); |
||||
} |
||||
|
||||
function toEquableDataFrames(data: DataFrame[]): DataFrameDTO[] { |
||||
return data.map((frame) => toDataFrameDTO(frame)); |
||||
} |
||||
@ -0,0 +1,122 @@ |
||||
import { PolynomialRegression } from 'ml-regression-polynomial'; |
||||
import { SimpleLinearRegression } from 'ml-regression-simple-linear'; |
||||
import { map } from 'rxjs'; |
||||
|
||||
import { |
||||
DataFrame, |
||||
DataTransformerID, |
||||
FieldMatcherID, |
||||
FieldType, |
||||
SynchronousDataTransformerInfo, |
||||
fieldMatchers, |
||||
} from '@grafana/data'; |
||||
|
||||
export enum ModelType { |
||||
linear = 'linear', |
||||
polynomial = 'polynomial', |
||||
} |
||||
|
||||
export interface RegressionTransformerOptions { |
||||
modelType?: ModelType; |
||||
degree?: number; |
||||
xFieldName?: string; |
||||
yFieldName?: string; |
||||
predictionCount?: number; |
||||
} |
||||
|
||||
export const DEFAULTS = { predictionCount: 100, modelType: ModelType.linear, degree: 2 }; |
||||
|
||||
export const RegressionTransformer: SynchronousDataTransformerInfo<RegressionTransformerOptions> = { |
||||
id: DataTransformerID.regression, |
||||
name: 'Regression analysis', |
||||
operator: (options, ctx) => (source) => |
||||
source.pipe(map((data) => RegressionTransformer.transformer(options, ctx)(data))), |
||||
transformer: (options, ctx) => { |
||||
return (frames: DataFrame[]) => { |
||||
const { predictionCount, modelType, degree } = { ...DEFAULTS, ...options }; |
||||
if (frames.length === 0) { |
||||
return frames; |
||||
} |
||||
const matchesY = fieldMatchers.get(FieldMatcherID.byName).get(options.yFieldName); |
||||
const matchesX = fieldMatchers.get(FieldMatcherID.byName).get(options.xFieldName); |
||||
|
||||
let xField; |
||||
let yField; |
||||
for (const frame of frames) { |
||||
const fy = frame.fields.find((f) => matchesY(f, frame, frames)); |
||||
if (fy) { |
||||
yField = fy; |
||||
const fx = frame.fields.find((f) => matchesX(f, frame, frames)); |
||||
if (fx) { |
||||
xField = fx; |
||||
break; |
||||
} else { |
||||
throw 'X and Y fields must be part of the same frame'; |
||||
} |
||||
} |
||||
} |
||||
|
||||
if (!xField || !yField) { |
||||
return frames; |
||||
} |
||||
|
||||
let xMin = xField.values[0]; |
||||
let xMax = xField.values[0]; |
||||
|
||||
for (let i = 1; i < xField.values.length; i++) { |
||||
if (xField.values[i] < xMin) { |
||||
xMin = xField.values[i]; |
||||
} |
||||
if (xField.values[i] > xMax) { |
||||
xMax = xField.values[i]; |
||||
} |
||||
} |
||||
|
||||
const resolution = (xMax - xMin + 1) / predictionCount; |
||||
|
||||
// These are the X values for which we should predict Y
|
||||
const predictionPoints = [...[...Array(predictionCount - 1).keys()].map((_, i) => i * resolution + xMin), xMax]; |
||||
|
||||
// If X is a time field we normalize the time to the start of the timeseries data
|
||||
const normalizationSubtrahend = xField.type === FieldType.time ? xMin : 0; |
||||
|
||||
const yValues = []; |
||||
const xValues = []; |
||||
|
||||
for (let i = 0; i < xField.values.length; i++) { |
||||
if (yField.values[i] !== null) { |
||||
xValues.push(xField.values[i] - normalizationSubtrahend); |
||||
yValues.push(yField.values[i]); |
||||
} |
||||
} |
||||
|
||||
let result: PolynomialRegression | SimpleLinearRegression; |
||||
switch (modelType) { |
||||
case ModelType.linear: |
||||
result = new SimpleLinearRegression(xValues, yValues); |
||||
break; |
||||
case ModelType.polynomial: |
||||
result = new PolynomialRegression(xValues, yValues, degree); |
||||
break; |
||||
default: |
||||
return frames; |
||||
} |
||||
|
||||
const newFrame: DataFrame = { |
||||
name: `${modelType} regression`, |
||||
length: predictionPoints.length, |
||||
fields: [ |
||||
{ name: xField.name, type: xField.type, values: predictionPoints, config: {} }, |
||||
{ |
||||
name: `${yField.name} predicted`, |
||||
type: yField.type, |
||||
values: predictionPoints.map((x) => result.predict(x - normalizationSubtrahend)), |
||||
config: {}, |
||||
}, |
||||
], |
||||
}; |
||||
|
||||
return [...frames, newFrame]; |
||||
}; |
||||
}, |
||||
}; |
||||
@ -0,0 +1,87 @@ |
||||
import { render, screen } from '@testing-library/react'; |
||||
import React from 'react'; |
||||
|
||||
import { FieldType, toDataFrame } from '@grafana/data'; |
||||
|
||||
import { ModelType } from './regression'; |
||||
import { RegressionTransformerEditor } from './regressionEditor'; |
||||
|
||||
describe('FieldToConfigMappingEditor', () => { |
||||
it('Should try to set the first time field as X and first number field as Y', async () => { |
||||
const onChangeMock = jest.fn(); |
||||
|
||||
const df = toDataFrame({ |
||||
name: 'data', |
||||
refId: 'A', |
||||
fields: [ |
||||
{ name: 'time', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'not this', type: FieldType.time, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'value', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'not this either', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
], |
||||
}); |
||||
|
||||
render(<RegressionTransformerEditor input={[df]} onChange={onChangeMock} options={{}} />); |
||||
|
||||
expect(onChangeMock).toBeCalledTimes(1); |
||||
expect(onChangeMock).toBeCalledWith({ xFieldName: 'time', yFieldName: 'value' }); |
||||
}); |
||||
|
||||
it('Should set the first field as X and the second as Y if there are no time fields', async () => { |
||||
const onChangeMock = jest.fn(); |
||||
|
||||
const df = toDataFrame({ |
||||
name: 'data', |
||||
refId: 'A', |
||||
fields: [ |
||||
{ name: 'not this', type: FieldType.string, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'foo', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'bar', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'not this either', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
], |
||||
}); |
||||
|
||||
render(<RegressionTransformerEditor input={[df]} onChange={onChangeMock} options={{}} />); |
||||
|
||||
expect(onChangeMock).toBeCalledTimes(1); |
||||
expect(onChangeMock).toBeCalledWith({ xFieldName: 'foo', yFieldName: 'bar' }); |
||||
}); |
||||
|
||||
it('should display degree if the model is polynomial', async () => { |
||||
const onChangeMock = jest.fn(); |
||||
|
||||
const df = toDataFrame({ |
||||
name: 'data', |
||||
refId: 'A', |
||||
fields: [ |
||||
{ name: 'foo', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'bar', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
], |
||||
}); |
||||
|
||||
render( |
||||
<RegressionTransformerEditor input={[df]} onChange={onChangeMock} options={{ modelType: ModelType.polynomial }} /> |
||||
); |
||||
|
||||
expect(await screen.findByText('Degree')).toBeInTheDocument(); |
||||
}); |
||||
|
||||
it('should not display degree if the model is linear', async () => { |
||||
const onChangeMock = jest.fn(); |
||||
|
||||
const df = toDataFrame({ |
||||
name: 'data', |
||||
refId: 'A', |
||||
fields: [ |
||||
{ name: 'foo', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
{ name: 'bar', type: FieldType.number, values: [0, 1, 2, 3, 4, 5] }, |
||||
], |
||||
}); |
||||
|
||||
render( |
||||
<RegressionTransformerEditor input={[df]} onChange={onChangeMock} options={{ modelType: ModelType.linear }} /> |
||||
); |
||||
|
||||
expect(await screen.queryByText('Degree')).not.toBeInTheDocument(); |
||||
}); |
||||
}); |
||||
@ -0,0 +1,142 @@ |
||||
import React, { useEffect } from 'react'; |
||||
|
||||
import { |
||||
DataTransformerID, |
||||
TransformerRegistryItem, |
||||
TransformerUIProps, |
||||
TransformerCategory, |
||||
fieldMatchers, |
||||
FieldMatcherID, |
||||
Field, |
||||
} from '@grafana/data'; |
||||
import { InlineField, Select } from '@grafana/ui'; |
||||
import { FieldNamePicker } from '@grafana/ui/src/components/MatchersUI/FieldNamePicker'; |
||||
import { NumberInput } from 'app/core/components/OptionsUI/NumberInput'; |
||||
|
||||
import { getTransformationContent } from '../docs/getTransformationContent'; |
||||
|
||||
import { DEFAULTS, ModelType, RegressionTransformer, RegressionTransformerOptions } from './regression'; |
||||
|
||||
const fieldNamePickerSettings = { |
||||
editor: FieldNamePicker, |
||||
id: '', |
||||
name: '', |
||||
settings: { width: 24, isClearable: false }, |
||||
}; |
||||
|
||||
const LABEL_WIDTH = 20; |
||||
|
||||
export const RegressionTransformerEditor = ({ |
||||
input, |
||||
options, |
||||
onChange, |
||||
}: TransformerUIProps<RegressionTransformerOptions>) => { |
||||
const modelTypeOptions = [ |
||||
{ label: 'Linear', value: ModelType.linear }, |
||||
{ label: 'Polynomial', value: ModelType.polynomial }, |
||||
]; |
||||
|
||||
useEffect(() => { |
||||
let x: Field | undefined; |
||||
let y: Field | undefined; |
||||
if (!options.xFieldName) { |
||||
const timeMatcher = fieldMatchers.get(FieldMatcherID.firstTimeField).get({}); |
||||
for (const frame of input) { |
||||
x = frame.fields.find((field) => timeMatcher(field, frame, input)); |
||||
if (x) { |
||||
break; |
||||
} |
||||
} |
||||
if (!x) { |
||||
const firstMatcher = fieldMatchers.get(FieldMatcherID.numeric).get({}); |
||||
for (const frame of input) { |
||||
x = frame.fields.find((field) => firstMatcher(field, frame, input)); |
||||
if (x) { |
||||
break; |
||||
} |
||||
} |
||||
} |
||||
} |
||||
if (!options.yFieldName) { |
||||
const numberMatcher = fieldMatchers.get(FieldMatcherID.numeric).get({}); |
||||
for (const frame of input) { |
||||
y = frame.fields.find((field) => numberMatcher(field, frame, input) && field !== x); |
||||
if (y) { |
||||
break; |
||||
} |
||||
} |
||||
} |
||||
|
||||
if (x && y) { |
||||
onChange({ ...options, xFieldName: x.name, yFieldName: y.name }); |
||||
} |
||||
}); |
||||
|
||||
return ( |
||||
<> |
||||
<InlineField labelWidth={LABEL_WIDTH} label="X field"> |
||||
<FieldNamePicker |
||||
context={{ data: input }} |
||||
value={options.xFieldName ?? ''} |
||||
item={fieldNamePickerSettings} |
||||
onChange={(v) => { |
||||
onChange({ ...options, xFieldName: v }); |
||||
}} |
||||
></FieldNamePicker> |
||||
</InlineField> |
||||
<InlineField labelWidth={LABEL_WIDTH} label="Y field"> |
||||
<FieldNamePicker |
||||
context={{ data: input }} |
||||
value={options.yFieldName ?? ''} |
||||
item={fieldNamePickerSettings} |
||||
onChange={(v) => { |
||||
onChange({ ...options, yFieldName: v }); |
||||
}} |
||||
></FieldNamePicker> |
||||
</InlineField> |
||||
<InlineField labelWidth={LABEL_WIDTH} label="Model type"> |
||||
<Select |
||||
value={options.modelType ?? DEFAULTS.modelType} |
||||
onChange={(v) => { |
||||
onChange({ ...options, modelType: v.value ?? DEFAULTS.modelType }); |
||||
}} |
||||
options={modelTypeOptions} |
||||
></Select> |
||||
</InlineField> |
||||
<InlineField labelWidth={LABEL_WIDTH} label="Predicted points" tooltip={'Number of X,Y points to predict'}> |
||||
<NumberInput |
||||
value={options.predictionCount ?? DEFAULTS.predictionCount} |
||||
onChange={(v) => { |
||||
onChange({ ...options, predictionCount: v }); |
||||
}} |
||||
></NumberInput> |
||||
</InlineField> |
||||
{options.modelType === ModelType.polynomial && ( |
||||
<InlineField labelWidth={LABEL_WIDTH} label="Degree"> |
||||
<Select<number> |
||||
value={options.degree ?? DEFAULTS.degree} |
||||
options={[ |
||||
{ label: 'Quadratic', value: 2 }, |
||||
{ label: 'Cubic', value: 3 }, |
||||
{ label: 'Quartic', value: 4 }, |
||||
{ label: 'Quintic', value: 5 }, |
||||
]} |
||||
onChange={(v) => { |
||||
onChange({ ...options, degree: v.value }); |
||||
}} |
||||
></Select> |
||||
</InlineField> |
||||
)} |
||||
</> |
||||
); |
||||
}; |
||||
|
||||
export const regressionTransformerRegistryItem: TransformerRegistryItem<RegressionTransformerOptions> = { |
||||
id: DataTransformerID.regression, |
||||
editor: RegressionTransformerEditor, |
||||
transformation: RegressionTransformer, |
||||
name: RegressionTransformer.name, |
||||
description: RegressionTransformer.description, |
||||
categories: new Set([TransformerCategory.CalculateNewFields]), |
||||
help: getTransformationContent(DataTransformerID.regression).helperDocs, |
||||
}; |
||||
|
After Width: | Height: | Size: 3.7 KiB |
|
After Width: | Height: | Size: 3.7 KiB |
Loading…
Reference in new issue