mirror of https://github.com/grafana/grafana
DataFrame: split DataFrameHelper into MutableDataFrame and FieldCache (#18795)
* add appending utility * add appending utility * update comment * rename to mutable * move mutable functions out of DataFrameHelper * move mutable functions out of DataFrameHelper * move mutable functions out of DataFrameHelper * turn DataFrameHelper into FieldCache * guess time from name * graph the numbers * return the timeField, not just the index * just warn on duplicate field names * only use a parser if the input is a string * append init all fields to the same length * typo * only parse string if value is a string * DataFrame: test fixes * Switch to null for missing values * Fixed testspull/18788/head^2
Ryan McKinley
6 years ago
committed by
Torkel Ödegaard
parent
13f55bc5e8
commit
c777301535
@ -1,229 +1,392 @@ |
||||
import { Field, FieldType, DataFrame, Vector, FieldDTO, DataFrameDTO } from '../types/dataFrame'; |
||||
import { Labels, QueryResultMeta } from '../types/data'; |
||||
import { guessFieldTypeForField, guessFieldTypeFromValue } from './processDataFrame'; |
||||
import { ArrayVector } from './vector'; |
||||
import { Labels, QueryResultMeta, KeyValue } from '../types/data'; |
||||
import { guessFieldTypeForField, guessFieldTypeFromValue, toDataFrameDTO } from './processDataFrame'; |
||||
import { ArrayVector, MutableVector, vectorToArray, CircularVector } from './vector'; |
||||
import isArray from 'lodash/isArray'; |
||||
import isString from 'lodash/isString'; |
||||
|
||||
export class DataFrameHelper implements DataFrame { |
||||
refId?: string; |
||||
meta?: QueryResultMeta; |
||||
name?: string; |
||||
fields: Field[]; |
||||
labels?: Labels; |
||||
length = 0; // updated so it is the length of all fields
|
||||
export class FieldCache { |
||||
fields: Field[] = []; |
||||
|
||||
private fieldByName: { [key: string]: Field } = {}; |
||||
private fieldByType: { [key: string]: Field[] } = {}; |
||||
|
||||
constructor(data?: DataFrame | DataFrameDTO) { |
||||
if (!data) { |
||||
data = { fields: [] }; //
|
||||
constructor(private data: DataFrame) { |
||||
this.fields = data.fields; |
||||
|
||||
for (const field of data.fields) { |
||||
// Make sure it has a type
|
||||
if (field.type === FieldType.other) { |
||||
const t = guessFieldTypeForField(field); |
||||
if (t) { |
||||
field.type = t; |
||||
} |
||||
} |
||||
if (!this.fieldByType[field.type]) { |
||||
this.fieldByType[field.type] = []; |
||||
} |
||||
this.fieldByType[field.type].push(field); |
||||
|
||||
if (this.fieldByName[field.name]) { |
||||
console.warn('Duplicate field names in DataFrame: ', field.name); |
||||
} else { |
||||
this.fieldByName[field.name] = field; |
||||
} |
||||
} |
||||
} |
||||
|
||||
getFields(type?: FieldType): Field[] { |
||||
if (!type) { |
||||
return [...this.data.fields]; // All fields
|
||||
} |
||||
this.refId = data.refId; |
||||
this.meta = data.meta; |
||||
this.name = data.name; |
||||
this.labels = data.labels; |
||||
this.fields = []; |
||||
for (let i = 0; i < data.fields.length; i++) { |
||||
this.addField(data.fields[i]); |
||||
const fields = this.fieldByType[type]; |
||||
if (fields) { |
||||
return [...fields]; |
||||
} |
||||
return []; |
||||
} |
||||
|
||||
addFieldFor(value: any, name?: string): Field { |
||||
if (!name) { |
||||
name = `Field ${this.fields.length + 1}`; |
||||
hasFieldOfType(type: FieldType): boolean { |
||||
const types = this.fieldByType[type]; |
||||
return types && types.length > 0; |
||||
} |
||||
|
||||
getFirstFieldOfType(type: FieldType): Field | undefined { |
||||
const arr = this.fieldByType[type]; |
||||
if (arr && arr.length > 0) { |
||||
return arr[0]; |
||||
} |
||||
return this.addField({ |
||||
name, |
||||
type: guessFieldTypeFromValue(value), |
||||
}); |
||||
return undefined; |
||||
} |
||||
|
||||
hasFieldNamed(name: string): boolean { |
||||
return !!this.fieldByName[name]; |
||||
} |
||||
|
||||
/** |
||||
* Reverse the direction of all fields |
||||
* Returns the first field with the given name. |
||||
*/ |
||||
reverse() { |
||||
for (const f of this.fields) { |
||||
f.values.toArray().reverse(); |
||||
getFieldByName(name: string): Field | undefined { |
||||
return this.fieldByName[name]; |
||||
} |
||||
} |
||||
|
||||
function makeFieldParser(value: any, field: Field): (value: string) => any { |
||||
if (!field.type) { |
||||
if (field.name === 'time' || field.name === 'Time') { |
||||
field.type = FieldType.time; |
||||
} else { |
||||
field.type = guessFieldTypeFromValue(value); |
||||
} |
||||
} |
||||
|
||||
private updateTypeIndex(field: Field) { |
||||
// Make sure it has a type
|
||||
if (field.type === FieldType.other) { |
||||
const t = guessFieldTypeForField(field); |
||||
if (t) { |
||||
field.type = t; |
||||
if (field.type === FieldType.number) { |
||||
return (value: string) => { |
||||
return parseFloat(value); |
||||
}; |
||||
} |
||||
|
||||
// Will convert anything that starts with "T" to true
|
||||
if (field.type === FieldType.boolean) { |
||||
return (value: string) => { |
||||
return !(value[0] === 'F' || value[0] === 'f' || value[0] === '0'); |
||||
}; |
||||
} |
||||
|
||||
// Just pass the string back
|
||||
return (value: string) => value; |
||||
} |
||||
|
||||
export type MutableField<T = any> = Field<T, MutableVector<T>>; |
||||
|
||||
type MutableVectorCreator = (buffer?: any[]) => MutableVector; |
||||
|
||||
export const MISSING_VALUE: any = null; |
||||
|
||||
export class MutableDataFrame<T = any> implements DataFrame, MutableVector<T> { |
||||
name?: string; |
||||
labels?: Labels; |
||||
refId?: string; |
||||
meta?: QueryResultMeta; |
||||
|
||||
fields: MutableField[] = []; |
||||
values: KeyValue<MutableVector> = {}; |
||||
|
||||
private first: Vector = new ArrayVector(); |
||||
private creator: MutableVectorCreator; |
||||
|
||||
constructor(source?: DataFrame | DataFrameDTO, creator?: MutableVectorCreator) { |
||||
// This creates the underlying storage buffers
|
||||
this.creator = creator |
||||
? creator |
||||
: (buffer?: any[]) => { |
||||
return new ArrayVector(buffer); |
||||
}; |
||||
|
||||
// Copy values from
|
||||
if (source) { |
||||
const { name, labels, refId, meta, fields } = source; |
||||
if (name) { |
||||
this.name = name; |
||||
} |
||||
if (labels) { |
||||
this.labels = labels; |
||||
} |
||||
if (refId) { |
||||
this.refId = refId; |
||||
} |
||||
if (meta) { |
||||
this.meta = meta; |
||||
} |
||||
if (fields) { |
||||
for (const f of fields) { |
||||
this.addField(f); |
||||
} |
||||
} |
||||
} |
||||
if (!this.fieldByType[field.type]) { |
||||
this.fieldByType[field.type] = []; |
||||
} |
||||
this.fieldByType[field.type].push(field); |
||||
|
||||
// Get Length to show up if you use spread
|
||||
Object.defineProperty(this, 'length', { |
||||
enumerable: true, |
||||
get: () => { |
||||
return this.first.length; |
||||
}, |
||||
}); |
||||
} |
||||
|
||||
// Defined for Vector interface
|
||||
get length() { |
||||
return this.first.length; |
||||
} |
||||
|
||||
addFieldFor(value: any, name?: string): MutableField { |
||||
return this.addField({ |
||||
name: name || '', // Will be filled in
|
||||
type: guessFieldTypeFromValue(value), |
||||
}); |
||||
} |
||||
|
||||
addField(f: Field | FieldDTO): Field { |
||||
const type = f.type || FieldType.other; |
||||
const values = |
||||
!f.values || isArray(f.values) |
||||
? new ArrayVector(f.values as any[] | undefined) // array or empty
|
||||
: (f.values as Vector); |
||||
addField(f: Field | FieldDTO, startLength?: number): MutableField { |
||||
let buffer: any[] | undefined = undefined; |
||||
|
||||
// And a name
|
||||
if (f.values) { |
||||
if (isArray(f.values)) { |
||||
buffer = f.values as any[]; |
||||
} else { |
||||
buffer = (f.values as Vector).toArray(); |
||||
} |
||||
} |
||||
|
||||
let type = f.type; |
||||
|
||||
if (!type && ('time' === f.name || 'Time' === f.name)) { |
||||
type = FieldType.time; |
||||
} else { |
||||
if (!type && buffer && buffer.length) { |
||||
type = guessFieldTypeFromValue(buffer[0]); |
||||
} |
||||
if (!type) { |
||||
type = FieldType.other; |
||||
} |
||||
} |
||||
|
||||
// Make sure it has a name
|
||||
let name = f.name; |
||||
if (!name) { |
||||
if (type === FieldType.time) { |
||||
name = `Time ${this.fields.length + 1}`; |
||||
name = this.values['Time'] ? `Time ${this.fields.length + 1}` : 'Time'; |
||||
} else { |
||||
name = `Column ${this.fields.length + 1}`; |
||||
name = `Field ${this.fields.length + 1}`; |
||||
} |
||||
} |
||||
const field: Field = { |
||||
|
||||
const field: MutableField = { |
||||
name, |
||||
type, |
||||
config: f.config || {}, |
||||
values, |
||||
values: this.creator(buffer), |
||||
}; |
||||
this.updateTypeIndex(field); |
||||
|
||||
if (this.fieldByName[field.name]) { |
||||
console.warn('Duplicate field names in DataFrame: ', field.name); |
||||
} else { |
||||
this.fieldByName[field.name] = field; |
||||
} |
||||
|
||||
// Make sure the lengths all match
|
||||
if (field.values.length !== this.length) { |
||||
if (field.values.length > this.length) { |
||||
// Add `null` to all other values
|
||||
const newlen = field.values.length; |
||||
for (const fx of this.fields) { |
||||
const arr = fx.values as ArrayVector; |
||||
while (fx.values.length !== newlen) { |
||||
arr.buffer.push(null); |
||||
} |
||||
} |
||||
this.length = field.values.length; |
||||
} else { |
||||
const arr = field.values as ArrayVector; |
||||
while (field.values.length !== this.length) { |
||||
arr.buffer.push(null); |
||||
} |
||||
if (type === FieldType.other) { |
||||
type = guessFieldTypeForField(field); |
||||
if (type) { |
||||
field.type = type; |
||||
} |
||||
} |
||||
|
||||
this.fields.push(field); |
||||
this.first = this.fields[0].values; |
||||
|
||||
// The Field Already exists
|
||||
if (this.values[name]) { |
||||
console.warn(`Duplicate field names found: ${name}, only the first will be accessible`); |
||||
} else { |
||||
this.values[name] = field.values; |
||||
} |
||||
|
||||
// Make sure the field starts with a given length
|
||||
if (startLength) { |
||||
while (field.values.length < startLength) { |
||||
field.values.add(MISSING_VALUE); |
||||
} |
||||
} else { |
||||
this.validate(); |
||||
} |
||||
|
||||
return field; |
||||
} |
||||
|
||||
validate() { |
||||
// Make sure all arrays are the same length
|
||||
const length = this.fields.reduce((v: number, f) => { |
||||
return Math.max(v, f.values.length); |
||||
}, 0); |
||||
|
||||
// Add empty elements until everything mastches
|
||||
for (const field of this.fields) { |
||||
while (field.values.length !== length) { |
||||
field.values.add(MISSING_VALUE); |
||||
} |
||||
} |
||||
} |
||||
|
||||
private addMissingFieldsFor(value: any) { |
||||
for (const key of Object.keys(value)) { |
||||
if (!this.values[key]) { |
||||
this.addField({ |
||||
name: key, |
||||
type: guessFieldTypeFromValue(value[key]), |
||||
}); |
||||
} |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Reverse all values |
||||
*/ |
||||
reverse() { |
||||
for (const f of this.fields) { |
||||
f.values.reverse(); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* This will add each value to the corresponding column |
||||
*/ |
||||
appendRow(row: any[]) { |
||||
// Add any extra columns
|
||||
for (let i = this.fields.length; i < row.length; i++) { |
||||
this.addFieldFor(row[i]); |
||||
this.addField({ |
||||
name: `Field ${i + 1}`, |
||||
type: guessFieldTypeFromValue(row[i]), |
||||
}); |
||||
} |
||||
|
||||
// The first line may change the field types
|
||||
if (this.length < 1) { |
||||
this.fieldByType = {}; |
||||
for (let i = 0; i < this.fields.length; i++) { |
||||
const f = this.fields[i]; |
||||
if (!f.type || f.type === FieldType.other) { |
||||
f.type = guessFieldTypeFromValue(row[i]); |
||||
} |
||||
this.updateTypeIndex(f); |
||||
} |
||||
} |
||||
|
||||
for (let i = 0; i < this.fields.length; i++) { |
||||
const f = this.fields[i]; |
||||
let v = row[i]; |
||||
if (!f.parse) { |
||||
f.parse = makeFieldParser(v, f); |
||||
if (f.type !== FieldType.string && isString(v)) { |
||||
if (!f.parse) { |
||||
f.parse = makeFieldParser(v, f); |
||||
} |
||||
v = f.parse(v); |
||||
} |
||||
v = f.parse(v); |
||||
|
||||
const arr = f.values as ArrayVector; |
||||
arr.buffer.push(v); // may be undefined
|
||||
f.values.add(v); |
||||
} |
||||
this.length++; |
||||
} |
||||
|
||||
/** |
||||
* Add any values that match the field names |
||||
* Add all properties of the value as fields on the frame |
||||
*/ |
||||
appendRowFrom(obj: { [key: string]: any }) { |
||||
for (const f of this.fields) { |
||||
const v = obj[f.name]; |
||||
if (!f.parse) { |
||||
f.parse = makeFieldParser(v, f); |
||||
add(value: T, addMissingFields?: boolean) { |
||||
if (addMissingFields) { |
||||
this.addMissingFieldsFor(value); |
||||
} |
||||
|
||||
// Will add one value for every field
|
||||
const obj = value as any; |
||||
for (const field of this.fields) { |
||||
let val = obj[field.name]; |
||||
|
||||
if (field.type !== FieldType.string && isString(val)) { |
||||
if (!field.parse) { |
||||
field.parse = makeFieldParser(val, field); |
||||
} |
||||
val = field.parse(val); |
||||
} |
||||
|
||||
if (val === undefined) { |
||||
val = MISSING_VALUE; |
||||
} |
||||
|
||||
const arr = f.values as ArrayVector; |
||||
arr.buffer.push(f.parse(v)); // may be undefined
|
||||
field.values.add(val); |
||||
} |
||||
this.length++; |
||||
} |
||||
|
||||
getFields(type?: FieldType): Field[] { |
||||
if (!type) { |
||||
return [...this.fields]; // All fields
|
||||
set(index: number, value: T, addMissingFields?: boolean) { |
||||
if (index > this.length) { |
||||
throw new Error('Unable ot set value beyond current length'); |
||||
} |
||||
const fields = this.fieldByType[type]; |
||||
if (fields) { |
||||
return [...fields]; |
||||
|
||||
if (addMissingFields) { |
||||
this.addMissingFieldsFor(value); |
||||
} |
||||
return []; |
||||
} |
||||
|
||||
hasFieldOfType(type: FieldType): boolean { |
||||
const types = this.fieldByType[type]; |
||||
return types && types.length > 0; |
||||
const obj = (value as any) || {}; |
||||
for (const field of this.fields) { |
||||
field.values.set(index, obj[field.name]); |
||||
} |
||||
} |
||||
|
||||
getFirstFieldOfType(type: FieldType): Field | undefined { |
||||
const arr = this.fieldByType[type]; |
||||
if (arr && arr.length > 0) { |
||||
return arr[0]; |
||||
/** |
||||
* Get an object with a property for each field in the DataFrame |
||||
*/ |
||||
get(idx: number): T { |
||||
const v: any = {}; |
||||
for (const field of this.fields) { |
||||
v[field.name] = field.values.get(idx); |
||||
} |
||||
return undefined; |
||||
return v as T; |
||||
} |
||||
|
||||
hasFieldNamed(name: string): boolean { |
||||
return !!this.fieldByName[name]; |
||||
toArray(): T[] { |
||||
return vectorToArray(this); |
||||
} |
||||
|
||||
/** |
||||
* Returns the first field with the given name. |
||||
* The simplified JSON values used in JSON.stringify() |
||||
*/ |
||||
getFieldByName(name: string): Field | undefined { |
||||
return this.fieldByName[name]; |
||||
toJSON() { |
||||
return toDataFrameDTO(this); |
||||
} |
||||
} |
||||
|
||||
function makeFieldParser(value: string, field: Field): (value: string) => any { |
||||
if (!field.type) { |
||||
if (field.name === 'time' || field.name === 'Time') { |
||||
field.type = FieldType.time; |
||||
} else { |
||||
field.type = guessFieldTypeFromValue(value); |
||||
} |
||||
} |
||||
|
||||
if (field.type === FieldType.number) { |
||||
return (value: string) => { |
||||
return parseFloat(value); |
||||
}; |
||||
} |
||||
interface CircularOptions { |
||||
append?: 'head' | 'tail'; |
||||
capacity?: number; |
||||
} |
||||
|
||||
// Will convert anything that starts with "T" to true
|
||||
if (field.type === FieldType.boolean) { |
||||
return (value: string) => { |
||||
return !(value[0] === 'F' || value[0] === 'f' || value[0] === '0'); |
||||
}; |
||||
/** |
||||
* This dataframe can have values constantly added, and will never |
||||
* exceed the given capacity |
||||
*/ |
||||
export class CircularDataFrame<T = any> extends MutableDataFrame<T> { |
||||
constructor(options: CircularOptions) { |
||||
super(undefined, (buffer?: any[]) => { |
||||
return new CircularVector({ |
||||
buffer, |
||||
...options, |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
// Just pass the string back
|
||||
return (value: string) => value; |
||||
} |
||||
|
||||
Loading…
Reference in new issue