Transformations: Convert field types to time string number or boolean (#38517)

* outline string to time

add stringToTime transformer

start to add format

add type and dateformat

rename stringToTime to fieldConversion

add more type support and use FieldNamePicker

add field conversion transformation

* adjust for performance feedback

rename and adjust labels and widths

shorten labels and null values

rename to convertFieldType

update test

* make updates

packages/grafana-data/src/field/overrides/processors.ts

@@ -182,4 +182,14 @@ export interface FieldNamePickerConfigSettings {
    * information, including validation etc
    */
   info?: ComponentType<FieldNamePickerInfoProps> | null;
+
+  /**
+   * Sets the width to a pixel value.
+   */
+  width?: number;
+
+  /**
+   * Placeholder text to display when nothing is selected.
+   */
+  placeholderText?: string;
 }

packages/grafana-data/src/transformations/index.ts

@@ -13,3 +13,4 @@ export { RegexpOrNamesMatcherOptions, ByNamesMatcherOptions, ByNamesMatcherMode
 export { RenameByRegexTransformerOptions } from './transformers/renameByRegex';
 export { outerJoinDataFrames } from './transformers/joinDataFrames';
 export * from './transformers/histogram';
+export { ensureTimeField } from './transformers/convertFieldType';

packages/grafana-data/src/transformations/transformers.ts

@@ -18,6 +18,7 @@ import { mergeTransformer } from './transformers/merge';
 import { renameByRegexTransformer } from './transformers/renameByRegex';
 import { filterByValueTransformer } from './transformers/filterByValue';
 import { histogramTransformer } from './transformers/histogram';
+import { convertFieldTypeTransformer } from './transformers/convertFieldType';
 
 export const standardTransformers = {
   noopTransformer,
@@ -41,4 +42,5 @@ export const standardTransformers = {
   mergeTransformer,
   renameByRegexTransformer,
   histogramTransformer,
+  convertFieldTypeTransformer,
 };

packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts

@@ -0,0 +1,235 @@
+import { toDataFrame } from '../../dataframe/processDataFrame';
+import { FieldType } from '../../types/dataFrame';
+import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
+import { ArrayVector } from '../../vector';
+import { ensureTimeField, convertFieldType, convertFieldTypes, convertFieldTypeTransformer } from './convertFieldType';
+
+describe('field convert type', () => {
+  it('will parse properly formatted strings to time', () => {
+    const options = { targetField: 'proper dates', destinationType: FieldType.time };
+
+    const stringTime = {
+      name: 'proper dates',
+      type: FieldType.string,
+      values: new ArrayVector([
+        '2021-07-19 00:00:00.000',
+        '2021-07-23 00:00:00.000',
+        '2021-07-25 00:00:00.000',
+        '2021-08-01 00:00:00.000',
+        '2021-08-02 00:00:00.000',
+      ]),
+      config: {},
+    };
+
+    const timefield = convertFieldType(stringTime, options);
+    expect(timefield).toEqual({
+      name: 'proper dates',
+      type: FieldType.time,
+      values: new ArrayVector([1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000]),
+      config: {},
+    });
+  });
+  it('will parse string time to specified format in time', () => {
+    const options = { targetField: 'format to year', destinationType: FieldType.time, dateFormat: 'YYYY' };
+
+    const yearFormat = {
+      name: 'format to year',
+      type: FieldType.string,
+      values: new ArrayVector([
+        '2017-07-19 00:00:00.000',
+        '2018-07-23 00:00:00.000',
+        '2019-07-25 00:00:00.000',
+        '2020-08-01 00:00:00.000',
+        '2021-08-02 00:00:00.000',
+      ]),
+      config: {},
+    };
+
+    const timefield = convertFieldType(yearFormat, options);
+    expect(timefield).toEqual({
+      name: 'format to year',
+      type: FieldType.time,
+      values: new ArrayVector([1483246800000, 1514782800000, 1546318800000, 1577854800000, 1609477200000]),
+      config: {},
+    });
+  });
+
+  it('will not parse improperly formatted date strings', () => {
+    const options = { targetField: 'misformatted dates', destinationType: FieldType.time };
+
+    const misformattedStrings = {
+      name: 'misformatted dates',
+      type: FieldType.string,
+      values: new ArrayVector(['2021/08-01 00:00.00:000', '2021/08/01 00.00-000', '2021/08-01 00:00.00:000']),
+      config: { unit: 'time' },
+    };
+
+    const timefield = convertFieldType(misformattedStrings, options);
+    expect(timefield).toEqual({
+      name: 'misformatted dates',
+      type: FieldType.time,
+      values: new ArrayVector([null, null, null]),
+      config: { unit: 'time' },
+    });
+  });
+
+  it('can convert strings to numbers', () => {
+    const options = { targetField: 'stringy nums', destinationType: FieldType.number };
+
+    const stringyNumbers = {
+      name: 'stringy nums',
+      type: FieldType.string,
+      values: new ArrayVector(['10', '12', '30', '14', '10']),
+      config: {},
+    };
+
+    const numbers = convertFieldType(stringyNumbers, options);
+
+    expect(numbers).toEqual({
+      name: 'stringy nums',
+      type: FieldType.number,
+      values: new ArrayVector([10, 12, 30, 14, 10]),
+      config: {},
+    });
+  });
+});
+
+describe('field convert types transformer', () => {
+  beforeAll(() => {
+    mockTransformationsRegistry([convertFieldTypeTransformer]);
+  });
+  it('can convert multiple fields', () => {
+    const options = {
+      conversions: [
+        { targetField: 'stringy nums', destinationType: FieldType.number },
+        { targetField: 'proper dates', destinationType: FieldType.time },
+      ],
+    };
+
+    const stringyNumbers = toDataFrame({
+      fields: [
+        { name: 'A', type: FieldType.number, values: [1, 2, 3, 4, 5] },
+        {
+          name: 'proper dates',
+          type: FieldType.string,
+          values: [
+            '2021-07-19 00:00:00.000',
+            '2021-07-23 00:00:00.000',
+            '2021-07-25 00:00:00.000',
+            '2021-08-01 00:00:00.000',
+            '2021-08-02 00:00:00.000',
+          ],
+        },
+        { name: 'stringy nums', type: FieldType.string, values: ['10', '12', '30', '14', '10'] },
+      ],
+    });
+
+    const numbers = convertFieldTypes(options, [stringyNumbers]);
+    expect(
+      numbers[0].fields.map((f) => ({
+        type: f.type,
+        values: f.values.toArray(),
+      }))
+    ).toEqual([
+      { type: FieldType.number, values: [1, 2, 3, 4, 5] },
+      {
+        type: FieldType.time,
+        values: [1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000],
+      },
+      {
+        type: FieldType.number,
+        values: [10, 12, 30, 14, 10],
+      },
+    ]);
+  });
+
+  it('will convert field to booleans', () => {
+    const options = {
+      conversions: [
+        { targetField: 'numbers', destinationType: FieldType.boolean },
+        { targetField: 'strings', destinationType: FieldType.boolean },
+      ],
+    };
+
+    const comboTypes = toDataFrame({
+      fields: [
+        { name: 'numbers', type: FieldType.number, values: [-100, 0, 1, null, NaN] },
+        {
+          name: 'strings',
+          type: FieldType.string,
+          values: ['true', 'false', '0', '99', '2021-08-02 00:00:00.000'],
+        },
+      ],
+    });
+
+    const booleans = convertFieldTypes(options, [comboTypes]);
+    expect(
+      booleans[0].fields.map((f) => ({
+        type: f.type,
+        values: f.values.toArray(),
+      }))
+    ).toEqual([
+      {
+        type: FieldType.boolean,
+        values: [true, false, true, false, false],
+      },
+      { type: FieldType.boolean, values: [true, true, true, true, true] },
+    ]);
+  });
+
+  it('will convert field to strings', () => {
+    const options = {
+      conversions: [{ targetField: 'numbers', destinationType: FieldType.string }],
+    };
+
+    const comboTypes = toDataFrame({
+      fields: [
+        { name: 'numbers', type: FieldType.number, values: [-100, 0, 1, null, NaN] },
+        {
+          name: 'strings',
+          type: FieldType.string,
+          values: ['true', 'false', '0', '99', '2021-08-02 00:00:00.000'],
+        },
+      ],
+    });
+
+    const stringified = convertFieldTypes(options, [comboTypes]);
+    expect(
+      stringified[0].fields.map((f) => ({
+        type: f.type,
+        values: f.values.toArray(),
+      }))
+    ).toEqual([
+      {
+        type: FieldType.string,
+        values: ['-100', '0', '1', 'null', 'NaN'],
+      },
+      {
+        type: FieldType.string,
+        values: ['true', 'false', '0', '99', '2021-08-02 00:00:00.000'],
+      },
+    ]);
+  });
+});
+
+describe('ensureTimeField', () => {
+  it('will make the field have a type of time if already a number', () => {
+    const stringTime = toDataFrame({
+      fields: [
+        {
+          name: 'proper dates',
+          type: FieldType.number,
+          values: [1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000],
+        },
+        { name: 'A', type: FieldType.number, values: [1, 2, 3, 4, 5] },
+      ],
+    });
+
+    expect(ensureTimeField(stringTime.fields[0])).toEqual({
+      config: {},
+      name: 'proper dates',
+      type: FieldType.time,
+      values: new ArrayVector([1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000]),
+    });
+  });
+});

packages/grafana-data/src/transformations/transformers/convertFieldType.ts

@@ -0,0 +1,170 @@
+import { SynchronousDataTransformerInfo } from '../../types';
+import { map } from 'rxjs/operators';
+
+import { DataTransformerID } from './ids';
+import { DataFrame, Field, FieldType } from '../../types/dataFrame';
+import { dateTimeParse } from '../../datetime';
+import { ArrayVector } from '../../vector';
+
+export interface ConvertFieldTypeTransformerOptions {
+  conversions: ConvertFieldTypeOptions[];
+}
+
+export interface ConvertFieldTypeOptions {
+  targetField?: string;
+  destinationType?: FieldType;
+  dateFormat?: string;
+}
+
+/**
+ * @alpha
+ */
+export const convertFieldTypeTransformer: SynchronousDataTransformerInfo<ConvertFieldTypeTransformerOptions> = {
+  id: DataTransformerID.convertFieldType,
+  name: 'Convert field type',
+  description: 'Convert a field to a specified field type',
+  defaultOptions: {
+    fields: {},
+    conversions: [{ targetField: undefined, destinationType: undefined, dateFormat: undefined }],
+  },
+
+  operator: (options) => (source) => source.pipe(map((data) => convertFieldTypeTransformer.transformer(options)(data))),
+
+  transformer: (options: ConvertFieldTypeTransformerOptions) => (data: DataFrame[]) => {
+    if (!Array.isArray(data) || data.length === 0) {
+      return data;
+    }
+    const timeParsed = convertFieldTypes(options, data);
+    if (!timeParsed) {
+      return [];
+    }
+    return timeParsed;
+  },
+};
+
+/**
+ * @alpha
+ */
+export function convertFieldTypes(options: ConvertFieldTypeTransformerOptions, frames: DataFrame[]): DataFrame[] {
+  if (!options.conversions.length) {
+    return frames;
+  }
+
+  const frameCopy: DataFrame[] = [];
+
+  frames.forEach((frame) => {
+    for (let fieldIdx = 0; fieldIdx < frame.fields.length; fieldIdx++) {
+      let field = frame.fields[fieldIdx];
+      for (let cIdx = 0; cIdx < options.conversions.length; cIdx++) {
+        if (field.name === options.conversions[cIdx].targetField) {
+          //check in about matchers with Ryan
+          const conversion = options.conversions[cIdx];
+          frame.fields[fieldIdx] = convertFieldType(field, conversion);
+          break;
+        }
+      }
+    }
+    frameCopy.push(frame);
+  });
+  return frameCopy;
+}
+
+export function convertFieldType(field: Field, opts: ConvertFieldTypeOptions): Field {
+  switch (opts.destinationType) {
+    case FieldType.time:
+      return ensureTimeField(field, opts.dateFormat);
+    case FieldType.number:
+      return fieldToNumberField(field);
+    case FieldType.string:
+      return fieldToStringField(field);
+    case FieldType.boolean:
+      return fieldToBooleanField(field);
+    default:
+      return field;
+  }
+}
+
+export function fieldToTimeField(field: Field, dateFormat?: string): Field {
+  let opts = dateFormat ? { format: dateFormat } : undefined;
+
+  const timeValues = field.values.toArray().slice();
+
+  for (let t = 0; t < timeValues.length; t++) {
+    if (timeValues[t]) {
+      let parsed = dateTimeParse(timeValues[t], opts).valueOf();
+      timeValues[t] = Number.isFinite(parsed) ? parsed : null;
+    } else {
+      timeValues[t] = null;
+    }
+  }
+
+  return {
+    ...field,
+    type: FieldType.time,
+    values: new ArrayVector(timeValues),
+  };
+}
+
+function fieldToNumberField(field: Field): Field {
+  const numValues = field.values.toArray().slice();
+
+  for (let n = 0; n < numValues.length; n++) {
+    if (numValues[n]) {
+      let number = +numValues[n];
+      numValues[n] = Number.isFinite(number) ? number : null;
+    } else {
+      numValues[n] = null;
+    }
+  }
+
+  return {
+    ...field,
+    type: FieldType.number,
+    values: new ArrayVector(numValues),
+  };
+}
+
+function fieldToBooleanField(field: Field): Field {
+  const booleanValues = field.values.toArray().slice();
+
+  for (let b = 0; b < booleanValues.length; b++) {
+    booleanValues[b] = Boolean(booleanValues[b]);
+  }
+
+  return {
+    ...field,
+    type: FieldType.boolean,
+    values: new ArrayVector(booleanValues),
+  };
+}
+
+function fieldToStringField(field: Field): Field {
+  const stringValues = field.values.toArray().slice();
+
+  for (let s = 0; s < stringValues.length; s++) {
+    stringValues[s] = `${stringValues[s]}`;
+  }
+
+  return {
+    ...field,
+    type: FieldType.string,
+    values: new ArrayVector(stringValues),
+  };
+}
+
+/**
+ * @alpha
+ */
+export function ensureTimeField(field: Field, dateFormat?: string): Field {
+  const firstValueTypeIsNumber = typeof field.values.get(0) === 'number';
+  if (field.type === FieldType.time && firstValueTypeIsNumber) {
+    return field; //already time
+  }
+  if (firstValueTypeIsNumber) {
+    return {
+      ...field,
+      type: FieldType.time, //assumes it should be time
+    };
+  }
+  return fieldToTimeField(field, dateFormat);
+}

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -26,4 +26,5 @@ export enum DataTransformerID {
   configFromData = 'configFromData',
   rowsToFields = 'rowsToFields',
   prepareTimeSeries = 'prepareTimeSeries',
+  convertFieldType = 'convertFieldType',
 }

packages/grafana-ui/src/components/MatchersUI/FieldNamePicker.tsx

@@ -30,9 +30,11 @@ export const FieldNamePicker: React.FC<StandardEditorProps<string, FieldNamePick
       <Select
         menuShouldPortal
         value={selectedOption}
+        placeholder={settings.placeholderText ?? 'Select field'}
         options={selectOptions}
         onChange={onSelectChange}
         noOptionsMessage={settings.noFieldsMessage}
+        width={settings.width}
       />
       {settings.info && <settings.info name={value} field={names.fields.get(value)} />}
     </>

packages/grafana-ui/src/components/uPlot/utils.ts

@@ -1,4 +1,4 @@
-import { DataFrame, dateTime, Field, FieldType } from '@grafana/data';
+import { DataFrame, ensureTimeField, Field, FieldType } from '@grafana/data';
 import { StackingMode } from '@grafana/schema';
 import { createLogger } from '../../utils/logger';
 import { attachDebugger } from '../../utils';
@@ -48,16 +48,7 @@ export function preparePlotData(frame: DataFrame, onStackMeta?: (meta: StackMeta
     const f = frame.fields[i];
 
     if (f.type === FieldType.time) {
-      if (f.values.length > 0 && typeof f.values.get(0) === 'string') {
+      result.push(ensureTimeField(f).values.toArray());
-        const timestamps = [];
-        for (let i = 0; i < f.values.length; i++) {
-          timestamps.push(dateTime(f.values.get(i)).valueOf());
-        }
-        result.push(timestamps);
-        seriesIndex++;
-        continue;
-      }
-      result.push(f.values.toArray());
       seriesIndex++;
       continue;
     }

public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx

@@ -0,0 +1,149 @@
+import React, { useCallback } from 'react';
+import {
+  DataTransformerID,
+  FieldNamePickerConfigSettings,
+  FieldType,
+  SelectableValue,
+  StandardEditorsRegistryItem,
+  standardTransformers,
+  TransformerRegistryItem,
+  TransformerUIProps,
+} from '@grafana/data';
+
+import { ConvertFieldTypeTransformerOptions } from '@grafana/data/src/transformations/transformers/convertFieldType';
+import { Button, InlineField, InlineFieldRow, Input, Select } from '@grafana/ui';
+import { FieldNamePicker } from '../../../../../packages/grafana-ui/src/components/MatchersUI/FieldNamePicker';
+import { ConvertFieldTypeOptions } from '../../../../../packages/grafana-data/src/transformations/transformers/convertFieldType';
+
+const fieldNamePickerSettings: StandardEditorsRegistryItem<string, FieldNamePickerConfigSettings> = {
+  settings: { width: 24 },
+} as any;
+
+export const ConvertFieldTypeTransformerEditor: React.FC<TransformerUIProps<ConvertFieldTypeTransformerOptions>> = ({
+  input,
+  options,
+  onChange,
+}) => {
+  const allTypes: Array<SelectableValue<FieldType>> = [
+    { value: FieldType.number, label: 'Numeric' },
+    { value: FieldType.string, label: 'String' },
+    { value: FieldType.time, label: 'Time' },
+    { value: FieldType.boolean, label: 'Boolean' },
+  ];
+
+  const onSelectField = useCallback(
+    (idx) => (value: string | undefined) => {
+      const conversions = options.conversions;
+      conversions[idx] = { ...conversions[idx], targetField: value ?? '' };
+      onChange({
+        ...options,
+        conversions: conversions,
+      });
+    },
+    [onChange, options]
+  );
+
+  const onSelectDestinationType = useCallback(
+    (idx) => (value: SelectableValue<FieldType>) => {
+      const conversions = options.conversions;
+      conversions[idx] = { ...conversions[idx], destinationType: value.value };
+      onChange({
+        ...options,
+        conversions: conversions,
+      });
+    },
+    [onChange, options]
+  );
+
+  const onInputFormat = useCallback(
+    (idx) => (value: SelectableValue<string>) => {
+      const conversions = options.conversions;
+      conversions[idx] = { ...conversions[idx], dateFormat: value.value };
+      onChange({
+        ...options,
+        conversions: conversions,
+      });
+    },
+    [onChange, options]
+  );
+
+  const onAddConvertFieldType = useCallback(() => {
+    onChange({
+      ...options,
+      conversions: [
+        ...options.conversions,
+        { targetField: undefined, destinationType: undefined, dateFormat: undefined },
+      ],
+    });
+  }, [onChange, options]);
+
+  const onRemoveConvertFieldType = useCallback(
+    (idx) => {
+      const removed = options.conversions;
+      removed.splice(idx, 1);
+      onChange({
+        ...options,
+        conversions: removed,
+      });
+    },
+    [onChange, options]
+  );
+
+  return (
+    <>
+      {options.conversions.map((c: ConvertFieldTypeOptions, idx: number) => {
+        return (
+          <InlineFieldRow key={`${c.targetField}-${idx}`}>
+            <InlineField label={'Field'}>
+              <FieldNamePicker
+                context={{ data: input }}
+                value={c.targetField ?? ''}
+                onChange={onSelectField(idx)}
+                item={fieldNamePickerSettings}
+              />
+            </InlineField>
+            <InlineField label={'as'}>
+              <Select
+                menuShouldPortal
+                options={allTypes}
+                value={c.destinationType}
+                placeholder={'Type'}
+                onChange={onSelectDestinationType(idx)}
+                width={18}
+              />
+            </InlineField>
+            {c.destinationType === FieldType.time && (
+              <InlineField label={'Date Format'}>
+                <Input value={c.dateFormat} placeholder={'e.g. YYYY-MM-DD'} onChange={onInputFormat(idx)} width={24} />
+              </InlineField>
+            )}
+            <Button
+              size="md"
+              icon="trash-alt"
+              variant="secondary"
+              onClick={() => onRemoveConvertFieldType(idx)}
+              aria-label={'Remove convert field type transformer'}
+            />
+          </InlineFieldRow>
+        );
+      })}
+      <Button
+        size="sm"
+        icon="plus"
+        onClick={onAddConvertFieldType}
+        variant="secondary"
+        aria-label={'Add a convert field type transformer'}
+      >
+        {'Convert field type'}
+      </Button>
+    </>
+  );
+};
+
+export const convertFieldTypeTransformRegistryItem: TransformerRegistryItem<ConvertFieldTypeTransformerOptions> = {
+  id: DataTransformerID.convertFieldType,
+  editor: ConvertFieldTypeTransformerEditor,
+  transformation: standardTransformers.convertFieldTypeTransformer,
+  name: standardTransformers.convertFieldTypeTransformer.name,
+  description: standardTransformers.convertFieldTypeTransformer.description,
+};

public/app/core/utils/standardTransformers.ts

@@ -17,6 +17,7 @@ import { histogramTransformRegistryItem } from '../components/TransformersUI/His
 import { rowsToFieldsTransformRegistryItem } from '../components/TransformersUI/rowsToFields/RowsToFieldsTransformerEditor';
 import { configFromQueryTransformRegistryItem } from '../components/TransformersUI/configFromQuery/ConfigFromQueryTransformerEditor';
 import { prepareTimeseriesTransformerRegistryItem } from '../components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor';
+import { convertFieldTypeTransformRegistryItem } from '../components/TransformersUI/ConvertFieldTypeTransformerEditor';
 
 export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> => {
   return [
@@ -38,5 +39,6 @@ export const getStandardTransformers = (): Array<TransformerRegistryItem<any>> =
     rowsToFieldsTransformRegistryItem,
     configFromQueryTransformRegistryItem,
     prepareTimeseriesTransformerRegistryItem,
+    convertFieldTypeTransformRegistryItem,
   ];
 };