diff --git a/public/app/core/utils/sheet.test.ts b/public/app/core/utils/sheet.test.ts new file mode 100644 index 00000000000..cb8b46ecfeb --- /dev/null +++ b/public/app/core/utils/sheet.test.ts @@ -0,0 +1,183 @@ +import { utils } from 'xlsx'; + +import { DataFrame } from '@grafana/data'; + +import { workSheetToFrame } from './sheet'; + +describe('sheets', () => { + it('will use first row as names', () => { + const sheet = utils.aoa_to_sheet([ + ['Number', 'String', 'Bool', 'Date', 'Object'], + [1, 'A', true, Date.UTC(2020, 1, 1), { hello: 'world' }], + [2, 'B', false, Date.UTC(2020, 1, 2), { hello: 'world' }], + ]); + const frame = workSheetToFrame(sheet); + + expect(toSnapshotFrame(frame)).toMatchInlineSnapshot(` + [ + { + "name": "Number", + "type": "number", + "values": [ + 1, + 2, + ], + }, + { + "name": "String", + "type": "string", + "values": [ + "A", + "B", + ], + }, + { + "name": "Bool", + "type": "boolean", + "values": [ + true, + false, + ], + }, + { + "name": "Date", + "type": "number", + "values": [ + 1580515200000, + 1580601600000, + ], + }, + { + "name": "Object", + "type": "string", + "values": [ + undefined, + undefined, + ], + }, + ] + `); + }); + + it('will use calculated data when cells are typed', () => { + const sheet = utils.aoa_to_sheet([ + [1, 'A', true, Date.UTC(2020, 1, 1), { hello: 'world' }], + [2, 'B', false, Date.UTC(2020, 1, 2), { hello: 'world' }], + [3, 'C', true, Date.UTC(2020, 1, 3), { hello: 'world' }], + ]); + const frame = workSheetToFrame(sheet); + + expect(toSnapshotFrame(frame)).toMatchInlineSnapshot(` + [ + { + "name": "A", + "type": "number", + "values": [ + 1, + 2, + 3, + ], + }, + { + "name": "B", + "type": "string", + "values": [ + "A", + "B", + "C", + ], + }, + { + "name": "C", + "type": "boolean", + "values": [ + true, + false, + true, + ], + }, + { + "name": "D", + "type": "number", + "values": [ + 1580515200000, + 1580601600000, + 1580688000000, + ], + }, + { + "name": "E", + "type": "string", + "values": [ + undefined, + undefined, + undefined, + ], + }, + ] + `); + }); + + it('is OK with nulls and undefineds, and misalignment', () => { + const sheet = utils.aoa_to_sheet([ + [null, 'A', true], + [2, 'B', null, Date.UTC(2020, 1, 2), { hello: 'world' }], + [3, 'C', true, undefined, { hello: 'world' }], + ]); + const frame = workSheetToFrame(sheet); + + expect(toSnapshotFrame(frame)).toMatchInlineSnapshot(` + [ + { + "name": "A", + "type": "number", + "values": [ + undefined, + 2, + 3, + ], + }, + { + "name": "B", + "type": "string", + "values": [ + "A", + "B", + "C", + ], + }, + { + "name": "C", + "type": "boolean", + "values": [ + true, + undefined, + true, + ], + }, + { + "name": "D", + "type": "number", + "values": [ + undefined, + 1580601600000, + undefined, + ], + }, + { + "name": "E", + "type": "string", + "values": [ + undefined, + undefined, + undefined, + ], + }, + ] + `); + }); +}); + +function toSnapshotFrame(frame: DataFrame) { + return frame.fields.map((f) => ({ name: f.name, values: f.values, type: f.type })); +} diff --git a/public/app/core/utils/sheet.ts b/public/app/core/utils/sheet.ts index 5771b772283..a97d31c2b33 100644 --- a/public/app/core/utils/sheet.ts +++ b/public/app/core/utils/sheet.ts @@ -1,12 +1,172 @@ -import { read, utils } from 'xlsx'; +import { read, utils, WorkSheet, WorkBook, Range, ColInfo, CellObject, ExcelDataType } from 'xlsx'; -import { ArrayDataFrame, DataFrame } from '@grafana/data'; +import { DataFrame, FieldType } from '@grafana/data'; export function readSpreadsheet(file: ArrayBuffer): DataFrame[] { - const wb = read(file, { type: 'buffer' }); - return wb.SheetNames.map((name) => { - const frame = new ArrayDataFrame(utils.sheet_to_json(wb.Sheets[name])); - frame.name = name; - return frame; - }); + return workBookToFrames(read(file, { type: 'buffer' })); +} + +export function workBookToFrames(wb: WorkBook): DataFrame[] { + return wb.SheetNames.map((name) => workSheetToFrame(wb.Sheets[name], name)); +} + +export function workSheetToFrame(sheet: WorkSheet, name?: string): DataFrame { + const columns = sheetAsColumns(sheet); + if (!columns?.length) { + return { + fields: [], + name: name, + length: 0, + }; + } + + return { + fields: columns.map((c, idx) => { + let type = FieldType.string; + let values: unknown[] = []; + switch (c.type ?? 's') { + case 'b': + type = FieldType.boolean; + values = c.data.map((v) => (v?.v == null ? v?.v : Boolean(v.v))); + break; + + case 'n': + type = FieldType.number; + values = c.data.map((v) => (v?.v == null ? v?.v : +v.v)); + break; + + case 'd': + type = FieldType.time; + values = c.data.map((v) => (v?.v == null ? v?.v : +v.v)); // ??? + break; + + default: + type = FieldType.string; + values = c.data.map((v) => (v?.v == null ? v?.v : utils.format_cell(v))); + break; + } + + return { + name: c.name, + config: {}, // TODO? we could apply decimal formatting from worksheet + type, + values, + }; + }), + name: name, + length: columns[0].data.length, + }; +} + +interface ColumnData { + index: number; + name: string; + info?: ColInfo; + data: CellObject[]; + type?: ExcelDataType; +} + +function sheetAsColumns(sheet: WorkSheet): ColumnData[] | null { + const r = sheet['!ref']; + if (!r) { + return null; + } + const columnInfo = sheet['!cols']; + const cols: ColumnData[] = []; + const range = safe_decode_range(r); + const types = new Set(); + let firstRowIsHeader = true; + + for (let c = range.s.c; c <= range.e.c; ++c) { + types.clear(); + const info = columnInfo?.[c] ?? {}; + if (info.hidden) { + continue; // skip the column + } + const field: ColumnData = { + index: c, + name: utils.encode_col(c), + data: [], + info, + }; + const pfix = utils.encode_col(c); + for (let r = range.s.r; r <= range.e.r; ++r) { + const cell = sheet[pfix + utils.encode_row(r)]; + if (cell) { + if (field.data.length) { + types.add(cell.t); + } else if (cell.t !== 's') { + firstRowIsHeader = false; + } + } + field.data.push(cell); + } + cols.push(field); + if (types.size === 1) { + field.type = Array.from(types)[0]; + } + } + + if (firstRowIsHeader) { + return cols.map((c) => { + const first = c.data[0]; + if (first?.v) { + c.name = utils.format_cell(first); + } + c.data = c.data.slice(1); + return c; + }); + } + return cols; +} + +/** + * Copied from Apache 2 licensed sheetjs: + * https://git.sheetjs.com/sheetjs/sheetjs/src/branch/master/xlsx.flow.js#L4338 + */ +function safe_decode_range(range: string): Range { + let o = { s: { c: 0, r: 0 }, e: { c: 0, r: 0 } }; + let idx = 0, + i = 0, + cc = 0; + let len = range.length; + for (idx = 0; i < len; ++i) { + if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) { + break; + } + idx = 26 * idx + cc; + } + o.s.c = --idx; + + for (idx = 0; i < len; ++i) { + if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) { + break; + } + idx = 10 * idx + cc; + } + o.s.r = --idx; + + if (i === len || cc !== 10) { + o.e.c = o.s.c; + o.e.r = o.s.r; + return o; + } + ++i; + + for (idx = 0; i !== len; ++i) { + if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) { + break; + } + idx = 26 * idx + cc; + } + o.e.c = --idx; + + for (idx = 0; i !== len; ++i) { + if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) { + break; + } + idx = 10 * idx + cc; + } + o.e.r = --idx; + return o; }