From faa29db241c1d52c016b23201225e92d2c4bb9ef Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:01:28 -0600 Subject: [PATCH] Logs Panel: Table UI - Pull logs table into dashboard panel (#77757) * Allows users to add a logs table in explore to a dashboard panel via the includeByName transformation --- .../loki-table-explore-to-dash.spec.ts | 214 ++++++++++++++++++ packages/grafana-data/src/types/explore.ts | 1 + public/app/features/explore/Logs/Logs.tsx | 13 +- .../explore/Logs/LogsTableNavColumn.tsx | 12 +- .../explore/Logs/LogsTableWrap.test.tsx | 16 +- .../features/explore/Logs/LogsTableWrap.tsx | 38 +++- .../AddToDashboard/AddToDashboardForm.tsx | 1 + .../AddToDashboard/addToDashboard.ts | 50 +++- public/app/features/logs/legacyLogsFrame.ts | 1 + public/app/features/logs/logsFrame.ts | 2 + 10 files changed, 328 insertions(+), 20 deletions(-) create mode 100644 e2e/various-suite/loki-table-explore-to-dash.spec.ts diff --git a/e2e/various-suite/loki-table-explore-to-dash.spec.ts b/e2e/various-suite/loki-table-explore-to-dash.spec.ts new file mode 100644 index 00000000000..5a6bb85dade --- /dev/null +++ b/e2e/various-suite/loki-table-explore-to-dash.spec.ts @@ -0,0 +1,214 @@ +import { e2e } from '../utils'; + +const dataSourceName = 'LokiEditor'; +const addDataSource = () => { + e2e.flows.addDataSource({ + type: 'Loki', + expectedAlertMessage: 'Unable to connect with Loki. Please check the server logs for more details.', + name: dataSourceName, + form: () => { + cy.get('#connection-url').type('http://loki-url:3100'); + }, + }); +}; + +const lokiQueryResult = { + status: 'success', + results: { + A: { + status: 200, + frames: [ + { + schema: { + refId: 'A', + meta: { + typeVersion: [0, 0], + custom: { + frameType: 'LabeledTimeValues', + }, + stats: [ + { + displayName: 'Summary: bytes processed per second', + unit: 'Bps', + value: 223921, + }, + { + displayName: 'Summary: total bytes processed', + unit: 'decbytes', + value: 4156, + }, + { + displayName: 'Summary: exec time', + unit: 's', + value: 0.01856, + }, + ], + executedQueryString: 'Expr: {targetLabelName="targetLabelValue"}', + }, + fields: [ + { + name: 'labels', + type: 'other', + typeInfo: { + frame: 'json.RawMessage', + }, + }, + { + name: 'Time', + type: 'time', + typeInfo: { + frame: 'time.Time', + }, + }, + { + name: 'Line', + type: 'string', + typeInfo: { + frame: 'string', + }, + }, + { + name: 'tsNs', + type: 'string', + typeInfo: { + frame: 'string', + }, + }, + { + name: 'id', + type: 'string', + typeInfo: { + frame: 'string', + }, + }, + ], + }, + data: { + values: [ + [ + { + targetLabelName: 'targetLabelValue', + instance: 'server\\1', + job: '"grafana/data"', + nonIndexed: 'value', + place: 'moon', + re: 'one.two$three^four', + source: 'data', + }, + ], + [1700077283237], + [ + '{"_entry":"log text with ANSI \\u001b[31mpart of the text\\u001b[0m [149702545]","counter":"22292","float":"NaN","wave":-0.5877852522916832,"label":"val3","level":"info"}', + ], + ['1700077283237000000'], + ['1700077283237000000_9b025d35'], + ], + }, + }, + ], + }, + }, +}; + +describe('Loki Query Editor', () => { + beforeEach(() => { + e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD')); + }); + + afterEach(() => { + e2e.flows.revertAllChanges(); + }); + + beforeEach(() => { + cy.window().then((win) => { + win.localStorage.setItem('grafana.featureToggles', 'logsExploreTableVisualisation=1'); + }); + }); + it('Should be able to add explore table to dashboard', () => { + addDataSource(); + + cy.intercept(/labels?/, (req) => { + req.reply({ status: 'success', data: ['instance', 'job', 'source'] }); + }); + + cy.intercept(/series?/, (req) => { + req.reply({ status: 'success', data: [{ instance: 'instance1' }] }); + }); + + cy.intercept(/\/api\/ds\/query\?ds_type=loki?/, (req) => { + req.reply(lokiQueryResult); + }); + + // Go to Explore and choose Loki data source + e2e.pages.Explore.visit(); + e2e.components.DataSourcePicker.container().should('be.visible').click(); + cy.contains(dataSourceName).scrollIntoView().should('be.visible').click(); + + cy.contains('Code').click({ force: true }); + + // Wait for lazy loading + const monacoLoadingText = 'Loading...'; + + e2e.components.QueryField.container().should('be.visible').should('have.text', monacoLoadingText); + e2e.components.QueryField.container().should('be.visible').should('not.have.text', monacoLoadingText); + + // Write a simple query + e2e.components.QueryField.container().type('query').type('{instance="instance1"'); + cy.get('.monaco-editor textarea:first').should(($el) => { + expect($el.val()).to.eq('query{instance="instance1"}'); + }); + + // Submit the query + e2e.components.QueryField.container().type('{shift+enter}'); + // Assert the no-data message is not visible + cy.get('[data-testid="explore-no-data"]').should('not.exist'); + + // Click on the table toggle + cy.contains('Table').click({ force: true }); + + // One row with two cells + cy.get('[role="cell"]').should('have.length', 2); + + cy.contains('label', 'targetLabelName').should('be.visible'); + cy.contains('label', 'targetLabelName').click(); + cy.contains('label', 'targetLabelName').within(() => { + cy.get('input[type="checkbox"]').check({ force: true }); + }); + + cy.contains('label', 'targetLabelName').within(() => { + cy.get('input[type="checkbox"]').should('be.checked'); + }); + + const exploreCells = cy.get('[role="cell"]'); + + // Now we should have a row with 3 columns + exploreCells.should('have.length', 3); + // And a value of "targetLabelValue" + exploreCells.should('contain', 'targetLabelValue'); + + const addToDashboardButton = cy.get('[aria-label="Add to dashboard"]'); + + // Now let's add this to a dashboard + addToDashboardButton.should('be.visible'); + addToDashboardButton.click(); + + const addPanelToDashboardButton = cy.contains('Add panel to dashboard'); + addPanelToDashboardButton.should('be.visible'); + + const openDashboardButton = cy.contains('Open dashboard'); + openDashboardButton.should('be.visible'); + openDashboardButton.click(); + + const panel = cy.get('[data-panelid="1"]'); + panel.should('be.visible'); + + const cells = panel.find('[role="table"] [role="cell"]'); + // Should have 3 columns + cells.should('have.length', 3); + // Cells contain strings found in log line + cells.contains('"wave":-0.5877852522916832'); + + // column has correct value of "targetLabelValue", need to requery the DOM because of the .contains call above + cy.get('[data-panelid="1"]').find('[role="table"] [role="cell"]').contains('targetLabelValue'); + }); +}); diff --git a/packages/grafana-data/src/types/explore.ts b/packages/grafana-data/src/types/explore.ts index f16e2d98ed9..537104f0418 100644 --- a/packages/grafana-data/src/types/explore.ts +++ b/packages/grafana-data/src/types/explore.ts @@ -51,6 +51,7 @@ export interface ExploreLogsPanelState { id?: string; columns?: Record; visualisationType?: 'table' | 'logs'; + labelFieldName?: string; // Used for logs table visualisation, contains the refId of the dataFrame that is currently visualized refId?: string; } diff --git a/public/app/features/explore/Logs/Logs.tsx b/public/app/features/explore/Logs/Logs.tsx index d86baa5bf0d..605ecd43b5e 100644 --- a/public/app/features/explore/Logs/Logs.tsx +++ b/public/app/features/explore/Logs/Logs.tsx @@ -173,8 +173,17 @@ class UnthemedLogs extends PureComponent { if (this.cancelFlippingTimer) { window.clearTimeout(this.cancelFlippingTimer); } + // Delete url state on unmount + if (this.props?.panelState?.logs?.columns) { + delete this.props.panelState.logs.columns; + } + if (this.props?.panelState?.logs?.refId) { + delete this.props.panelState.logs.refId; + } + if (this.props?.panelState?.logs?.labelFieldName) { + delete this.props.panelState.logs.labelFieldName; + } } - updatePanelState = (logsPanelState: Partial) => { const state: ExploreItemState | undefined = getState().explore.panes[this.props.exploreId]; if (state?.panelsState) { @@ -183,6 +192,7 @@ class UnthemedLogs extends PureComponent { ...state.panelsState.logs, columns: logsPanelState.columns ?? this.props.panelState?.logs?.columns, visualisationType: logsPanelState.visualisationType ?? this.state.visualisationType, + labelFieldName: logsPanelState.labelFieldName, refId: logsPanelState.refId ?? this.props.panelState?.logs?.refId, }) ); @@ -193,6 +203,7 @@ class UnthemedLogs extends PureComponent { if (this.props.loading && !prevProps.loading && this.props.panelState?.logs?.id) { // loading stopped, so we need to remove any permalinked log lines delete this.props.panelState.logs.id; + dispatch( changePanelState(this.props.exploreId, 'logs', { ...this.props.panelState, diff --git a/public/app/features/explore/Logs/LogsTableNavColumn.tsx b/public/app/features/explore/Logs/LogsTableNavColumn.tsx index 05ea2f22c88..ec32e1caf37 100644 --- a/public/app/features/explore/Logs/LogsTableNavColumn.tsx +++ b/public/app/features/explore/Logs/LogsTableNavColumn.tsx @@ -72,6 +72,16 @@ function sortLabels(labels: Record) { } } + if (labels[b].active && labels[a].active) { + // Sort alphabetically + if (a < b) { + return -1; + } + if (a > b) { + return 1; + } + } + // If just one label is active, sort it first if (labels[b].active) { return 1; @@ -97,7 +107,7 @@ function sortLabels(labels: Record) { return 1; } - // Finally sort by percent enabled, this could have conflicts with the special fields above, except they are always on 100% of logs + // Finally sort by name if (a < b) { return -1; } diff --git a/public/app/features/explore/Logs/LogsTableWrap.test.tsx b/public/app/features/explore/Logs/LogsTableWrap.test.tsx index fea39488c4f..9e5b59c93f9 100644 --- a/public/app/features/explore/Logs/LogsTableWrap.test.tsx +++ b/public/app/features/explore/Logs/LogsTableWrap.test.tsx @@ -1,5 +1,6 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/react'; import React, { ComponentProps } from 'react'; +import { act } from 'react-test-renderer'; import { createTheme, @@ -92,24 +93,31 @@ describe('LogsTableWrap', () => { expect.assertions(3); - const checkboxLabel = screen.getByLabelText('app'); - expect(checkboxLabel).toBeInTheDocument(); + expect(screen.getByLabelText('app')).toBeInTheDocument(); // Add a new column + act(() => { + screen.getByLabelText('app').click(); + }); + await waitFor(() => { - checkboxLabel.click(); expect(updatePanelState).toBeCalledWith({ visualisationType: 'table', columns: { 0: 'app', 1: 'Line', 2: 'Time' }, + labelFieldName: 'labels', }); }); // Remove the same column + act(() => { + screen.getByLabelText('app').click(); + }); + await waitFor(() => { - checkboxLabel.click(); expect(updatePanelState).toBeCalledWith({ visualisationType: 'table', columns: { 0: 'Line', 1: 'Time' }, + labelFieldName: 'labels', }); }); }); diff --git a/public/app/features/explore/Logs/LogsTableWrap.tsx b/public/app/features/explore/Logs/LogsTableWrap.tsx index d46b2352ccd..5484e51eeac 100644 --- a/public/app/features/explore/Logs/LogsTableWrap.tsx +++ b/public/app/features/explore/Logs/LogsTableWrap.tsx @@ -44,8 +44,8 @@ type fieldName = string; type fieldNameMetaStore = Record; export function LogsTableWrap(props: Props) { - const { logsFrames } = props; - + const { logsFrames, updatePanelState, panelState } = props; + const propsColumns = panelState?.columns; // Save the normalized cardinality of each label const [columnsWithMeta, setColumnsWithMeta] = useState(undefined); @@ -74,6 +74,18 @@ export function LogsTableWrap(props: Props) { }, [props.panelState?.columns] ); + const logsFrame = parseLogsFrame(currentDataFrame); + + useEffect(() => { + if (logsFrame?.timeField.name && logsFrame?.bodyField.name && !propsColumns) { + const defaultColumns = { 0: logsFrame?.timeField.name ?? '', 1: logsFrame?.bodyField.name ?? '' }; + updatePanelState({ + columns: Object.values(defaultColumns), + visualisationType: 'table', + labelFieldName: logsFrame?.getLabelFieldName() ?? undefined, + }); + } + }, [logsFrame, propsColumns, updatePanelState]); /** * When logs frame updates (e.g. query|range changes), we need to set the selected frame to state @@ -266,22 +278,26 @@ export function LogsTableWrap(props: Props) { setFilteredColumnsWithMeta(pendingFilteredLabelState); } + const newColumns: Record = Object.assign( + {}, + // Get the keys of the object as an array + Object.keys(pendingLabelState) + // Only include active filters + .filter((key) => pendingLabelState[key]?.active) + ); + + const defaultColumns = { 0: logsFrame?.timeField.name ?? '', 1: logsFrame?.bodyField.name ?? '' }; const newPanelState: ExploreLogsPanelState = { ...props.panelState, // URL format requires our array of values be an object, so we convert it using object.assign - columns: Object.assign( - {}, - // Get the keys of the object as an array - Object.keys(pendingLabelState) - // Only include active filters - .filter((key) => pendingLabelState[key]?.active) - ), + columns: Object.keys(newColumns).length ? newColumns : defaultColumns, refId: currentDataFrame.refId, visualisationType: 'table', + labelFieldName: logsFrame?.getLabelFieldName() ?? undefined, }; // Update url state - props.updatePanelState(newPanelState); + updatePanelState(newPanelState); }; // uFuzzy search dispatcher, adds any matches to the local state @@ -323,7 +339,7 @@ export function LogsTableWrap(props: Props) { if (matchingDataFrame) { setCurrentDataFrame(logsFrames.find((frame) => frame.refId === value.value) ?? logsFrames[0]); } - props.updatePanelState({ refId: value.value }); + props.updatePanelState({ refId: value.value, labelFieldName: logsFrame?.getLabelFieldName() ?? undefined }); }; const sidebarWidth = 220; diff --git a/public/app/features/explore/extensions/AddToDashboard/AddToDashboardForm.tsx b/public/app/features/explore/extensions/AddToDashboard/AddToDashboardForm.tsx index 83924809dcb..9ac895b189f 100644 --- a/public/app/features/explore/extensions/AddToDashboard/AddToDashboardForm.tsx +++ b/public/app/features/explore/extensions/AddToDashboard/AddToDashboardForm.tsx @@ -108,6 +108,7 @@ export function AddToDashboardForm(props: Props): ReactElement { datasource: exploreItem.datasourceInstance?.getRef(), queries: exploreItem.queries, queryResponse: exploreItem.queryResponse, + panelState: exploreItem?.panelsState, }); } catch (error) { switch (error) { diff --git a/public/app/features/explore/extensions/AddToDashboard/addToDashboard.ts b/public/app/features/explore/extensions/AddToDashboard/addToDashboard.ts index 3bee074b11c..bb780a01da5 100644 --- a/public/app/features/explore/extensions/AddToDashboard/addToDashboard.ts +++ b/public/app/features/explore/extensions/AddToDashboard/addToDashboard.ts @@ -1,5 +1,6 @@ -import { DataFrame } from '@grafana/data'; +import { DataFrame, ExplorePanelsState } from '@grafana/data'; import { DataQuery, DataSourceRef } from '@grafana/schema'; +import { DataTransformerConfig } from '@grafana/schema/dist/esm/raw/dashboard/x/dashboard_types.gen'; import { backendSrv } from 'app/core/services/backend_srv'; import { getNewDashboardModelData, @@ -17,6 +18,7 @@ interface AddPanelToDashboardOptions { queryResponse: ExplorePanelData; datasource?: DataSourceRef; dashboardUid?: string; + panelState?: ExplorePanelsState; } function createDashboard(): DashboardDTO { @@ -28,14 +30,53 @@ function createDashboard(): DashboardDTO { return dto; } +/** + * Returns transformations for the logs table visualisation in explore. + * If the logs table supports a labels column, we need to extract the fields. + * Then we can set the columns to show in the table via the organize/includeByName transformation + * @param panelType + * @param options + */ +function getLogsTableTransformations(panelType: string, options: AddPanelToDashboardOptions): DataTransformerConfig[] { + let transformations: DataTransformerConfig[] = []; + if (panelType === 'table' && options.panelState?.logs?.columns) { + // If we have a labels column, we need to extract the fields from it + if (options.panelState.logs?.labelFieldName) { + transformations.push({ + id: 'extractFields', + options: { + source: options.panelState.logs.labelFieldName, + }, + }); + } + + // Show the columns that the user selected in explore + transformations.push({ + id: 'organize', + options: { + includeByName: Object.values(options.panelState.logs.columns).reduce( + (acc: Record, value: string) => ({ + ...acc, + [value]: true, + }), + {} + ), + }, + }); + } + return transformations; +} + export async function setDashboardInLocalStorage(options: AddPanelToDashboardOptions) { - const panelType = getPanelType(options.queries, options.queryResponse); + const panelType = getPanelType(options.queries, options.queryResponse, options?.panelState); + const panel = { targets: options.queries, type: panelType, title: 'New Panel', gridPos: { x: 0, y: 0, w: 12, h: 8 }, datasource: options.datasource, + transformations: getLogsTableTransformations(panelType, options), }; let dto: DashboardDTO; @@ -62,7 +103,7 @@ export async function setDashboardInLocalStorage(options: AddPanelToDashboardOpt const isVisible = (query: DataQuery) => !query.hide; const hasRefId = (refId: DataFrame['refId']) => (frame: DataFrame) => frame.refId === refId; -function getPanelType(queries: DataQuery[], queryResponse: ExplorePanelData) { +function getPanelType(queries: DataQuery[], queryResponse: ExplorePanelData, panelState?: ExplorePanelsState) { for (const { refId } of queries.filter(isVisible)) { const hasQueryRefId = hasRefId(refId); if (queryResponse.flameGraphFrames.some(hasQueryRefId)) { @@ -72,6 +113,9 @@ function getPanelType(queries: DataQuery[], queryResponse: ExplorePanelData) { return 'timeseries'; } if (queryResponse.logsFrames.some(hasQueryRefId)) { + if (panelState?.logs?.visualisationType) { + return panelState.logs.visualisationType; + } return 'logs'; } if (queryResponse.nodeGraphFrames.some(hasQueryRefId)) { diff --git a/public/app/features/logs/legacyLogsFrame.ts b/public/app/features/logs/legacyLogsFrame.ts index c50d2eec349..9b9ac9bc011 100644 --- a/public/app/features/logs/legacyLogsFrame.ts +++ b/public/app/features/logs/legacyLogsFrame.ts @@ -72,6 +72,7 @@ export function parseLegacyLogsFrame(frame: DataFrame): LogsFrame | null { idField, getLogFrameLabels: getL, getLogFrameLabelsAsLabels: getL, + getLabelFieldName: () => labelsField?.name ?? null, extraFields, }; } diff --git a/public/app/features/logs/logsFrame.ts b/public/app/features/logs/logsFrame.ts index cdb931c299f..513f56a3e1e 100644 --- a/public/app/features/logs/logsFrame.ts +++ b/public/app/features/logs/logsFrame.ts @@ -16,6 +16,7 @@ export type LogsFrame = { idField: FieldWithIndex | null; getLogFrameLabels: () => LogFrameLabels[] | null; // may be slow, so we only do it when asked for it explicitly getLogFrameLabelsAsLabels: () => Labels[] | null; // temporarily exists to make the labels=>attributes migration simpler + getLabelFieldName: () => string | null; extraFields: FieldWithIndex[]; }; @@ -78,6 +79,7 @@ function parseDataplaneLogsFrame(frame: DataFrame): LogsFrame | null { getLogFrameLabels: () => labels, timeNanosecondField: null, getLogFrameLabelsAsLabels: () => (labels !== null ? labels.map(logFrameLabelsToLabels) : null), + getLabelFieldName: () => (labelsField !== null ? labelsField.name : null), extraFields, }; }