Logs Panel: Table UI - Pull logs table into dashboard panel (#77757)

* Allows users to add a logs table in explore to a dashboard panel via the includeByName transformation

e2e/various-suite/loki-table-explore-to-dash.spec.ts

@@ -0,0 +1,214 @@
+import { e2e } from '../utils';
+
+const dataSourceName = 'LokiEditor';
+const addDataSource = () => {
+  e2e.flows.addDataSource({
+    type: 'Loki',
+    expectedAlertMessage: 'Unable to connect with Loki. Please check the server logs for more details.',
+    name: dataSourceName,
+    form: () => {
+      cy.get('#connection-url').type('http://loki-url:3100');
+    },
+  });
+};
+
+const lokiQueryResult = {
+  status: 'success',
+  results: {
+    A: {
+      status: 200,
+      frames: [
+        {
+          schema: {
+            refId: 'A',
+            meta: {
+              typeVersion: [0, 0],
+              custom: {
+                frameType: 'LabeledTimeValues',
+              },
+              stats: [
+                {
+                  displayName: 'Summary: bytes processed per second',
+                  unit: 'Bps',
+                  value: 223921,
+                },
+                {
+                  displayName: 'Summary: total bytes processed',
+                  unit: 'decbytes',
+                  value: 4156,
+                },
+                {
+                  displayName: 'Summary: exec time',
+                  unit: 's',
+                  value: 0.01856,
+                },
+              ],
+              executedQueryString: 'Expr: {targetLabelName="targetLabelValue"}',
+            },
+            fields: [
+              {
+                name: 'labels',
+                type: 'other',
+                typeInfo: {
+                  frame: 'json.RawMessage',
+                },
+              },
+              {
+                name: 'Time',
+                type: 'time',
+                typeInfo: {
+                  frame: 'time.Time',
+                },
+              },
+              {
+                name: 'Line',
+                type: 'string',
+                typeInfo: {
+                  frame: 'string',
+                },
+              },
+              {
+                name: 'tsNs',
+                type: 'string',
+                typeInfo: {
+                  frame: 'string',
+                },
+              },
+              {
+                name: 'id',
+                type: 'string',
+                typeInfo: {
+                  frame: 'string',
+                },
+              },
+            ],
+          },
+          data: {
+            values: [
+              [
+                {
+                  targetLabelName: 'targetLabelValue',
+                  instance: 'server\\1',
+                  job: '"grafana/data"',
+                  nonIndexed: 'value',
+                  place: 'moon',
+                  re: 'one.two$three^four',
+                  source: 'data',
+                },
+              ],
+              [1700077283237],
+              [
+                '{"_entry":"log text with ANSI \\u001b[31mpart of the text\\u001b[0m [149702545]","counter":"22292","float":"NaN","wave":-0.5877852522916832,"label":"val3","level":"info"}',
+              ],
+              ['1700077283237000000'],
+              ['1700077283237000000_9b025d35'],
+            ],
+          },
+        },
+      ],
+    },
+  },
+};
+
+describe('Loki Query Editor', () => {
+  beforeEach(() => {
+    e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'));
+  });
+
+  afterEach(() => {
+    e2e.flows.revertAllChanges();
+  });
+
+  beforeEach(() => {
+    cy.window().then((win) => {
+      win.localStorage.setItem('grafana.featureToggles', 'logsExploreTableVisualisation=1');
+    });
+  });
+  it('Should be able to add explore table to dashboard', () => {
+    addDataSource();
+
+    cy.intercept(/labels?/, (req) => {
+      req.reply({ status: 'success', data: ['instance', 'job', 'source'] });
+    });
+
+    cy.intercept(/series?/, (req) => {
+      req.reply({ status: 'success', data: [{ instance: 'instance1' }] });
+    });
+
+    cy.intercept(/\/api\/ds\/query\?ds_type=loki?/, (req) => {
+      req.reply(lokiQueryResult);
+    });
+
+    // Go to Explore and choose Loki data source
+    e2e.pages.Explore.visit();
+    e2e.components.DataSourcePicker.container().should('be.visible').click();
+    cy.contains(dataSourceName).scrollIntoView().should('be.visible').click();
+
+    cy.contains('Code').click({ force: true });
+
+    // Wait for lazy loading
+    const monacoLoadingText = 'Loading...';
+
+    e2e.components.QueryField.container().should('be.visible').should('have.text', monacoLoadingText);
+    e2e.components.QueryField.container().should('be.visible').should('not.have.text', monacoLoadingText);
+
+    // Write a simple query
+    e2e.components.QueryField.container().type('query').type('{instance="instance1"');
+    cy.get('.monaco-editor textarea:first').should(($el) => {
+      expect($el.val()).to.eq('query{instance="instance1"}');
+    });
+
+    // Submit the query
+    e2e.components.QueryField.container().type('{shift+enter}');
+    // Assert the no-data message is not visible
+    cy.get('[data-testid="explore-no-data"]').should('not.exist');
+
+    // Click on the table toggle
+    cy.contains('Table').click({ force: true });
+
+    // One row with two cells
+    cy.get('[role="cell"]').should('have.length', 2);
+
+    cy.contains('label', 'targetLabelName').should('be.visible');
+    cy.contains('label', 'targetLabelName').click();
+    cy.contains('label', 'targetLabelName').within(() => {
+      cy.get('input[type="checkbox"]').check({ force: true });
+    });
+
+    cy.contains('label', 'targetLabelName').within(() => {
+      cy.get('input[type="checkbox"]').should('be.checked');
+    });
+
+    const exploreCells = cy.get('[role="cell"]');
+
+    // Now we should have a row with 3 columns
+    exploreCells.should('have.length', 3);
+    // And a value of "targetLabelValue"
+    exploreCells.should('contain', 'targetLabelValue');
+
+    const addToDashboardButton = cy.get('[aria-label="Add to dashboard"]');
+
+    // Now let's add this to a dashboard
+    addToDashboardButton.should('be.visible');
+    addToDashboardButton.click();
+
+    const addPanelToDashboardButton = cy.contains('Add panel to dashboard');
+    addPanelToDashboardButton.should('be.visible');
+
+    const openDashboardButton = cy.contains('Open dashboard');
+    openDashboardButton.should('be.visible');
+    openDashboardButton.click();
+
+    const panel = cy.get('[data-panelid="1"]');
+    panel.should('be.visible');
+
+    const cells = panel.find('[role="table"] [role="cell"]');
+    // Should have 3 columns
+    cells.should('have.length', 3);
+    // Cells contain strings found in log line
+    cells.contains('"wave":-0.5877852522916832');
+
+    // column has correct value of "targetLabelValue", need to requery the DOM because of the .contains call above
+    cy.get('[data-panelid="1"]').find('[role="table"] [role="cell"]').contains('targetLabelValue');
+  });
+});

packages/grafana-data/src/types/explore.ts

@@ -51,6 +51,7 @@ export interface ExploreLogsPanelState {
   id?: string;
   columns?: Record<number, string>;
   visualisationType?: 'table' | 'logs';
+  labelFieldName?: string;
   // Used for logs table visualisation, contains the refId of the dataFrame that is currently visualized
   refId?: string;
 }

public/app/features/explore/Logs/Logs.tsx

@@ -173,8 +173,17 @@ class UnthemedLogs extends PureComponent<Props, State> {
     if (this.cancelFlippingTimer) {
       window.clearTimeout(this.cancelFlippingTimer);
     }
+    // Delete url state on unmount
+    if (this.props?.panelState?.logs?.columns) {
+      delete this.props.panelState.logs.columns;
+    }
+    if (this.props?.panelState?.logs?.refId) {
+      delete this.props.panelState.logs.refId;
+    }
+    if (this.props?.panelState?.logs?.labelFieldName) {
+      delete this.props.panelState.logs.labelFieldName;
+    }
   }
-
   updatePanelState = (logsPanelState: Partial<ExploreLogsPanelState>) => {
     const state: ExploreItemState | undefined = getState().explore.panes[this.props.exploreId];
     if (state?.panelsState) {
@@ -183,6 +192,7 @@ class UnthemedLogs extends PureComponent<Props, State> {
           ...state.panelsState.logs,
           columns: logsPanelState.columns ?? this.props.panelState?.logs?.columns,
           visualisationType: logsPanelState.visualisationType ?? this.state.visualisationType,
+          labelFieldName: logsPanelState.labelFieldName,
           refId: logsPanelState.refId ?? this.props.panelState?.logs?.refId,
         })
       );
@@ -193,6 +203,7 @@ class UnthemedLogs extends PureComponent<Props, State> {
     if (this.props.loading && !prevProps.loading && this.props.panelState?.logs?.id) {
       // loading stopped, so we need to remove any permalinked log lines
       delete this.props.panelState.logs.id;
+
       dispatch(
         changePanelState(this.props.exploreId, 'logs', {
           ...this.props.panelState,

public/app/features/explore/Logs/LogsTableNavColumn.tsx

@@ -72,6 +72,16 @@ function sortLabels(labels: Record<string, fieldNameMeta>) {
       }
     }
 
+    if (labels[b].active && labels[a].active) {
+      // Sort alphabetically
+      if (a < b) {
+        return -1;
+      }
+      if (a > b) {
+        return 1;
+      }
+    }
+
     // If just one label is active, sort it first
     if (labels[b].active) {
       return 1;
@@ -97,7 +107,7 @@ function sortLabels(labels: Record<string, fieldNameMeta>) {
       return 1;
     }
 
-    // Finally sort by percent enabled, this could have conflicts with the special fields above, except they are always on 100% of logs
+    // Finally sort by name
     if (a < b) {
       return -1;
     }

public/app/features/explore/Logs/LogsTableWrap.test.tsx

@@ -1,5 +1,6 @@
 import { fireEvent, render, screen, waitFor } from '@testing-library/react';
 import React, { ComponentProps } from 'react';
+import { act } from 'react-test-renderer';
 
 import {
   createTheme,
@@ -92,24 +93,31 @@ describe('LogsTableWrap', () => {
 
     expect.assertions(3);
 
-    const checkboxLabel = screen.getByLabelText('app');
+    expect(screen.getByLabelText('app')).toBeInTheDocument();
-    expect(checkboxLabel).toBeInTheDocument();
 
     // Add a new column
+    act(() => {
+      screen.getByLabelText('app').click();
+    });
+
     await waitFor(() => {
-      checkboxLabel.click();
       expect(updatePanelState).toBeCalledWith({
         visualisationType: 'table',
         columns: { 0: 'app', 1: 'Line', 2: 'Time' },
+        labelFieldName: 'labels',
       });
     });
 
     // Remove the same column
+    act(() => {
+      screen.getByLabelText('app').click();
+    });
+
     await waitFor(() => {
-      checkboxLabel.click();
       expect(updatePanelState).toBeCalledWith({
         visualisationType: 'table',
         columns: { 0: 'Line', 1: 'Time' },
+        labelFieldName: 'labels',
       });
     });
   });

public/app/features/explore/Logs/LogsTableWrap.tsx

@@ -44,8 +44,8 @@ type fieldName = string;
 type fieldNameMetaStore = Record<fieldName, fieldNameMeta>;
 
 export function LogsTableWrap(props: Props) {
-  const { logsFrames } = props;
+  const { logsFrames, updatePanelState, panelState } = props;
-
+  const propsColumns = panelState?.columns;
   // Save the normalized cardinality of each label
   const [columnsWithMeta, setColumnsWithMeta] = useState<fieldNameMetaStore | undefined>(undefined);
 
@@ -74,6 +74,18 @@ export function LogsTableWrap(props: Props) {
     },
     [props.panelState?.columns]
   );
+  const logsFrame = parseLogsFrame(currentDataFrame);
+
+  useEffect(() => {
+    if (logsFrame?.timeField.name && logsFrame?.bodyField.name && !propsColumns) {
+      const defaultColumns = { 0: logsFrame?.timeField.name ?? '', 1: logsFrame?.bodyField.name ?? '' };
+      updatePanelState({
+        columns: Object.values(defaultColumns),
+        visualisationType: 'table',
+        labelFieldName: logsFrame?.getLabelFieldName() ?? undefined,
+      });
+    }
+  }, [logsFrame, propsColumns, updatePanelState]);
 
   /**
    * When logs frame updates (e.g. query|range changes), we need to set the selected frame to state
@@ -266,22 +278,26 @@ export function LogsTableWrap(props: Props) {
       setFilteredColumnsWithMeta(pendingFilteredLabelState);
     }
 
+    const newColumns: Record<number, string> = Object.assign(
+      {},
+      // Get the keys of the object as an array
+      Object.keys(pendingLabelState)
+        // Only include active filters
+        .filter((key) => pendingLabelState[key]?.active)
+    );
+
+    const defaultColumns = { 0: logsFrame?.timeField.name ?? '', 1: logsFrame?.bodyField.name ?? '' };
     const newPanelState: ExploreLogsPanelState = {
       ...props.panelState,
       // URL format requires our array of values be an object, so we convert it using object.assign
-      columns: Object.assign(
+      columns: Object.keys(newColumns).length ? newColumns : defaultColumns,
-        {},
-        // Get the keys of the object as an array
-        Object.keys(pendingLabelState)
-          // Only include active filters
-          .filter((key) => pendingLabelState[key]?.active)
-      ),
       refId: currentDataFrame.refId,
       visualisationType: 'table',
+      labelFieldName: logsFrame?.getLabelFieldName() ?? undefined,
     };
 
     // Update url state
-    props.updatePanelState(newPanelState);
+    updatePanelState(newPanelState);
   };
 
   // uFuzzy search dispatcher, adds any matches to the local state
@@ -323,7 +339,7 @@ export function LogsTableWrap(props: Props) {
     if (matchingDataFrame) {
       setCurrentDataFrame(logsFrames.find((frame) => frame.refId === value.value) ?? logsFrames[0]);
     }
-    props.updatePanelState({ refId: value.value });
+    props.updatePanelState({ refId: value.value, labelFieldName: logsFrame?.getLabelFieldName() ?? undefined });
   };
 
   const sidebarWidth = 220;

public/app/features/explore/extensions/AddToDashboard/AddToDashboardForm.tsx

@@ -108,6 +108,7 @@ export function AddToDashboardForm(props: Props): ReactElement {
         datasource: exploreItem.datasourceInstance?.getRef(),
         queries: exploreItem.queries,
         queryResponse: exploreItem.queryResponse,
+        panelState: exploreItem?.panelsState,
       });
     } catch (error) {
       switch (error) {

public/app/features/explore/extensions/AddToDashboard/addToDashboard.ts

@@ -1,5 +1,6 @@
-import { DataFrame } from '@grafana/data';
+import { DataFrame, ExplorePanelsState } from '@grafana/data';
 import { DataQuery, DataSourceRef } from '@grafana/schema';
+import { DataTransformerConfig } from '@grafana/schema/dist/esm/raw/dashboard/x/dashboard_types.gen';
 import { backendSrv } from 'app/core/services/backend_srv';
 import {
   getNewDashboardModelData,
@@ -17,6 +18,7 @@ interface AddPanelToDashboardOptions {
   queryResponse: ExplorePanelData;
   datasource?: DataSourceRef;
   dashboardUid?: string;
+  panelState?: ExplorePanelsState;
 }
 
 function createDashboard(): DashboardDTO {
@@ -28,14 +30,53 @@ function createDashboard(): DashboardDTO {
   return dto;
 }
 
+/**
+ * Returns transformations for the logs table visualisation in explore.
+ * If the logs table supports a labels column, we need to extract the fields.
+ * Then we can set the columns to show in the table via the organize/includeByName transformation
+ * @param panelType
+ * @param options
+ */
+function getLogsTableTransformations(panelType: string, options: AddPanelToDashboardOptions): DataTransformerConfig[] {
+  let transformations: DataTransformerConfig[] = [];
+  if (panelType === 'table' && options.panelState?.logs?.columns) {
+    // If we have a labels column, we need to extract the fields from it
+    if (options.panelState.logs?.labelFieldName) {
+      transformations.push({
+        id: 'extractFields',
+        options: {
+          source: options.panelState.logs.labelFieldName,
+        },
+      });
+    }
+
+    // Show the columns that the user selected in explore
+    transformations.push({
+      id: 'organize',
+      options: {
+        includeByName: Object.values(options.panelState.logs.columns).reduce(
+          (acc: Record<string, boolean>, value: string) => ({
+            ...acc,
+            [value]: true,
+          }),
+          {}
+        ),
+      },
+    });
+  }
+  return transformations;
+}
+
 export async function setDashboardInLocalStorage(options: AddPanelToDashboardOptions) {
-  const panelType = getPanelType(options.queries, options.queryResponse);
+  const panelType = getPanelType(options.queries, options.queryResponse, options?.panelState);
+
   const panel = {
     targets: options.queries,
     type: panelType,
     title: 'New Panel',
     gridPos: { x: 0, y: 0, w: 12, h: 8 },
     datasource: options.datasource,
+    transformations: getLogsTableTransformations(panelType, options),
   };
 
   let dto: DashboardDTO;
@@ -62,7 +103,7 @@ export async function setDashboardInLocalStorage(options: AddPanelToDashboardOpt
 const isVisible = (query: DataQuery) => !query.hide;
 const hasRefId = (refId: DataFrame['refId']) => (frame: DataFrame) => frame.refId === refId;
 
-function getPanelType(queries: DataQuery[], queryResponse: ExplorePanelData) {
+function getPanelType(queries: DataQuery[], queryResponse: ExplorePanelData, panelState?: ExplorePanelsState) {
   for (const { refId } of queries.filter(isVisible)) {
     const hasQueryRefId = hasRefId(refId);
     if (queryResponse.flameGraphFrames.some(hasQueryRefId)) {
@@ -72,6 +113,9 @@ function getPanelType(queries: DataQuery[], queryResponse: ExplorePanelData) {
       return 'timeseries';
     }
     if (queryResponse.logsFrames.some(hasQueryRefId)) {
+      if (panelState?.logs?.visualisationType) {
+        return panelState.logs.visualisationType;
+      }
       return 'logs';
     }
     if (queryResponse.nodeGraphFrames.some(hasQueryRefId)) {

public/app/features/logs/legacyLogsFrame.ts

@@ -72,6 +72,7 @@ export function parseLegacyLogsFrame(frame: DataFrame): LogsFrame | null {
     idField,
     getLogFrameLabels: getL,
     getLogFrameLabelsAsLabels: getL,
+    getLabelFieldName: () => labelsField?.name ?? null,
     extraFields,
   };
 }

public/app/features/logs/logsFrame.ts

@@ -16,6 +16,7 @@ export type LogsFrame = {
   idField: FieldWithIndex | null;
   getLogFrameLabels: () => LogFrameLabels[] | null; // may be slow, so we only do it when asked for it explicitly
   getLogFrameLabelsAsLabels: () => Labels[] | null; // temporarily exists to make the labels=>attributes migration simpler
+  getLabelFieldName: () => string | null;
   extraFields: FieldWithIndex[];
 };
 
@@ -78,6 +79,7 @@ function parseDataplaneLogsFrame(frame: DataFrame): LogsFrame | null {
     getLogFrameLabels: () => labels,
     timeNanosecondField: null,
     getLogFrameLabelsAsLabels: () => (labels !== null ? labels.map(logFrameLabelsToLabels) : null),
+    getLabelFieldName: () => (labelsField !== null ? labelsField.name : null),
     extraFields,
   };
 }