Loki: Preserve pipeline stages in context query (#70472)

* add pipeline stages to context query

* add ui

* improve `Postion`

* add `processPipelineStagesToExpr` to logcontextprovider

* add ui toggle

* fix lokicontextui tests

* remove import

* contextually hide the toggle

* Update `SHOULD_INCLUDE_PIPELINE_OPERATIONS` name

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* add setIncludePipelineOperations to false in revert

* add prepareExpression method

* remove unused method

* fix test and add `runContextQuery`

* set correct revert state

* let let be const

* remove argument

---------

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

public/app/plugins/datasource/loki/LogContextProvider.test.ts

@@ -3,14 +3,26 @@ import { of } from 'rxjs';
 import { DataQueryResponse, FieldType, LogRowContextQueryDirection, LogRowModel, createDataFrame } from '@grafana/data';
 
 import LokiLanguageProvider from './LanguageProvider';
-import { LogContextProvider, LOKI_LOG_CONTEXT_PRESERVED_LABELS } from './LogContextProvider';
+import {
+  LogContextProvider,
+  LOKI_LOG_CONTEXT_PRESERVED_LABELS,
+  SHOULD_INCLUDE_PIPELINE_OPERATIONS,
+} from './LogContextProvider';
 import { createLokiDatasource } from './mocks';
 import { LokiQuery } from './types';
 
 jest.mock('app/core/store', () => {
   return {
-    get() {
-      return window.localStorage.getItem(LOKI_LOG_CONTEXT_PRESERVED_LABELS);
+    get(item: string) {
+      return window.localStorage.getItem(item);
+    },
+    getBool(key: string, defaultValue?: boolean) {
+      const item = window.localStorage.getItem(key);
+      if (item === null) {
+        return defaultValue;
+      } else {
+        return item === 'true';
+      }
     },
   };
 });
@@ -205,6 +217,143 @@ describe('LogContextProvider', () => {
 
       expect(contextQuery.query.expr).toEqual(`{bar="baz"}`);
     });
+
+    it('should not apply line_format if flag is not set by default', async () => {
+      logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
+      const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt | line_format = "foo"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt`);
+    });
+
+    it('should not apply line_format if flag is not set', async () => {
+      window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'false');
+      logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
+      const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt  | line_format = "foo"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt`);
+    });
+
+    it('should apply line_format if flag is set', async () => {
+      window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
+      logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
+      const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt | line_format = "foo"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
+    });
+
+    it('should not apply line filters if flag is set', async () => {
+      window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
+      logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
+      let contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt | line_format = "foo" |= "bar"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
+
+      contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt | line_format = "foo" |~ "bar"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
+
+      contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt | line_format = "foo" !~ "bar"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
+
+      contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt | line_format = "foo" != "bar"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format = "foo"`);
+    });
+
+    it('should not apply line filters if nested between two operations', async () => {
+      window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
+      logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
+      const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt | line_format "foo" |= "bar" | label_format a="baz"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format "foo" | label_format a="baz"`);
+    });
+
+    it('should not apply label filters', async () => {
+      window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
+      logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
+      const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt | line_format "foo" | bar > 1 | label_format a="baz"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"} | logfmt | line_format "foo" | label_format a="baz"`);
+    });
+
+    it('should not apply additional parsers', async () => {
+      window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
+      logContextProvider.appliedContextFilters = [{ value: 'baz', enabled: true, fromParser: false, label: 'bar' }];
+      const contextQuery = await logContextProvider.prepareLogRowContextQueryTarget(
+        defaultLogRow,
+        10,
+        LogRowContextQueryDirection.Backward,
+        {
+          expr: '{bar="baz"} | logfmt | line_format "foo" | json | label_format a="baz"',
+        } as unknown as LokiQuery
+      );
+
+      expect(contextQuery.query.expr).toEqual(`{bar="baz"}`);
+    });
   });
 
   describe('getInitContextFiltersFromLabels', () => {
@@ -312,4 +461,38 @@ describe('LogContextProvider', () => {
       });
     });
   });
+
+  describe('queryContainsValidPipelineStages', () => {
+    it('should return true if query contains a line_format stage', () => {
+      expect(
+        logContextProvider.queryContainsValidPipelineStages({ expr: '{foo="bar"} | line_format "foo"' } as LokiQuery)
+      ).toBe(true);
+    });
+
+    it('should return true if query contains a label_format stage', () => {
+      expect(
+        logContextProvider.queryContainsValidPipelineStages({ expr: '{foo="bar"} | label_format a="foo"' } as LokiQuery)
+      ).toBe(true);
+    });
+
+    it('should return false if query contains a parser', () => {
+      expect(logContextProvider.queryContainsValidPipelineStages({ expr: '{foo="bar"} | json' } as LokiQuery)).toBe(
+        false
+      );
+    });
+
+    it('should return false if query contains a line filter', () => {
+      expect(logContextProvider.queryContainsValidPipelineStages({ expr: '{foo="bar"} |= "test"' } as LokiQuery)).toBe(
+        false
+      );
+    });
+
+    it('should return true if query contains a line filter and a label_format', () => {
+      expect(
+        logContextProvider.queryContainsValidPipelineStages({
+          expr: '{foo="bar"} |= "test" | label_format a="foo"',
+        } as LokiQuery)
+      ).toBe(true);
+    });
+  });
 });

public/app/plugins/datasource/loki/LogContextProvider.ts

@@ -14,6 +14,7 @@ import {
   LogRowContextQueryDirection,
   LogRowContextOptions,
 } from '@grafana/data';
+import { LabelParser, LabelFilter, LineFilters, PipelineStage } from '@grafana/lezer-logql';
 import { Labels } from '@grafana/schema';
 import { notifyApp } from 'app/core/actions';
 import { createSuccessNotification } from 'app/core/copy/appNotification';
@@ -24,11 +25,18 @@ import { LokiContextUi } from './components/LokiContextUi';
 import { LokiDatasource, makeRequest, REF_ID_STARTER_LOG_ROW_CONTEXT } from './datasource';
 import { escapeLabelValueInExactSelector } from './languageUtils';
 import { addLabelToQuery, addParserToQuery } from './modifyQuery';
-import { getParserFromQuery, getStreamSelectorsFromQuery, isQueryWithParser } from './queryUtils';
+import {
+  getNodePositionsFromQuery,
+  getParserFromQuery,
+  getStreamSelectorsFromQuery,
+  isQueryWithParser,
+} from './queryUtils';
 import { sortDataFrameByTime, SortDirection } from './sortDataFrame';
 import { ContextFilter, LokiQuery, LokiQueryDirection, LokiQueryType } from './types';
 
 export const LOKI_LOG_CONTEXT_PRESERVED_LABELS = 'lokiLogContextPreservedLabels';
+export const SHOULD_INCLUDE_PIPELINE_OPERATIONS = 'lokiLogContextShouldIncludePipelineOperations';
+
 export type PreservedLabels = {
   removedLabels: string[];
   selectedExtractedLabels: string[];
@@ -109,7 +117,8 @@ export class LogContextProvider {
     direction: LogRowContextQueryDirection,
     origQuery?: LokiQuery
   ): Promise<{ query: LokiQuery; range: TimeRange }> {
-    const expr = this.processContextFiltersToExpr(row, this.appliedContextFilters, origQuery);
+    const expr = this.prepareExpression(this.appliedContextFilters, origQuery);
+
     const contextTimeBuffer = 2 * 60 * 60 * 1000; // 2h buffer
 
     const queryDirection =
@@ -180,10 +189,19 @@ export class LogContextProvider {
       updateFilter,
       onClose: this.onContextClose,
       logContextProvider: this,
+      runContextQuery,
     });
   }
 
-  processContextFiltersToExpr = (row: LogRowModel, contextFilters: ContextFilter[], query: LokiQuery | undefined) => {
+  prepareExpression(contextFilters: ContextFilter[], query: LokiQuery | undefined): string {
+    let preparedExpression = this.processContextFiltersToExpr(contextFilters, query);
+    if (store.getBool(SHOULD_INCLUDE_PIPELINE_OPERATIONS, false)) {
+      preparedExpression = this.processPipelineStagesToExpr(preparedExpression, query);
+    }
+    return preparedExpression;
+  }
+
+  processContextFiltersToExpr = (contextFilters: ContextFilter[], query: LokiQuery | undefined): string => {
     const labelFilters = contextFilters
       .map((filter) => {
         if (!filter.fromParser && filter.enabled) {
@@ -216,6 +234,51 @@ export class LogContextProvider {
     return expr;
   };
 
+  processPipelineStagesToExpr = (currentExpr: string, query: LokiQuery | undefined): string => {
+    let newExpr = currentExpr;
+    const origExpr = query?.expr ?? '';
+
+    if (isQueryWithParser(origExpr).parserCount > 1) {
+      return newExpr;
+    }
+
+    const allNodePositions = getNodePositionsFromQuery(origExpr, [
+      PipelineStage,
+      LabelParser,
+      LineFilters,
+      LabelFilter,
+    ]);
+    const pipelineStagePositions = allNodePositions.filter((position) => position.type?.id === PipelineStage);
+    const otherNodePositions = allNodePositions.filter((position) => position.type?.id !== PipelineStage);
+
+    for (const pipelineStagePosition of pipelineStagePositions) {
+      // we don't process pipeline stages that contain label parsers, line filters or label filters
+      if (otherNodePositions.some((position) => pipelineStagePosition.contains(position))) {
+        continue;
+      }
+
+      newExpr += ` ${pipelineStagePosition.getExpression(origExpr)}`;
+    }
+
+    return newExpr;
+  };
+
+  queryContainsValidPipelineStages = (query: LokiQuery | undefined): boolean => {
+    const origExpr = query?.expr ?? '';
+    const allNodePositions = getNodePositionsFromQuery(origExpr, [
+      PipelineStage,
+      LabelParser,
+      LineFilters,
+      LabelFilter,
+    ]);
+    const pipelineStagePositions = allNodePositions.filter((position) => position.type?.id === PipelineStage);
+    const otherNodePositions = allNodePositions.filter((position) => position.type?.id !== PipelineStage);
+
+    return pipelineStagePositions.some((pipelineStagePosition) =>
+      otherNodePositions.every((position) => pipelineStagePosition.contains(position) === false)
+    );
+  };
+
   getInitContextFilters = async (labels: Labels, query?: LokiQuery) => {
     if (!query || isEmpty(labels)) {
       return [];

public/app/plugins/datasource/loki/components/LokiContextUi.test.tsx

@@ -5,10 +5,10 @@ import { selectOptionInTest } from 'test/helpers/selectOptionInTest';
 
 import { LogRowModel } from '@grafana/data';
 
-import { LogContextProvider } from '../LogContextProvider';
+import { LogContextProvider, SHOULD_INCLUDE_PIPELINE_OPERATIONS } from '../LogContextProvider';
 import { ContextFilter, LokiQuery } from '../types';
 
-import { LokiContextUi, LokiContextUiProps } from './LokiContextUi';
+import { IS_LOKI_LOG_CONTEXT_UI_OPEN, LokiContextUi, LokiContextUiProps } from './LokiContextUi';
 
 // we have to mock out reportInteraction, otherwise it crashes the test.
 jest.mock('@grafana/runtime', () => ({
@@ -19,8 +19,13 @@ jest.mock('@grafana/runtime', () => ({
 jest.mock('app/core/store', () => {
   return {
     set() {},
-    getBool() {
-      return true;
+    getBool(key: string, defaultValue?: boolean) {
+      const item = window.localStorage.getItem(key);
+      if (item === null) {
+        return defaultValue;
+      } else {
+        return item === 'true';
+      }
     },
     delete() {},
   };
@@ -28,7 +33,7 @@ jest.mock('app/core/store', () => {
 
 const setupProps = (): LokiContextUiProps => {
   const defaults: LokiContextUiProps = {
-    logContextProvider: mockLogContextProvider as unknown as LogContextProvider,
+    logContextProvider: Object.assign({}, mockLogContextProvider) as unknown as LogContextProvider,
     updateFilter: jest.fn(),
     row: {
       entry: 'WARN test 1.23 on [xxx]',
@@ -42,6 +47,7 @@ const setupProps = (): LokiContextUiProps => {
       expr: '{label1="value1"} | logfmt',
       refId: 'A',
     },
+    runContextQuery: jest.fn(),
   };
 
   return defaults;
@@ -55,13 +61,24 @@ const mockLogContextProvider = {
     ])
   ),
   processContextFiltersToExpr: jest.fn().mockImplementation(
-    (row: LogRowModel, contextFilters: ContextFilter[], query: LokiQuery | undefined) =>
+    (contextFilters: ContextFilter[], query: LokiQuery | undefined) =>
       `{${contextFilters
         .filter((filter) => filter.enabled)
         .map((filter) => `${filter.label}="${filter.value}"`)
         .join('` ')}}`
   ),
+  processPipelineStagesToExpr: jest
+    .fn()
+    .mockImplementation((currentExpr: string, query: LokiQuery | undefined) => `${currentExpr} | newOperation`),
   getLogRowContext: jest.fn(),
+  queryContainsValidPipelineStages: jest.fn().mockReturnValue(true),
+  prepareExpression: jest.fn().mockImplementation(
+    (contextFilters: ContextFilter[], query: LokiQuery | undefined) =>
+      `{${contextFilters
+        .filter((filter) => filter.enabled)
+        .map((filter) => `${filter.label}="${filter.value}"`)
+        .join('` ')}}`
+  ),
 };
 
 describe('LokiContextUi', () => {
@@ -80,6 +97,15 @@ describe('LokiContextUi', () => {
     global = savedGlobal;
   });
 
+  beforeEach(() => {
+    window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
+    window.localStorage.setItem(IS_LOKI_LOG_CONTEXT_UI_OPEN, 'true');
+  });
+
+  afterEach(() => {
+    window.localStorage.clear();
+  });
+
   it('renders and shows executed query text', async () => {
     const props = setupProps();
     render(<LokiContextUi {...props} />);
@@ -193,6 +219,72 @@ describe('LokiContextUi', () => {
     });
   });
 
+  it('renders pipeline operations switch as enabled when saved in localstorage', async () => {
+    const props = setupProps();
+    const newProps = {
+      ...props,
+      origQuery: {
+        expr: '{label1="value1"} | logfmt',
+        refId: 'A',
+      },
+    };
+    window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
+    render(<LokiContextUi {...newProps} />);
+    await waitFor(() => {
+      expect((screen.getByRole('checkbox') as HTMLInputElement).checked).toBe(true);
+    });
+  });
+
+  it('renders pipeline operations switch as disabled when saved in localstorage', async () => {
+    const props = setupProps();
+    const newProps = {
+      ...props,
+      origQuery: {
+        expr: '{label1="value1"} | logfmt',
+        refId: 'A',
+      },
+    };
+    window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'false');
+    render(<LokiContextUi {...newProps} />);
+    await waitFor(() => {
+      expect((screen.getByRole('checkbox') as HTMLInputElement).checked).toBe(false);
+    });
+  });
+
+  it('renders pipeline operations switch if query contains valid pipeline stages', async () => {
+    const props = setupProps();
+    (props.logContextProvider.queryContainsValidPipelineStages as jest.Mock).mockReturnValue(true);
+    const newProps = {
+      ...props,
+      origQuery: {
+        expr: '{label1="value1"} | logfmt',
+        refId: 'A',
+      },
+    };
+    window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
+    render(<LokiContextUi {...newProps} />);
+    await waitFor(() => {
+      expect(screen.getByRole('checkbox')).toBeInTheDocument();
+    });
+  });
+
+  it('does not render pipeline operations switch if query does not contain valid pipeline stages', async () => {
+    const props = setupProps();
+    (props.logContextProvider.queryContainsValidPipelineStages as jest.Mock).mockReturnValue(false);
+    const newProps = {
+      ...props,
+      origQuery: {
+        expr: '{label1="value1"} | logfmt',
+        refId: 'A',
+      },
+    };
+    window.localStorage.setItem(SHOULD_INCLUDE_PIPELINE_OPERATIONS, 'true');
+    render(<LokiContextUi {...newProps} />);
+    await waitFor(() => {
+      expect(screen.queryByRole('checkbox')).toBeNull();
+    });
+  });
+
   it('does not show parsed labels section if origQuery has 2 parsers', async () => {
     const props = setupProps();
     const newProps = {

public/app/plugins/datasource/loki/components/LokiContextUi.tsx

@@ -2,13 +2,31 @@ import { css } from '@emotion/css';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import { useAsync } from 'react-use';
 
-import { GrafanaTheme2, LogRowModel, SelectableValue } from '@grafana/data';
+import { GrafanaTheme2, LogRowModel, renderMarkdown, SelectableValue } from '@grafana/data';
 import { reportInteraction } from '@grafana/runtime';
-import { Button, Collapse, Icon, Label, MultiSelect, Spinner, Tooltip, useStyles2 } from '@grafana/ui';
+import {
+  Button,
+  Collapse,
+  Icon,
+  InlineField,
+  InlineFieldRow,
+  InlineSwitch,
+  Label,
+  MultiSelect,
+  RenderUserContentAsHTML,
+  Spinner,
+  Tooltip,
+  useStyles2,
+} from '@grafana/ui';
 import store from 'app/core/store';
 
 import { RawQuery } from '../../prometheus/querybuilder/shared/RawQuery';
-import { LogContextProvider, LOKI_LOG_CONTEXT_PRESERVED_LABELS, PreservedLabels } from '../LogContextProvider';
+import {
+  LogContextProvider,
+  LOKI_LOG_CONTEXT_PRESERVED_LABELS,
+  PreservedLabels,
+  SHOULD_INCLUDE_PIPELINE_OPERATIONS,
+} from '../LogContextProvider';
 import { escapeLabelValueInSelector } from '../languageUtils';
 import { isQueryWithParser } from '../queryUtils';
 import { lokiGrammar } from '../syntax';
@@ -20,6 +38,7 @@ export interface LokiContextUiProps {
   updateFilter: (value: ContextFilter[]) => void;
   onClose: () => void;
   origQuery?: LokiQuery;
+  runContextQuery?: () => void;
 }
 
 function getStyles(theme: GrafanaTheme2) {
@@ -74,13 +93,22 @@ function getStyles(theme: GrafanaTheme2) {
       right: ${theme.spacing(1)};
       z-index: ${theme.zIndex.navbarFixed};
     `,
+    operationsToggle: css`
+      margin: ${theme.spacing(1)} 0 ${theme.spacing(-1)} 0;
+      & > div {
+        margin: 0;
+        & > label {
+          padding: 0;
+        }
+      }
+    `,
   };
 }
 
-const IS_LOKI_LOG_CONTEXT_UI_OPEN = 'isLogContextQueryUiOpen';
+export const IS_LOKI_LOG_CONTEXT_UI_OPEN = 'isLogContextQueryUiOpen';
 
 export function LokiContextUi(props: LokiContextUiProps) {
-  const { row, logContextProvider, updateFilter, onClose, origQuery } = props;
+  const { row, logContextProvider, updateFilter, onClose, origQuery, runContextQuery } = props;
   const styles = useStyles2(getStyles);
 
   const [contextFilters, setContextFilters] = useState<ContextFilter[]>([]);
@@ -88,19 +116,27 @@ export function LokiContextUi(props: LokiContextUiProps) {
   const [initialized, setInitialized] = useState(false);
   const [loading, setLoading] = useState(false);
   const [isOpen, setIsOpen] = useState(store.getBool(IS_LOKI_LOG_CONTEXT_UI_OPEN, false));
+  const [includePipelineOperations, setIncludePipelineOperations] = useState(
+    store.getBool(SHOULD_INCLUDE_PIPELINE_OPERATIONS, false)
+  );
 
   const timerHandle = React.useRef<number>();
   const previousInitialized = React.useRef<boolean>(false);
   const previousContextFilters = React.useRef<ContextFilter[]>([]);
 
-  const isInitialQuery = useMemo(() => {
+  const isInitialState = useMemo(() => {
     // Initial query has all regular labels enabled and all parsed labels disabled
     if (initialized && contextFilters.some((filter) => filter.fromParser === filter.enabled)) {
       return false;
     }
 
+    // if we include pipeline operations, we also want to enable the revert button
+    if (includePipelineOperations && logContextProvider.queryContainsValidPipelineStages(origQuery)) {
+      return false;
+    }
+
     return true;
-  }, [contextFilters, initialized]);
+  }, [contextFilters, includePipelineOperations, initialized, logContextProvider, origQuery]);
 
   useEffect(() => {
     if (!initialized) {
@@ -200,6 +236,10 @@ export function LokiContextUi(props: LokiContextUiProps) {
   // Currently we support adding of parser and showing parsed labels only if there is 1 parser
   const showParsedLabels = origQuery && isQueryWithParser(origQuery.expr).parserCount === 1 && parsedLabels.length > 0;
 
+  let queryExpr = logContextProvider.prepareExpression(
+    contextFilters.filter(({ enabled }) => enabled),
+    origQuery
+  );
   return (
     <div className={styles.wrapper}>
       <Tooltip content={'Revert to initial log context query.'}>
@@ -208,7 +248,7 @@ export function LokiContextUi(props: LokiContextUiProps) {
             data-testid="revert-button"
             icon="history-alt"
             variant="secondary"
-            disabled={isInitialQuery}
+            disabled={isInitialState}
             onClick={(e) => {
               reportInteraction('grafana_explore_logs_loki_log_context_reverted', {
                 logRowUid: row.uid,
@@ -222,6 +262,8 @@ export function LokiContextUi(props: LokiContextUiProps) {
               });
               // We are removing the preserved labels from local storage so we can preselect the labels in the UI
               store.delete(LOKI_LOG_CONTEXT_PRESERVED_LABELS);
+              store.delete(SHOULD_INCLUDE_PIPELINE_OPERATIONS);
+              setIncludePipelineOperations(false);
             }}
           />
         </div>
@@ -242,15 +284,7 @@ export function LokiContextUi(props: LokiContextUiProps) {
           <div className={styles.rawQueryContainer}>
             {initialized ? (
               <>
-                <RawQuery
-                  lang={{ grammar: lokiGrammar, name: 'loki' }}
-                  query={logContextProvider.processContextFiltersToExpr(
-                    row,
-                    contextFilters.filter(({ enabled }) => enabled),
-                    origQuery
-                  )}
-                  className={styles.rawQuery}
-                />
+                <RawQuery lang={{ grammar: lokiGrammar, name: 'loki' }} query={queryExpr} className={styles.rawQuery} />
                 <Tooltip content="The initial log context query is created from all labels defining the stream for the selected log line. Use the editor below to customize the log context query.">
                   <Icon name="info-circle" size="sm" className={styles.queryDescription} />
                 </Tooltip>
@@ -345,6 +379,37 @@ export function LokiContextUi(props: LokiContextUiProps) {
               />
             </>
           )}
+          {logContextProvider.queryContainsValidPipelineStages(origQuery) && (
+            <InlineFieldRow className={styles.operationsToggle}>
+              <InlineField
+                label="Include LogQL pipeline operations"
+                tooltip={
+                  <RenderUserContentAsHTML
+                    content={renderMarkdown(
+                      "This will include LogQL operations such as `line_format` or `label_format`. It won't include line or label filter operations."
+                    )}
+                  />
+                }
+              >
+                <InlineSwitch
+                  value={includePipelineOperations}
+                  showLabel={true}
+                  transparent={true}
+                  onChange={(e) => {
+                    reportInteraction('grafana_explore_logs_loki_log_context_pipeline_toggled', {
+                      logRowUid: row.uid,
+                      action: e.currentTarget.checked ? 'enable' : 'disable',
+                    });
+                    store.set(SHOULD_INCLUDE_PIPELINE_OPERATIONS, e.currentTarget.checked);
+                    setIncludePipelineOperations(e.currentTarget.checked);
+                    if (runContextQuery) {
+                      runContextQuery();
+                    }
+                  }}
+                />
+              </InlineField>
+            </InlineFieldRow>
+          )}
         </div>
       </Collapse>
     </div>

public/app/plugins/datasource/loki/modifyQuery.test.ts

@@ -1,8 +1,11 @@
+import { SyntaxNode } from '@lezer/common';
+
 import {
   addLabelFormatToQuery,
   addLabelToQuery,
   addNoPipelineErrorToQuery,
   addParserToQuery,
+  NodePosition,
   removeCommentsFromQuery,
 } from './modifyQuery';
 
@@ -187,3 +190,59 @@ describe('removeCommentsFromQuery', () => {
     expect(removeCommentsFromQuery(query)).toBe(expectedResult);
   });
 });
+
+describe('NodePosition', () => {
+  describe('contains', () => {
+    it('should return true if the position is contained within the current position', () => {
+      const position = new NodePosition(5, 10);
+      const containedPosition = new NodePosition(6, 9);
+      const result = position.contains(containedPosition);
+      expect(result).toBe(true);
+    });
+
+    it('should return false if the position is not contained within the current position', () => {
+      const position = new NodePosition(5, 10);
+      const outsidePosition = new NodePosition(11, 15);
+      const result = position.contains(outsidePosition);
+      expect(result).toBe(false);
+    });
+
+    it('should return true if the position is the same as the current position', () => {
+      const position = new NodePosition(5, 10);
+      const samePosition = new NodePosition(5, 10);
+      const result = position.contains(samePosition);
+      expect(result).toBe(true);
+    });
+  });
+
+  describe('getExpression', () => {
+    it('should return the substring of the query within the given position', () => {
+      const position = new NodePosition(7, 12);
+      const query = 'Hello, world!';
+      const result = position.getExpression(query);
+      expect(result).toBe('world');
+    });
+
+    it('should return an empty string if the position is out of range', () => {
+      const position = new NodePosition(15, 20);
+      const query = 'Hello, world!';
+      const result = position.getExpression(query);
+      expect(result).toBe('');
+    });
+  });
+
+  describe('fromNode', () => {
+    it('should create a new NodePosition instance from a SyntaxNode', () => {
+      const syntaxNode = {
+        from: 5,
+        to: 10,
+        type: 'identifier',
+      } as unknown as SyntaxNode;
+      const result = NodePosition.fromNode(syntaxNode);
+      expect(result).toBeInstanceOf(NodePosition);
+      expect(result.from).toBe(5);
+      expect(result.to).toBe(10);
+      expect(result.type).toBe('identifier');
+    });
+  });
+});

public/app/plugins/datasource/loki/modifyQuery.ts

@@ -1,4 +1,4 @@
-import { SyntaxNode } from '@lezer/common';
+import { NodeType, SyntaxNode } from '@lezer/common';
 import { sortBy } from 'lodash';
 
 import {
@@ -22,7 +22,29 @@ import { unescapeLabelValue } from './languageUtils';
 import { LokiQueryModeller } from './querybuilder/LokiQueryModeller';
 import { buildVisualQueryFromString } from './querybuilder/parsing';
 
-export type Position = { from: number; to: number };
+export class NodePosition {
+  from: number;
+  to: number;
+  type?: NodeType;
+
+  constructor(from: number, to: number, type?: NodeType) {
+    this.from = from;
+    this.to = to;
+    this.type = type;
+  }
+
+  static fromNode(node: SyntaxNode): NodePosition {
+    return new NodePosition(node.from, node.to, node.type);
+  }
+
+  contains(position: NodePosition): boolean {
+    return this.from <= position.from && this.to >= position.to;
+  }
+
+  getExpression(query: string): string {
+    return query.substring(this.from, this.to);
+  }
+}
 /**
  * Adds label filter to existing query. Useful for query modification for example for ad hoc filters.
  *
@@ -139,13 +161,13 @@ export function removeCommentsFromQuery(query: string): string {
  * selector.
  * @param query
  */
-export function getStreamSelectorPositions(query: string): Position[] {
+export function getStreamSelectorPositions(query: string): NodePosition[] {
   const tree = parser.parse(query);
-  const positions: Position[] = [];
+  const positions: NodePosition[] = [];
   tree.iterate({
     enter: ({ type, from, to }): false | void => {
       if (type.id === Selector) {
-        positions.push({ from, to });
+        positions.push(new NodePosition(from, to, type));
         return false;
       }
     },
@@ -153,9 +175,9 @@ export function getStreamSelectorPositions(query: string): Position[] {
   return positions;
 }
 
-function getMatcherInStreamPositions(query: string): Position[] {
+function getMatcherInStreamPositions(query: string): NodePosition[] {
   const tree = parser.parse(query);
-  const positions: Position[] = [];
+  const positions: NodePosition[] = [];
   tree.iterate({
     enter: ({ node }): false | void => {
       if (node.type.id === Selector) {
@@ -170,13 +192,13 @@ function getMatcherInStreamPositions(query: string): Position[] {
  * Parse the string and get all LabelParser positions in the query.
  * @param query
  */
-export function getParserPositions(query: string): Position[] {
+export function getParserPositions(query: string): NodePosition[] {
   const tree = parser.parse(query);
-  const positions: Position[] = [];
+  const positions: NodePosition[] = [];
   tree.iterate({
     enter: ({ type, from, to }): false | void => {
       if (type.id === LabelParser || type.id === JsonExpressionParser) {
-        positions.push({ from, to });
+        positions.push(new NodePosition(from, to, type));
         return false;
       }
     },
@@ -188,13 +210,13 @@ export function getParserPositions(query: string): Position[] {
  * Parse the string and get all LabelFilter positions in the query.
  * @param query
  */
-export function getLabelFilterPositions(query: string): Position[] {
+export function getLabelFilterPositions(query: string): NodePosition[] {
   const tree = parser.parse(query);
-  const positions: Position[] = [];
+  const positions: NodePosition[] = [];
   tree.iterate({
     enter: ({ type, from, to }): false | void => {
       if (type.id === LabelFilter) {
-        positions.push({ from, to });
+        positions.push(new NodePosition(from, to, type));
         return false;
       }
     },
@@ -206,13 +228,13 @@ export function getLabelFilterPositions(query: string): Position[] {
  * Parse the string and get all Line filter positions in the query.
  * @param query
  */
-function getLineFiltersPositions(query: string): Position[] {
+function getLineFiltersPositions(query: string): NodePosition[] {
   const tree = parser.parse(query);
-  const positions: Position[] = [];
+  const positions: NodePosition[] = [];
   tree.iterate({
-    enter: ({ type, node }): false | void => {
+    enter: ({ type, from, to }): false | void => {
       if (type.id === LineFilters) {
-        positions.push({ from: node.from, to: node.to });
+        positions.push(new NodePosition(from, to, type));
         return false;
       }
     },
@@ -224,13 +246,13 @@ function getLineFiltersPositions(query: string): Position[] {
  * Parse the string and get all Log query positions in the query.
  * @param query
  */
-function getLogQueryPositions(query: string): Position[] {
+function getLogQueryPositions(query: string): NodePosition[] {
   const tree = parser.parse(query);
-  const positions: Position[] = [];
+  const positions: NodePosition[] = [];
   tree.iterate({
     enter: ({ type, from, to, node }): false | void => {
       if (type.id === LogExpr) {
-        positions.push({ from, to });
+        positions.push(new NodePosition(from, to, type));
         return false;
       }
 
@@ -238,25 +260,25 @@ function getLogQueryPositions(query: string): Position[] {
       if (type.id === LogRangeExpr) {
         // Unfortunately, LogRangeExpr includes both log and non-log (e.g. Duration/Range/...) parts of query.
         // We get position of all log-parts within LogRangeExpr: Selector, PipelineExpr and UnwrapExpr.
-        const logPartsPositions: Position[] = [];
+        const logPartsPositions: NodePosition[] = [];
         const selector = node.getChild(Selector);
         if (selector) {
-          logPartsPositions.push({ from: selector.from, to: selector.to });
+          logPartsPositions.push(NodePosition.fromNode(selector));
         }
 
         const pipeline = node.getChild(PipelineExpr);
         if (pipeline) {
-          logPartsPositions.push({ from: pipeline.from, to: pipeline.to });
+          logPartsPositions.push(NodePosition.fromNode(pipeline));
         }
 
         const unwrap = node.getChild(UnwrapExpr);
         if (unwrap) {
-          logPartsPositions.push({ from: unwrap.from, to: unwrap.to });
+          logPartsPositions.push(NodePosition.fromNode(unwrap));
         }
 
         // We sort them and then pick "from" from first position and "to" from last position.
         const sorted = sortBy(logPartsPositions, (position) => position.to);
-        positions.push({ from: sorted[0].from, to: sorted[sorted.length - 1].to });
+        positions.push(new NodePosition(sorted[0].from, sorted[sorted.length - 1].to));
         return false;
       }
     },
@@ -277,7 +299,7 @@ export function toLabelFilter(key: string, value: string, operator: string): Que
  */
 function addFilterToStreamSelector(
   query: string,
-  vectorSelectorPositions: Position[],
+  vectorSelectorPositions: NodePosition[],
   filter: QueryBuilderLabelFilter
 ): string {
   const modeller = new LokiQueryModeller();
@@ -313,7 +335,7 @@ function addFilterToStreamSelector(
  */
 export function addFilterAsLabelFilter(
   query: string,
-  positionsToAddAfter: Position[],
+  positionsToAddAfter: NodePosition[],
   filter: QueryBuilderLabelFilter
 ): string {
   let newQuery = '';
@@ -349,7 +371,7 @@ export function addFilterAsLabelFilter(
  * @param queryPartPositions
  * @param parser
  */
-function addParser(query: string, queryPartPositions: Position[], parser: string): string {
+function addParser(query: string, queryPartPositions: NodePosition[], parser: string): string {
   let newQuery = '';
   let prev = 0;
 
@@ -376,7 +398,7 @@ function addParser(query: string, queryPartPositions: Position[], parser: string
  */
 function addLabelFormat(
   query: string,
-  logQueryPositions: Position[],
+  logQueryPositions: NodePosition[],
   labelFormat: { originalLabel: string; renameTo: string }
 ): string {
   let newQuery = '';
@@ -405,13 +427,13 @@ export function addLineFilter(query: string): string {
   return newQueryExpr;
 }
 
-function getLineCommentPositions(query: string): Position[] {
+function getLineCommentPositions(query: string): NodePosition[] {
   const tree = parser.parse(query);
-  const positions: Position[] = [];
+  const positions: NodePosition[] = [];
   tree.iterate({
     enter: ({ type, from, to }): false | void => {
       if (type.id === LineComment) {
-        positions.push({ from, to });
+        positions.push(new NodePosition(from, to, type));
         return false;
       }
     },
@@ -432,16 +454,16 @@ function labelExists(labels: QueryBuilderLabelFilter[], filter: QueryBuilderLabe
  * Return the last position based on "to" property
  * @param positions
  */
-export function findLastPosition(positions: Position[]): Position {
+export function findLastPosition(positions: NodePosition[]): NodePosition {
   return positions.reduce((prev, current) => (prev.to > current.to ? prev : current));
 }
 
-function getAllPositionsInNodeByType(query: string, node: SyntaxNode, type: number): Position[] {
+function getAllPositionsInNodeByType(query: string, node: SyntaxNode, type: number): NodePosition[] {
   if (node.type.id === type) {
-    return [{ from: node.from, to: node.to }];
+    return [NodePosition.fromNode(node)];
   }
 
-  const positions: Position[] = [];
+  const positions: NodePosition[] = [];
   let pos = 0;
   let child = node.childAfter(pos);
   while (child) {

public/app/plugins/datasource/loki/queryUtils.test.ts

@@ -1,3 +1,5 @@
+import { String } from '@grafana/lezer-logql';
+
 import {
   getHighlighterExpressionsFromQuery,
   getLokiQueryType,
@@ -14,6 +16,7 @@ import {
   isQueryPipelineErrorFiltering,
   getLogQueryFromMetricsQuery,
   getNormalizedLokiQuery,
+  getNodePositionsFromQuery,
 } from './queryUtils';
 import { LokiQuery, LokiQueryType } from './types';
 
@@ -416,3 +419,24 @@ describe('getLogQueryFromMetricsQuery', () => {
     ).toBe('{label="$var"} | logfmt | __error__=``');
   });
 });
+
+describe('getNodePositionsFromQuery', () => {
+  it('returns the right amount of positions without type', () => {
+    // LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
+    expect(getNodePositionsFromQuery('{job="grafana"}').length).toBe(9);
+  });
+
+  it('returns the right position of a string in a stream selector', () => {
+    // LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
+    const nodePositions = getNodePositionsFromQuery('{job="grafana"}', [String]);
+    expect(nodePositions.length).toBe(1);
+    expect(nodePositions[0].from).toBe(5);
+    expect(nodePositions[0].to).toBe(14);
+  });
+
+  it('returns an empty array with a wrong expr', () => {
+    // LogQL, Expr, LogExpr, Selector, Matchers, Matcher, Identifier, Eq, String
+    const nodePositions = getNodePositionsFromQuery('not loql', [String]);
+    expect(nodePositions.length).toBe(0);
+  });
+});

public/app/plugins/datasource/loki/queryUtils.ts

@@ -24,7 +24,7 @@ import { DataQuery } from '@grafana/schema';
 
 import { ErrorId } from '../prometheus/querybuilder/shared/parsingUtils';
 
-import { getStreamSelectorPositions } from './modifyQuery';
+import { getStreamSelectorPositions, NodePosition } from './modifyQuery';
 import { LokiQuery, LokiQueryType } from './types';
 
 export function formatQuery(selector: string | undefined): string {
@@ -145,12 +145,12 @@ export function isQueryWithNode(query: string, nodeType: number): boolean {
   return isQueryWithNode;
 }
 
-export function getNodesFromQuery(query: string, nodeTypes: number[]): SyntaxNode[] {
+export function getNodesFromQuery(query: string, nodeTypes?: number[]): SyntaxNode[] {
   const nodes: SyntaxNode[] = [];
   const tree = parser.parse(query);
   tree.iterate({
     enter: (node): false | void => {
-      if (nodeTypes.includes(node.type.id)) {
+      if (nodeTypes === undefined || nodeTypes.includes(node.type.id)) {
         nodes.push(node.node);
       }
     },
@@ -158,6 +158,19 @@ export function getNodesFromQuery(query: string, nodeTypes: number[]): SyntaxNod
   return nodes;
 }
 
+export function getNodePositionsFromQuery(query: string, nodeTypes?: number[]): NodePosition[] {
+  const positions: NodePosition[] = [];
+  const tree = parser.parse(query);
+  tree.iterate({
+    enter: (node): false | void => {
+      if (nodeTypes === undefined || nodeTypes.includes(node.type.id)) {
+        positions.push(NodePosition.fromNode(node.node));
+      }
+    },
+  });
+  return positions;
+}
+
 export function getNodeFromQuery(query: string, nodeType: number): SyntaxNode | undefined {
   const nodes = getNodesFromQuery(query, [nodeType]);
   return nodes.length > 0 ? nodes[0] : undefined;