feat(logql): Support drop labels in logql pipeline (#7975)

This PR introduces `drop` stage in logql pipeline. 

Fixes #7870, Fixes #7368

CHANGELOG.md

@@ -19,6 +19,7 @@
 * [7964](https://github.com/grafana/loki/pull/7964) **slim-bean**: Add a `since` query parameter to allow querying based on relative time.
 * [7989](https://github.com/grafana/loki/pull/7989) **liguozhong**: logql support `sort` and `sort_desc`.
 * [7997](https://github.com/grafana/loki/pull/7997) **kavirajk**: fix(promtail): Fix cri tags extra new lines when joining partial lines
+* [7975](https://github.com/grafana/loki/pull/7975) **adityacs**: Support drop labels in logql
 * [7946](https://github.com/grafana/loki/pull/7946) **ashwanthgoli** config: Add support for named stores
 * [8027](https://github.com/grafana/loki/pull/8027) **kavirajk**: chore(promtail): Make `batchwait` and `batchsize` config explicit with yaml tags
 * [7978](https://github.com/grafana/loki/pull/7978) **chaudum**: Shut down query frontend gracefully to allow inflight requests to complete.

docs/sources/logql/log_queries.md

@@ -556,3 +556,55 @@ In both cases, if the destination label doesn't exist, then a new one is created
 The renaming form `dst=src` will _drop_ the `src` label after remapping it to the `dst` label. However, the _template_ form will preserve the referenced labels, such that  `dst="{{.src}}"` results in both `dst` and `src` having the same value.
 
 > A single label name can only appear once per expression. This means `| label_format foo=bar,foo="new"` is not allowed but you can use two expressions for the desired effect: `| label_format foo=bar | label_format foo="new"`
+
+### Drop Labels expression
+
+**Syntax**:  `|drop name, other_name, some_name="some_value"`
+
+The `=` operator after the label name is a **label matching operator**.
+The following label matching operators are supported:
+
+- `=`: exactly equal
+- `!=`: not equal
+- `=~`: regex matches
+- `!~`: regex does not match
+
+The `| drop` expression will drop the given labels in the pipeline. For example, for the query `{job="varlogs"}|json|drop level, method="GET"`, with below log line
+
+```
+{"level": "info", "method": "GET", "path": "/", "host": "grafana.net", "status": "200"}
+```
+
+the result will be
+
+```
+{host="grafana.net", path="status="200"} {"level": "info", "method": "GET", "path": "/", "host": "grafana.net", "status": "200"}
+```
+
+Similary, this expression can be used to drop `__error__` labels as well. For example, for the query `{job="varlogs"}|json|drop __error__`, with below log line
+
+```
+INFO GET / loki.net 200
+```
+
+the result will be
+
+```
+{} INFO GET / loki.net 200
+```
+
+Example with regex and multiple names
+
+For the query `{job="varlogs"}|json|drop level, path, app=~"some-api.*"`, with below log lines
+
+```
+{"app": "some-api-service", "level": "info", "method": "GET", "path": "/", "host": "grafana.net", "status": "200}
+{"app: "other-service", "level": "info", "method": "GET", "path": "/", "host": "grafana.net", "status": "200}
+```
+
+the result will be
+
+```
+{host="grafana.net", job="varlogs", method="GET", status="200"} {""app": "some-api-service",", "level": "info", "method": "GET", "path": "/", "host": "grafana.net", "status": "200"}
+{app="other-service", host="grafana.net", job="varlogs", method="GET", status="200"} {"app": "other-service",, "level": "info", "method": "GET", "path": "/", "host": "grafana.net", "status": "200"}
+```

pkg/logql/log/drop_labels.go

@@ -0,0 +1,85 @@
+package log
+
+import (
+	"github.com/grafana/loki/pkg/logqlmodel"
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+type DropLabels struct {
+	dropLabels []DropLabel
+}
+
+type DropLabel struct {
+	Matcher *labels.Matcher
+	Name    string
+}
+
+func NewDropLabel(matcher *labels.Matcher, name string) DropLabel {
+	return DropLabel{
+		Matcher: matcher,
+		Name:    name,
+	}
+}
+
+func NewDropLabels(dl []DropLabel) *DropLabels {
+	return &DropLabels{dropLabels: dl}
+}
+
+func (dl *DropLabels) Process(ts int64, line []byte, lbls *LabelsBuilder) ([]byte, bool) {
+	for _, dropLabel := range dl.dropLabels {
+		if dropLabel.Matcher != nil {
+			dropLabelMatches(dropLabel.Matcher, lbls)
+			continue
+		}
+		name := dropLabel.Name
+		dropLabelNames(name, lbls)
+	}
+	return line, true
+}
+
+func (dl *DropLabels) RequiredLabelNames() []string { return []string{} }
+
+func isErrorLabel(name string) bool {
+	return name == logqlmodel.ErrorLabel
+}
+
+func isErrorDetailsLabel(name string) bool {
+	return name == logqlmodel.ErrorDetailsLabel
+}
+
+func dropLabelNames(name string, lbls *LabelsBuilder) {
+	if isErrorLabel(name) {
+		lbls.ResetError()
+		return
+	}
+	if isErrorDetailsLabel(name) {
+		lbls.ResetErrorDetails()
+		return
+	}
+	if _, ok := lbls.Get(name); ok {
+		lbls.Del(name)
+	}
+}
+
+func dropLabelMatches(matcher *labels.Matcher, lbls *LabelsBuilder) {
+	var value string
+	name := matcher.Name
+	if isErrorLabel(name) {
+		value = lbls.GetErr()
+		if matcher.Matches(value) {
+			lbls.ResetError()
+		}
+		return
+	}
+	if isErrorDetailsLabel(name) {
+		value = lbls.GetErrorDetails()
+		if matcher.Matches(value) {
+			lbls.ResetErrorDetails()
+		}
+		return
+	}
+	value, _ = lbls.Get(name)
+	if matcher.Matches(value) {
+		lbls.Del(name)
+	}
+}

pkg/logql/log/drop_labels_test.go

@@ -0,0 +1,160 @@
+package log
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/loki/pkg/logqlmodel"
+)
+
+func Test_DropLabels(t *testing.T) {
+	tests := []struct {
+		Name       string
+		dropLabels []DropLabel
+		err        string
+		errDetails string
+		lbs        labels.Labels
+		want       labels.Labels
+	}{
+		{
+			"drop by name",
+			[]DropLabel{
+				{
+					nil,
+					"app",
+				},
+				{
+					nil,
+					"namespace",
+				},
+			},
+			"",
+			"",
+			labels.Labels{
+				{Name: "app", Value: "foo"},
+				{Name: "namespace", Value: "prod"},
+				{Name: "pod_uuid", Value: "foo"},
+			},
+			labels.Labels{
+				{Name: "pod_uuid", Value: "foo"},
+			},
+		},
+		{
+			"drop by __error__",
+			[]DropLabel{
+				{
+					labels.MustNewMatcher(labels.MatchEqual, logqlmodel.ErrorLabel, errJSON),
+					"",
+				},
+				{
+					nil,
+					"__error_details__",
+				},
+			},
+			errJSON,
+			"json error",
+			labels.Labels{
+				{Name: "app", Value: "foo"},
+				{Name: "namespace", Value: "prod"},
+				{Name: "pod_uuid", Value: "foo"},
+			},
+			labels.Labels{
+				{Name: "app", Value: "foo"},
+				{Name: "namespace", Value: "prod"},
+				{Name: "pod_uuid", Value: "foo"},
+			},
+		},
+		{
+			"drop with wrong __error__ value",
+			[]DropLabel{
+				{
+					labels.MustNewMatcher(labels.MatchEqual, logqlmodel.ErrorLabel, errLogfmt),
+					"",
+				},
+			},
+			errJSON,
+			"json error",
+			labels.Labels{
+				{Name: "app", Value: "foo"},
+				{Name: "namespace", Value: "prod"},
+				{Name: "pod_uuid", Value: "foo"},
+			},
+			labels.Labels{
+				{Name: "app", Value: "foo"},
+				{Name: "namespace", Value: "prod"},
+				{Name: "pod_uuid", Value: "foo"},
+				{Name: logqlmodel.ErrorLabel, Value: errJSON},
+				{Name: logqlmodel.ErrorDetailsLabel, Value: "json error"},
+			},
+		},
+		{
+			"drop by __error_details__",
+			[]DropLabel{
+				{
+					labels.MustNewMatcher(labels.MatchRegexp, logqlmodel.ErrorDetailsLabel, "expecting json.*"),
+					"",
+				},
+				{
+					nil,
+					"__error__",
+				},
+			},
+			errJSON,
+			"expecting json object but it is not",
+			labels.Labels{
+				{Name: "app", Value: "foo"},
+				{Name: "namespace", Value: "prod"},
+				{Name: "pod_uuid", Value: "foo"},
+			},
+			labels.Labels{
+				{Name: "app", Value: "foo"},
+				{Name: "namespace", Value: "prod"},
+				{Name: "pod_uuid", Value: "foo"},
+			},
+		},
+		{
+			"drop labels with names and matcher",
+			[]DropLabel{
+				{
+					labels.MustNewMatcher(labels.MatchEqual, logqlmodel.ErrorLabel, errJSON),
+					"",
+				},
+				{
+					nil,
+					"__error_details__",
+				},
+				{
+					nil,
+					"app",
+				},
+				{
+					nil,
+					"namespace",
+				},
+			},
+			errJSON,
+			"json error",
+			labels.Labels{
+				{Name: "app", Value: "foo"},
+				{Name: "namespace", Value: "prod"},
+				{Name: "pod_uuid", Value: "foo"},
+			},
+			labels.Labels{
+				{Name: "pod_uuid", Value: "foo"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		dropLabels := NewDropLabels(tt.dropLabels)
+		lbls := NewBaseLabelsBuilder().ForLabels(tt.lbs, tt.lbs.Hash())
+		lbls.Reset()
+		lbls.SetErr(tt.err)
+		lbls.SetErrorDetails(tt.errDetails)
+		dropLabels.Process(0, []byte(""), lbls)
+		sort.Sort(tt.want)
+		require.Equal(t, tt.want, lbls.LabelsResult().Labels())
+	}
+}

pkg/logql/log/labels.go

@@ -166,6 +166,16 @@ func (b *LabelsBuilder) SetErrorDetails(desc string) *LabelsBuilder {
 	return b
 }
 
+func (b *LabelsBuilder) ResetError() *LabelsBuilder {
+	b.err = ""
+	return b
+}
+
+func (b *LabelsBuilder) ResetErrorDetails() *LabelsBuilder {
+	b.errDetails = ""
+	return b
+}
+
 func (b *LabelsBuilder) GetErrorDetails() string {
 	return b.errDetails
 }

pkg/logql/log/pipeline_test.go

@@ -1,6 +1,7 @@
 package log
 
 import (
+	"sort"
 	"testing"
 	"time"
 
@@ -134,6 +135,109 @@ var (
 	resSample     float64
 )
 
+func TestDropLabelsPipeline(t *testing.T) {
+	tests := []struct {
+		name       string
+		stages     []Stage
+		lines      [][]byte
+		wantLine   [][]byte
+		wantLabels []labels.Labels
+	}{
+		{
+			"drop __error__",
+			[]Stage{
+				NewLogfmtParser(),
+				NewJSONParser(),
+				NewDropLabels([]DropLabel{
+					{
+						nil,
+						"__error__",
+					},
+					{
+						nil,
+						"__error_details__",
+					},
+				}),
+			},
+			[][]byte{
+				[]byte(`level=info ts=2020-10-18T18:04:22.147378997Z caller=metrics.go:81 status=200`),
+				[]byte(`{"app":"foo","namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar"}}}`),
+			},
+			[][]byte{
+				[]byte(`level=info ts=2020-10-18T18:04:22.147378997Z caller=metrics.go:81 status=200`),
+				[]byte(`{"app":"foo","namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar"}}}`),
+			},
+			[]labels.Labels{
+				{
+					{Name: "level", Value: "info"},
+					{Name: "ts", Value: "2020-10-18T18:04:22.147378997Z"},
+					{Name: "caller", Value: "metrics.go:81"},
+					{Name: "status", Value: "200"},
+				},
+				{
+					{Name: "app", Value: "foo"},
+					{Name: "namespace", Value: "prod"},
+					{Name: "pod_uuid", Value: "foo"},
+					{Name: "pod_deployment_ref", Value: "foobar"},
+				},
+			},
+		},
+		{
+			"drop __error__ with matching value",
+			[]Stage{
+				NewLogfmtParser(),
+				NewJSONParser(),
+				NewDropLabels([]DropLabel{
+					{
+						labels.MustNewMatcher(labels.MatchEqual, logqlmodel.ErrorLabel, errLogfmt),
+						"",
+					},
+					{
+						labels.MustNewMatcher(labels.MatchEqual, "status", "200"),
+						"",
+					},
+					{
+						nil,
+						"app",
+					},
+				}),
+			},
+			[][]byte{
+				[]byte(`level=info ts=2020-10-18T18:04:22.147378997Z caller=metrics.go:81 status=200`),
+				[]byte(`{"app":"foo","namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar"}}}`),
+			},
+			[][]byte{
+				[]byte(`level=info ts=2020-10-18T18:04:22.147378997Z caller=metrics.go:81 status=200`),
+				[]byte(`{"app":"foo","namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar"}}}`),
+			},
+			[]labels.Labels{
+				{
+					{Name: "level", Value: "info"},
+					{Name: "ts", Value: "2020-10-18T18:04:22.147378997Z"},
+					{Name: "caller", Value: "metrics.go:81"},
+					{Name: logqlmodel.ErrorLabel, Value: errJSON},
+					{Name: logqlmodel.ErrorDetailsLabel, Value: "expecting json object(6), but it is not"},
+				},
+				{
+					{Name: "namespace", Value: "prod"},
+					{Name: "pod_uuid", Value: "foo"},
+					{Name: "pod_deployment_ref", Value: "foobar"},
+					{Name: logqlmodel.ErrorDetailsLabel, Value: "logfmt syntax error at pos 2 : unexpected '\"'"},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		p := NewPipeline(tt.stages)
+		sp := p.ForStream(labels.Labels{})
+		for i, line := range tt.lines {
+			_, finalLbs, _ := sp.Process(0, line)
+			sort.Sort(tt.wantLabels[i])
+			require.Equal(t, tt.wantLabels[i], finalLbs.Labels())
+		}
+	}
+
+}
 func Benchmark_Pipeline(b *testing.B) {
 	b.ReportAllocs()
 

pkg/logql/syntax/ast.go

@@ -446,6 +446,44 @@ func (e *DecolorizeExpr) String() string {
 }
 func (e *DecolorizeExpr) Walk(f WalkFn) { f(e) }
 
+type DropLabelsExpr struct {
+	dropLabels []log.DropLabel
+	implicit
+}
+
+func newDropLabelsExpr(dropLabels []log.DropLabel) *DropLabelsExpr {
+	return &DropLabelsExpr{dropLabels: dropLabels}
+}
+
+func (e *DropLabelsExpr) Shardable() bool { return true }
+
+func (e *DropLabelsExpr) Stage() (log.Stage, error) {
+	return log.NewDropLabels(e.dropLabels), nil
+}
+func (e *DropLabelsExpr) String() string {
+	var sb strings.Builder
+
+	sb.WriteString(fmt.Sprintf("%s %s ", OpPipe, OpDrop))
+
+	for i, dropLabel := range e.dropLabels {
+		if dropLabel.Matcher != nil {
+			sb.WriteString(dropLabel.Matcher.String())
+			if i+1 != len(e.dropLabels) {
+				sb.WriteString(",")
+			}
+		}
+		if dropLabel.Name != "" {
+			sb.WriteString(dropLabel.Name)
+			if i+1 != len(e.dropLabels) {
+				sb.WriteString(",")
+			}
+		}
+	}
+	str := sb.String()
+	return str
+}
+func (e *DropLabelsExpr) Walk(f WalkFn) { f(e) }
+
 func (e *LineFmtExpr) Shardable() bool { return true }
 
 func (e *LineFmtExpr) Walk(f WalkFn) { f(e) }
@@ -460,7 +498,6 @@ func (e *LineFmtExpr) String() string {
 
 type LabelFmtExpr struct {
 	Formats []log.LabelFmt
-
 	implicit
 }
 
@@ -480,7 +517,9 @@ func (e *LabelFmtExpr) Stage() (log.Stage, error) {
 
 func (e *LabelFmtExpr) String() string {
 	var sb strings.Builder
+
 	sb.WriteString(fmt.Sprintf("%s %s ", OpPipe, OpFmtLabel))
+
 	for i, f := range e.Formats {
 		sb.WriteString(f.Name)
 		sb.WriteString("=")
@@ -725,6 +764,9 @@ const (
 
 	// function filters
 	OpFilterIP = "ip"
+
+	// drop labels
+	OpDrop = "drop"
 )
 
 func IsComparisonOperator(op string) bool {

pkg/logql/syntax/expr.y

@@ -59,6 +59,9 @@ import (
   UnwrapExpr              *UnwrapExpr
   DecolorizeExpr          *DecolorizeExpr
   OffsetExpr              *OffsetExpr
+  DropLabel               log.DropLabel
+  DropLabels              []log.DropLabel
+  DropLabelsExpr          *DropLabelsExpr 
 }
 
 %start root
@@ -98,6 +101,9 @@ import (
 %type <LineFilter>            lineFilter
 %type <LineFormatExpr>        lineFormatExpr
 %type <DecolorizeExpr>        decolorizeExpr
+%type <DropLabelsExpr>        dropLabelsExpr
+%type <DropLabels>            dropLabels
+%type <DropLabel>             dropLabel
 %type <LabelFormatExpr>       labelFormatExpr
 %type <LabelFormat>           labelFormat
 %type <LabelsFormat>          labelsFormat
@@ -117,7 +123,7 @@ import (
                   BYTES_OVER_TIME BYTES_RATE BOOL JSON REGEXP LOGFMT PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME
                   MAX_OVER_TIME STDVAR_OVER_TIME STDDEV_OVER_TIME QUANTILE_OVER_TIME BYTES_CONV DURATION_CONV DURATION_SECONDS_CONV
                   FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME VECTOR LABEL_REPLACE UNPACK OFFSET PATTERN IP ON IGNORING GROUP_LEFT GROUP_RIGHT
-                  DECOLORIZE
+                  DECOLORIZE DROP
 
 // Operators are listed with increasing precedence.
 %left <binOp> OR
@@ -254,6 +260,7 @@ pipelineStage:
   | PIPE lineFormatExpr          { $$ = $2 }
   | PIPE decolorizeExpr          { $$ = $2 }
   | PIPE labelFormatExpr         { $$ = $2 }
+  | PIPE dropLabelsExpr          { $$ = $2 }
   ;
 
 filterOp:
@@ -296,7 +303,8 @@ labelsFormat:
   | labelsFormat COMMA error
   ;
 
-labelFormatExpr: LABEL_FMT labelsFormat { $$ = newLabelFmtExpr($2) };
+labelFormatExpr:
+      LABEL_FMT labelsFormat { $$ = newLabelFmtExpr($2) };
 
 labelFilter:
       matcher                                        { $$ = log.NewStringLabelFilter($1) }
@@ -358,6 +366,17 @@ numberFilter:
     | IDENTIFIER CMP_EQ NUMBER  { $$ = log.NewNumericLabelFilter(log.LabelFilterEqual, $1, mustNewFloat($3))}
     ;
 
+dropLabel: 
+      IDENTIFIER { $$ = log.NewDropLabel(nil, $1) }
+    | matcher { $$ = log.NewDropLabel($1, "") }
+
+dropLabels:
+      dropLabel                  { $$ = []log.DropLabel{$1}}
+    | dropLabels COMMA dropLabel { $$ = append($1, $3) }
+    ;
+
+dropLabelsExpr: DROP dropLabels { $$ = newDropLabelsExpr($2) }
+
 // Operator precedence only works if each of these is listed separately.
 binOpExpr:
          expr OR binOpModifier expr          { $$ = mustNewBinOpExpr("or", $3, $1, $4) }

pkg/logql/syntax/lex.go

@@ -72,6 +72,9 @@ var tokens = map[string]int{
 	// filter functions
 	OpFilterIP:   IP,
 	OpDecolorize: DECOLORIZE,
+
+	// drop labels
+	OpDrop: DROP,
 }
 
 // functionTokens are tokens that needs to be suffixes with parenthesis

pkg/logql/syntax/prettier.go

@@ -110,6 +110,10 @@ func (e *LabelParserExpr) Pretty(level int) string {
 	return commonPrefixIndent(level, e)
 }
 
+func (e *DropLabelsExpr) Pretty(level int) string {
+	return commonPrefixIndent(level, e)
+}
+
 // e.g: | level!="error"
 func (e *LabelFilterExpr) Pretty(level int) string {
 	return commonPrefixIndent(level, e)