apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

logql/log tests: use abstractions over labels (#9532)

Browse Source 
**What this PR does / why we need it**:

Call functions like `FromStrings`, so we don't have to know how the data
structure is laid out.

`FromStrings` sorts labels, so we don't have to call `Sort()`.

**Special notes for your reviewer**:

This PR only touches tests. 

Upstream Prometheus is using a different implementation of
`labels.Labels`, so I plan to create a series of PRs which make Loki
compatible with that.

**Checklist**
- [x] Reviewed the
[`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md)
guide (**required**)
- NA Documentation added
- [x] Tests updated
- NA `CHANGELOG.md` updated
- NA Changes that require user attention or interaction to upgrade are
documented in `docs/sources/upgrading/_index.md`
- NA For Helm chart changes bump the Helm chart version.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>
pull/9561/head
Bryan Boreham 3 years ago committed by GitHub
parent aad436b799
commit ee74fbdf3a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 758 additions and 1058 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 9
      pkg/logql/log/distinct_test.go
  2. 86
      pkg/logql/log/drop_labels_test.go
  3. 364
      pkg/logql/log/fmt_test.go
  4. 2
      pkg/logql/log/ip_test.go
  5. 224
      pkg/logql/log/label_filter_test.go
  6. 91
      pkg/logql/log/labels_test.go
  7. 165
      pkg/logql/log/metrics_extraction_test.go
  8. 8
      pkg/logql/log/parser_hints_test.go
  9. 738
      pkg/logql/log/parser_test.go
  10. 129
      pkg/logql/log/pipeline_test.go

9
pkg/logql/log/distinct_test.go
Unescape View File

@ -21,11 +21,10 @@ func Test_DistinctFilter(t *testing.T) {
}{
name: "distinct test",
label: []string{"id", "time", "none"},
lbs: labels.Labels{
{Name: logqlmodel.ErrorLabel, Value: errJSON},
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
lbs: labels.FromStrings(logqlmodel.ErrorLabel, errJSON,
"status", "200",
"method", "POST",
),
input: []string{
`{"event": "access", "id": "1", "time": "1"}`,
`{"event": "access", "id": "1", "time": "2"}`,

86
pkg/logql/log/drop_labels_test.go
Unescape View File

@ -1,7 +1,6 @@
package log
import (
"sort"
"testing"
"github.com/prometheus/prometheus/model/labels"
@ -33,14 +32,11 @@ func Test_DropLabels(t *testing.T) {
},
"",
"",
labels.Labels{
{Name: "app", Value: "foo"},
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
},
labels.Labels{
{Name: "pod_uuid", Value: "foo"},
},
labels.FromStrings("app", "foo",
"namespace", "prod",
"pod_uuid", "foo",
),
labels.FromStrings("pod_uuid", "foo"),
},
{
"drop by __error__",
@ -56,16 +52,14 @@ func Test_DropLabels(t *testing.T) {
},
errJSON,
"json error",
labels.Labels{
{Name: "app", Value: "foo"},
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
},
labels.Labels{
{Name: "app", Value: "foo"},
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
},
labels.FromStrings("app", "foo",
"namespace", "prod",
"pod_uuid", "foo",
),
labels.FromStrings("app", "foo",
"namespace", "prod",
"pod_uuid", "foo",
),
},
{
"drop with wrong __error__ value",
@ -77,18 +71,16 @@ func Test_DropLabels(t *testing.T) {
},
errJSON,
"json error",
labels.Labels{
{Name: "app", Value: "foo"},
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
},
labels.Labels{
{Name: "app", Value: "foo"},
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
{Name: logqlmodel.ErrorLabel, Value: errJSON},
{Name: logqlmodel.ErrorDetailsLabel, Value: "json error"},
},
labels.FromStrings("app", "foo",
"namespace", "prod",
"pod_uuid", "foo",
),
labels.FromStrings("app", "foo",
"namespace", "prod",
"pod_uuid", "foo",
logqlmodel.ErrorLabel, errJSON,
logqlmodel.ErrorDetailsLabel, "json error",
),
},
{
"drop by __error_details__",
@ -104,16 +96,14 @@ func Test_DropLabels(t *testing.T) {
},
errJSON,
"expecting json object but it is not",
labels.Labels{
{Name: "app", Value: "foo"},
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
},
labels.Labels{
{Name: "app", Value: "foo"},
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
},
labels.FromStrings("app", "foo",
"namespace", "prod",
"pod_uuid", "foo",
),
labels.FromStrings("app", "foo",
"namespace", "prod",
"pod_uuid", "foo",
),
},
{
"drop labels with names and matcher",
@ -137,14 +127,11 @@ func Test_DropLabels(t *testing.T) {
},
errJSON,
"json error",
labels.Labels{
{Name: "app", Value: "foo"},
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
},
labels.Labels{
{Name: "pod_uuid", Value: "foo"},
},
labels.FromStrings("app", "foo",
"namespace", "prod",
"pod_uuid", "foo",
),
labels.FromStrings("pod_uuid", "foo"),
},
}
for _, tt := range tests {
@ -154,7 +141,6 @@ func Test_DropLabels(t *testing.T) {
lbls.SetErr(tt.err)
lbls.SetErrorDetails(tt.errDetails)
dropLabels.Process(0, []byte(""), lbls)
sort.Sort(tt.want)
require.Equal(t, tt.want, lbls.LabelsResult().Labels())
}
}

364
pkg/logql/log/fmt_test.go
Unescape View File

@ -2,7 +2,6 @@ package log
import (
"fmt"
"sort"
"testing"
"github.com/prometheus/prometheus/model/labels"
@ -27,10 +26,10 @@ func Test_lineFormatter_Format(t *testing.T) {
newMustLineFormatter(
`{{.foo | count "abc" }}`,
),
labels.Labels{{Name: "foo", Value: "abc abc abc"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "abc abc abc", "bar", "blop"),
0,
[]byte("3"),
labels.Labels{{Name: "foo", Value: "abc abc abc"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "abc abc abc", "bar", "blop"),
nil,
},
{
@ -38,46 +37,46 @@ func Test_lineFormatter_Format(t *testing.T) {
newMustLineFormatter(
`{{.foo | count "a|b|c" }}`,
),
labels.Labels{{Name: "foo", Value: "abc abc abc"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "abc abc abc", "bar", "blop"),
0,
[]byte("9"),
labels.Labels{{Name: "foo", Value: "abc abc abc"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "abc abc abc", "bar", "blop"),
nil,
},
{
"combining",
newMustLineFormatter("foo{{.foo}}buzz{{ .bar }}"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
0,
[]byte("fooblipbuzzblop"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
nil,
},
{
"Replace",
newMustLineFormatter(`foo{{.foo}}buzz{{ Replace .bar "blop" "bar" -1 }}`),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
0,
[]byte("fooblipbuzzbar"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
nil,
},
{
"replace",
newMustLineFormatter(`foo{{.foo}}buzz{{ .bar | replace "blop" "bar" }}`),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
0,
[]byte("fooblipbuzzbar"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
nil,
},
{
"title",
newMustLineFormatter(`{{.foo | title }}`),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
0,
[]byte("Blip"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
nil,
},
{
@ -85,10 +84,10 @@ func Test_lineFormatter_Format(t *testing.T) {
newMustLineFormatter(
`{{.foo | substr 1 3 }} {{ .bar | trunc 1 }} {{ .bar | trunc 3 }}`,
),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
0,
[]byte("li b blo"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
nil,
},
{
@ -96,381 +95,373 @@ func Test_lineFormatter_Format(t *testing.T) {
newMustLineFormatter(
`{{.foo | trim }} {{ .bar | trimAll "op" }} {{ .bar | trimPrefix "b" }} {{ .bar | trimSuffix "p" }}`,
),
labels.Labels{{Name: "foo", Value: " blip "}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", " blip ", "bar", "blop"),
0,
[]byte("blip bl lop blo"),
labels.Labels{{Name: "foo", Value: " blip "}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", " blip ", "bar", "blop"),
nil,
},
{
"lower and upper",
newMustLineFormatter(`{{.foo | lower }} {{ .bar | upper }}`),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
0,
[]byte("blip BLOP"),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
nil,
},
{
"urlencode",
newMustLineFormatter(`{{.foo | urlencode }} {{ urlencode .foo }}`), // assert both syntax forms
labels.Labels{
{Name: "foo", Value: `/loki/api/v1/query?query=sum(count_over_time({stream_filter="some_stream",environment="prod", host=~"someec2.*"}`},
},
labels.FromStrings("foo", `/loki/api/v1/query?query=sum(count_over_time({stream_filter="some_stream",environment="prod", host=~"someec2.*"}`),
0,
[]byte("%2Floki%2Fapi%2Fv1%2Fquery%3Fquery%3Dsum%28count_over_time%28%7Bstream_filter%3D%22some_stream%22%2Cenvironment%3D%22prod%22%2C+host%3D~%22someec2.%2A%22%7D %2Floki%2Fapi%2Fv1%2Fquery%3Fquery%3Dsum%28count_over_time%28%7Bstream_filter%3D%22some_stream%22%2Cenvironment%3D%22prod%22%2C+host%3D~%22someec2.%2A%22%7D"),
labels.Labels{{Name: "foo", Value: `/loki/api/v1/query?query=sum(count_over_time({stream_filter="some_stream",environment="prod", host=~"someec2.*"}`}},
labels.FromStrings("foo", `/loki/api/v1/query?query=sum(count_over_time({stream_filter="some_stream",environment="prod", host=~"someec2.*"}`),
nil,
},
{
"urldecode",
newMustLineFormatter(`{{.foo | urldecode }} {{ urldecode .foo }}`), // assert both syntax forms
labels.Labels{
{Name: "foo", Value: `%2Floki%2Fapi%2Fv1%2Fquery%3Fquery%3Dsum%28count_over_time%28%7Bstream_filter%3D%22some_stream%22%2Cenvironment%3D%22prod%22%2C+host%3D~%22someec2.%2A%22%7D`},
},
labels.FromStrings("foo", `%2Floki%2Fapi%2Fv1%2Fquery%3Fquery%3Dsum%28count_over_time%28%7Bstream_filter%3D%22some_stream%22%2Cenvironment%3D%22prod%22%2C+host%3D~%22someec2.%2A%22%7D`),
0,
[]byte(`/loki/api/v1/query?query=sum(count_over_time({stream_filter="some_stream",environment="prod", host=~"someec2.*"} /loki/api/v1/query?query=sum(count_over_time({stream_filter="some_stream",environment="prod", host=~"someec2.*"}`),
labels.Labels{{Name: "foo", Value: `%2Floki%2Fapi%2Fv1%2Fquery%3Fquery%3Dsum%28count_over_time%28%7Bstream_filter%3D%22some_stream%22%2Cenvironment%3D%22prod%22%2C+host%3D~%22someec2.%2A%22%7D`}},
labels.FromStrings("foo", `%2Floki%2Fapi%2Fv1%2Fquery%3Fquery%3Dsum%28count_over_time%28%7Bstream_filter%3D%22some_stream%22%2Cenvironment%3D%22prod%22%2C+host%3D~%22someec2.%2A%22%7D`),
nil,
},
{
"repeat",
newMustLineFormatter(`{{ "foo" | repeat 3 }}`),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
0,
[]byte("foofoofoo"),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
nil,
},
{
"indent",
newMustLineFormatter(`{{ "foo\n bar" | indent 4 }}`),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
0,
[]byte(" foo\n bar"),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
nil,
},
{
"nindent",
newMustLineFormatter(`{{ "foo" | nindent 2 }}`),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
0,
[]byte("\n foo"),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
nil,
},
{
"contains",
newMustLineFormatter(`{{ if .foo | contains "p"}}yes{{end}}-{{ if .foo | contains "z"}}no{{end}}`),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
0,
[]byte("yes-"),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
nil,
},
{
"hasPrefix",
newMustLineFormatter(`{{ if .foo | hasPrefix "BL" }}yes{{end}}-{{ if .foo | hasPrefix "p"}}no{{end}}`),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
0,
[]byte("yes-"),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
nil,
},
{
"hasSuffix",
newMustLineFormatter(`{{ if .foo | hasSuffix "Ip" }}yes{{end}}-{{ if .foo | hasSuffix "pw"}}no{{end}}`),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
0,
[]byte("yes-"),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
nil,
},
{
"regexReplaceAll",
newMustLineFormatter(`{{ regexReplaceAll "(p)" .foo "t" }}`),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
0,
[]byte("BLIt"),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
nil,
},
{
"regexReplaceAllLiteral",
newMustLineFormatter(`{{ regexReplaceAllLiteral "(p)" .foo "${1}" }}`),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
0,
[]byte("BLI${1}"),
labels.Labels{{Name: "foo", Value: "BLIp"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "BLIp", "bar", "blop"),
nil,
},
{
"err",
newMustLineFormatter(`{{.foo Replace "foo"}}`),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
0,
nil,
labels.Labels{
{Name: "__error__", Value: "TemplateFormatErr"},
{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"},
{Name: "__error_details__", Value: "template: line:1:2: executing \"line\" at <.foo>: foo is not a method but has arguments"},
},
labels.FromStrings("__error__", "TemplateFormatErr",
"foo", "blip", "bar", "blop",
"__error_details__", "template: line:1:2: executing \"line\" at <.foo>: foo is not a method but has arguments",
),
nil,
},
{
"missing",
newMustLineFormatter("foo {{.foo}}buzz{{ .bar }}"),
labels.Labels{{Name: "bar", Value: "blop"}},
labels.FromStrings("bar", "blop"),
0,
[]byte("foo buzzblop"),
labels.Labels{{Name: "bar", Value: "blop"}},
labels.FromStrings("bar", "blop"),
nil,
},
{
"function",
newMustLineFormatter("foo {{.foo | ToUpper }} buzz{{ .bar }}"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
0,
[]byte("foo BLIP buzzblop"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
nil,
},
{
"mathint",
newMustLineFormatter("{{ add .foo 1 | sub .bar | mul .baz | div .bazz}}"),
labels.Labels{{Name: "foo", Value: "1"}, {Name: "bar", Value: "3"}, {Name: "baz", Value: "10"}, {Name: "bazz", Value: "20"}},
labels.FromStrings("foo", "1", "bar", "3", "baz", "10", "bazz", "20"),
0,
[]byte("2"),
labels.Labels{{Name: "foo", Value: "1"}, {Name: "bar", Value: "3"}, {Name: "baz", Value: "10"}, {Name: "bazz", Value: "20"}},
labels.FromStrings("foo", "1", "bar", "3", "baz", "10", "bazz", "20"),
nil,
},
{
"mathfloat",
newMustLineFormatter("{{ addf .foo 1.5 | subf .bar 1.5 | mulf .baz | divf .bazz }}"),
labels.Labels{{Name: "foo", Value: "1.5"}, {Name: "bar", Value: "5"}, {Name: "baz", Value: "10.5"}, {Name: "bazz", Value: "20.2"}},
labels.FromStrings("foo", "1.5", "bar", "5", "baz", "10.5", "bazz", "20.2"),
0,
[]byte("3.8476190476190477"),
labels.Labels{{Name: "foo", Value: "1.5"}, {Name: "bar", Value: "5"}, {Name: "baz", Value: "10.5"}, {Name: "bazz", Value: "20.2"}},
labels.FromStrings("foo", "1.5", "bar", "5", "baz", "10.5", "bazz", "20.2"),
nil,
},
{
"mathfloatround",
newMustLineFormatter("{{ round (addf .foo 1.5 | subf .bar | mulf .baz | divf .bazz) 5 .2}}"),
labels.Labels{{Name: "foo", Value: "1.5"}, {Name: "bar", Value: "3.5"}, {Name: "baz", Value: "10.5"}, {Name: "bazz", Value: "20.4"}},
labels.FromStrings("foo", "1.5", "bar", "3.5", "baz", "10.5", "bazz", "20.4"),
0,
[]byte("3.88572"),
labels.Labels{{Name: "foo", Value: "1.5"}, {Name: "bar", Value: "3.5"}, {Name: "baz", Value: "10.5"}, {Name: "bazz", Value: "20.4"}},
labels.FromStrings("foo", "1.5", "bar", "3.5", "baz", "10.5", "bazz", "20.4"),
nil,
},
{
"min",
newMustLineFormatter("min is {{ min .foo .bar .baz }} and max is {{ max .foo .bar .baz }}"),
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "10"}, {Name: "baz", Value: "15"}},
labels.FromStrings("foo", "5", "bar", "10", "baz", "15"),
0,
[]byte("min is 5 and max is 15"),
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "10"}, {Name: "baz", Value: "15"}},
labels.FromStrings("foo", "5", "bar", "10", "baz", "15"),
nil,
},
{
"max",
newMustLineFormatter("minf is {{ minf .foo .bar .baz }} and maxf is {{maxf .foo .bar .baz}}"),
labels.Labels{{Name: "foo", Value: "5.3"}, {Name: "bar", Value: "10.5"}, {Name: "baz", Value: "15.2"}},
labels.FromStrings("foo", "5.3", "bar", "10.5", "baz", "15.2"),
0,
[]byte("minf is 5.3 and maxf is 15.2"),
labels.Labels{{Name: "foo", Value: "5.3"}, {Name: "bar", Value: "10.5"}, {Name: "baz", Value: "15.2"}},
labels.FromStrings("foo", "5.3", "bar", "10.5", "baz", "15.2"),
nil,
},
{
"ceilfloor",
newMustLineFormatter("ceil is {{ ceil .foo }} and floor is {{floor .foo }}"),
labels.Labels{{Name: "foo", Value: "5.3"}},
labels.FromStrings("foo", "5.3"),
0,
[]byte("ceil is 6 and floor is 5"),
labels.Labels{{Name: "foo", Value: "5.3"}},
labels.FromStrings("foo", "5.3"),
nil,
},
{
"mod",
newMustLineFormatter("mod is {{ mod .foo 3 }}"),
labels.Labels{{Name: "foo", Value: "20"}},
labels.FromStrings("foo", "20"),
0,
[]byte("mod is 2"),
labels.Labels{{Name: "foo", Value: "20"}},
labels.FromStrings("foo", "20"),
nil,
},
{
"float64int",
newMustLineFormatter("{{ \"2.5\" | float64 | int | add 10}}"),
labels.Labels{{Name: "foo", Value: "2.5"}},
labels.FromStrings("foo", "2.5"),
0,
[]byte("12"),
labels.Labels{{Name: "foo", Value: "2.5"}},
labels.FromStrings("foo", "2.5"),
nil,
},
{
"datetime",
newMustLineFormatter("{{ sub (unixEpoch (toDate \"2006-01-02\" \"2021-11-02\")) (unixEpoch (toDate \"2006-01-02\" \"2021-11-01\")) }}"),
labels.Labels{},
labels.EmptyLabels(),
0,
[]byte("86400"),
labels.Labels{},
labels.EmptyLabels(),
nil,
},
{
"dateformat",
newMustLineFormatter("{{ date \"2006-01-02\" (toDate \"2006-01-02\" \"2021-11-02\") }}"),
labels.Labels{},
labels.EmptyLabels(),
0,
[]byte("2021-11-02"),
labels.Labels{},
labels.EmptyLabels(),
nil,
},
{
"now",
newMustLineFormatter("{{ div (unixEpoch now) (unixEpoch now) }}"),
labels.Labels{},
labels.EmptyLabels(),
0,
[]byte("1"),
labels.Labels{},
labels.EmptyLabels(),
nil,
},
{
"line",
newMustLineFormatter("{{ __line__ }} bar {{ .bar }}"),
labels.Labels{{Name: "bar", Value: "2"}},
labels.FromStrings("bar", "2"),
0,
[]byte("1 bar 2"),
labels.Labels{{Name: "bar", Value: "2"}},
labels.FromStrings("bar", "2"),
[]byte("1"),
},
{
"default",
newMustLineFormatter(`{{.foo | default "-" }}{{.bar | default "-"}}{{.unknown | default "-"}}`),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: ""}},
labels.FromStrings("foo", "blip", "bar", ""),
0,
[]byte("blip--"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: ""}},
labels.FromStrings("foo", "blip", "bar", ""),
nil,
},
{
"timestamp",
newMustLineFormatter("{{ __timestamp__ | date \"2006-01-02\" }} bar {{ .bar }}"),
labels.Labels{{Name: "bar", Value: "2"}},
labels.FromStrings("bar", "2"),
1656353124120000000,
[]byte("2022-06-27 bar 2"),
labels.Labels{{Name: "bar", Value: "2"}},
labels.FromStrings("bar", "2"),
[]byte("1"),
},
{
"timestamp_unix",
newMustLineFormatter("{{ __timestamp__ | unixEpoch }} bar {{ .bar }}"),
labels.Labels{{Name: "bar", Value: "2"}},
labels.FromStrings("bar", "2"),
1656353124120000000,
[]byte("1656353124 bar 2"),
labels.Labels{{Name: "bar", Value: "2"}},
labels.FromStrings("bar", "2"),
[]byte("1"),
},
{
"template_error",
newMustLineFormatter("{{.foo | now}}"),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
0,
nil,
labels.Labels{
{Name: "foo", Value: "blip"},
{Name: "bar", Value: "blop"},
{Name: "__error__", Value: "TemplateFormatErr"},
{Name: "__error_details__", Value: "template: line:1:9: executing \"line\" at <now>: wrong number of args for now: want 0 got 1"},
},
labels.FromStrings("foo", "blip",
"bar", "blop",
"__error__", "TemplateFormatErr",
"__error_details__", "template: line:1:9: executing \"line\" at <now>: wrong number of args for now: want 0 got 1",
),
nil,
},
{
"bytes 1",
newMustLineFormatter("{{ .foo | bytes }}"),
labels.Labels{{Name: "foo", Value: "3 kB"}},
labels.FromStrings("foo", "3 kB"),
1656353124120000000,
[]byte("3000"),
labels.Labels{{Name: "foo", Value: "3 kB"}},
labels.FromStrings("foo", "3 kB"),
[]byte("1"),
},
{
"bytes 2",
newMustLineFormatter("{{ .foo | bytes }}"),
labels.Labels{{Name: "foo", Value: "3MB"}},
labels.FromStrings("foo", "3MB"),
1656353124120000000,
[]byte("3e+06"),
labels.Labels{{Name: "foo", Value: "3MB"}},
labels.FromStrings("foo", "3MB"),
[]byte("1"),
},
{
"duration 1",
newMustLineFormatter("{{ .foo | duration }}"),
labels.Labels{{Name: "foo", Value: "3ms"}},
labels.FromStrings("foo", "3ms"),
1656353124120000000,
[]byte("0.003"),
labels.Labels{{Name: "foo", Value: "3ms"}},
labels.FromStrings("foo", "3ms"),
[]byte("1"),
},
{
"duration 2",
newMustLineFormatter("{{ .foo | duration_seconds }}"),
labels.Labels{{Name: "foo", Value: "3m10s"}},
labels.FromStrings("foo", "3m10s"),
1656353124120000000,
[]byte("190"),
labels.Labels{{Name: "foo", Value: "3m10s"}},
labels.FromStrings("foo", "3m10s"),
[]byte("1"),
},
{
"toDateInZone",
newMustLineFormatter("{{ .foo | toDateInZone \"2006-01-02T15:04:05.999999999Z\" \"UTC\" | unixEpochMillis }}"),
labels.Labels{{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"}},
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z"),
1656353124120000000,
[]byte("1678411960340"),
labels.Labels{{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"}},
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z"),
[]byte("1"),
},
{
"unixEpochMillis",
newMustLineFormatter("{{ .foo | toDateInZone \"2006-01-02T15:04:05.999999999Z\" \"UTC\" | unixEpochMillis }}"),
labels.Labels{{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"}},
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z"),
1656353124120000000,
[]byte("1678411960340"),
labels.Labels{{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"}},
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z"),
[]byte("1"),
},
{
"unixEpochNanos",
newMustLineFormatter("{{ .foo | toDateInZone \"2006-01-02T15:04:05.999999999Z\" \"UTC\" | unixEpochNanos }}"),
labels.Labels{{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"}},
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z"),
1656353124120000000,
[]byte("1678411960340485723"),
labels.Labels{{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"}},
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z"),
[]byte("1"),
},
{
"base64encode",
newMustLineFormatter("{{ .foo | b64enc }}"),
labels.Labels{{Name: "foo", Value: "i'm a string, encode me!"}},
labels.FromStrings("foo", "i'm a string, encode me!"),
1656353124120000000,
[]byte("aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh"),
labels.Labels{{Name: "foo", Value: "i'm a string, encode me!"}},
labels.FromStrings("foo", "i'm a string, encode me!"),
[]byte("1"),
},
{
"base64decode",
newMustLineFormatter("{{ .foo | b64dec }}"),
labels.Labels{{Name: "foo", Value: "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh"}},
labels.FromStrings("foo", "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh"),
1656353124120000000,
[]byte("i'm a string, encode me!"),
labels.Labels{{Name: "foo", Value: "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh"}},
labels.FromStrings("foo", "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh"),
[]byte("1"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sort.Sort(tt.lbs)
sort.Sort(tt.wantLbs)
builder := NewBaseLabelsBuilder().ForLabels(tt.lbs, tt.lbs.Hash())
builder.Reset()
outLine, _ := tt.fmter.Process(tt.ts, tt.in, builder)
@ -499,8 +490,8 @@ func Test_labelsFormatter_Format(t *testing.T) {
{
"combined with template",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("foo", "{{.foo}} and {{.bar}}")}),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.Labels{{Name: "foo", Value: "blip and blop"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
labels.FromStrings("foo", "blip and blop", "bar", "blop"),
},
{
"combined with template and rename",
@ -508,8 +499,8 @@ func Test_labelsFormatter_Format(t *testing.T) {
NewTemplateLabelFmt("blip", "{{.foo}} and {{.bar}}"),
NewRenameLabelFmt("bar", "foo"),
}),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.Labels{{Name: "blip", Value: "blip and blop"}, {Name: "bar", Value: "blip"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
labels.FromStrings("blip", "blip and blop", "bar", "blip"),
},
{
"fn",
@ -517,162 +508,134 @@ func Test_labelsFormatter_Format(t *testing.T) {
NewTemplateLabelFmt("blip", "{{.foo | ToUpper }} and {{.bar}}"),
NewRenameLabelFmt("bar", "foo"),
}),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.Labels{{Name: "blip", Value: "BLIP and blop"}, {Name: "bar", Value: "blip"}},
labels.FromStrings("foo", "blip", "bar", "blop"),
labels.FromStrings("blip", "BLIP and blop", "bar", "blip"),
},
{
"math",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("status", "{{div .status 100 }}")}),
labels.Labels{{Name: "status", Value: "200"}},
labels.Labels{{Name: "status", Value: "2"}},
labels.FromStrings("status", "200"),
labels.FromStrings("status", "2"),
},
{
"default",
mustNewLabelsFormatter([]LabelFmt{
NewTemplateLabelFmt("blip", `{{.foo | default "-" }} and {{.bar}}`),
}),
labels.Labels{{Name: "bar", Value: "blop"}},
labels.Labels{{Name: "blip", Value: "- and blop"}, {Name: "bar", Value: "blop"}},
labels.FromStrings("bar", "blop"),
labels.FromStrings("blip", "- and blop", "bar", "blop"),
},
{
"template error",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{replace \"test\" .foo}}")}),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "blip"},
{Name: "bar", Value: "blop"},
{Name: "__error__", Value: "TemplateFormatErr"},
{Name: "__error_details__", Value: "template: label:1:2: executing \"label\" at <replace>: wrong number of args for replace: want 3 got 2"},
},
labels.FromStrings("foo", "blip", "bar", "blop"),
labels.FromStrings("foo", "blip",
"bar", "blop",
"__error__", "TemplateFormatErr",
"__error_details__", "template: label:1:2: executing \"label\" at <replace>: wrong number of args for replace: want 3 got 2",
),
},
{
"line",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("line", "{{ __line__ }}")}),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "blip"},
{Name: "bar", Value: "blop"},
{Name: "line", Value: "test line"},
},
labels.FromStrings("foo", "blip", "bar", "blop"),
labels.FromStrings("foo", "blip",
"bar", "blop",
"line", "test line",
),
},
{
"timestamp",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("ts", "{{ __timestamp__ | date \"2006-01-02\" }}")}),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "blip"},
{Name: "bar", Value: "blop"},
{Name: "ts", Value: "2022-08-26"},
},
labels.FromStrings("foo", "blip", "bar", "blop"),
labels.FromStrings("foo", "blip",
"bar", "blop",
"ts", "2022-08-26",
),
},
{
"timestamp_unix",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("ts", "{{ __timestamp__ | unixEpoch }}")}),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "blip"},
{Name: "bar", Value: "blop"},
{Name: "ts", Value: "1661518453"},
},
labels.FromStrings("foo", "blip", "bar", "blop"),
labels.FromStrings("foo", "blip",
"bar", "blop",
"ts", "1661518453",
),
},
{
"count",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("count", `{{ __line__ | count "test" }}`)}),
labels.Labels{{Name: "foo", Value: "blip"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "blip"},
{Name: "bar", Value: "blop"},
{Name: "count", Value: "1"},
},
labels.FromStrings("foo", "blip", "bar", "blop"),
labels.FromStrings("foo", "blip",
"bar", "blop",
"count", "1",
),
},
{
"count regex no matches",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("count", `{{ __line__ | count "notmatching.*" }}`)}),
labels.Labels{},
labels.Labels{
{Name: "count", Value: "0"},
},
labels.EmptyLabels(),
labels.FromStrings("count", "0"),
},
{
"bytes 1",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{ .foo | bytes }}")}),
labels.Labels{{Name: "foo", Value: "3 kB"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "3 kB"},
{Name: "bar", Value: "3000"},
},
labels.FromStrings("foo", "3 kB", "bar", "blop"),
labels.FromStrings("foo", "3 kB", "bar", "3000"),
},
{
"bytes 2",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{ .foo | bytes }}")}),
labels.Labels{{Name: "foo", Value: "3MB"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "3MB"},
{Name: "bar", Value: "3e+06"},
},
labels.FromStrings("foo", "3MB", "bar", "blop"),
labels.FromStrings("foo", "3MB", "bar", "3e+06"),
},
{
"duration 1",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{ .foo | duration }}")}),
labels.Labels{{Name: "foo", Value: "3ms"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "3ms"},
{Name: "bar", Value: "0.003"},
},
labels.FromStrings("foo", "3ms", "bar", "blop"),
labels.FromStrings("foo", "3ms",
"bar", "0.003",
),
},
{
"duration 2",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{ .foo | duration }}")}),
labels.Labels{{Name: "foo", Value: "3m10s"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "3m10s"},
{Name: "bar", Value: "190"},
},
labels.FromStrings("foo", "3m10s", "bar", "blop"),
labels.FromStrings("foo", "3m10s", "bar", "190"),
},
{
"toDateInZone",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{ .foo | toDateInZone \"2006-01-02T15:04:05.999999999Z\" \"UTC\" | unixEpochMillis }}")}),
labels.Labels{{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"},
{Name: "bar", Value: "1678411960340"},
},
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z", "bar", "blop"),
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z", "bar", "1678411960340"),
},
{
"unixEpochMillis",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{ .foo | toDateInZone \"2006-01-02T15:04:05.999999999Z\" \"UTC\" | unixEpochMillis }}")}),
labels.Labels{{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"},
{Name: "bar", Value: "1678411960340"},
},
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z", "bar", "blop"),
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z", "bar", "1678411960340"),
},
{
"unixEpochNanos",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{ .foo | toDateInZone \"2006-01-02T15:04:05.999999999Z\" \"UTC\" | unixEpochNanos }}")}),
labels.Labels{{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "2023-03-10T01:32:40.340485723Z"},
{Name: "bar", Value: "1678411960340485723"},
},
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z", "bar", "blop"),
labels.FromStrings("foo", "2023-03-10T01:32:40.340485723Z", "bar", "1678411960340485723"),
},
{
"base64encode",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{ .foo | b64enc }}")}),
labels.Labels{{Name: "foo", Value: "i'm a string, encode me!"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "i'm a string, encode me!"},
{Name: "bar", Value: "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh"},
},
labels.FromStrings("foo", "i'm a string, encode me!", "bar", "blop"),
labels.FromStrings("foo", "i'm a string, encode me!",
"bar", "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh",
),
},
{
"base64decode",
mustNewLabelsFormatter([]LabelFmt{NewTemplateLabelFmt("bar", "{{ .foo | b64dec }}")}),
labels.Labels{{Name: "foo", Value: "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh"}, {Name: "bar", Value: "blop"}},
labels.Labels{
{Name: "foo", Value: "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh"},
{Name: "bar", Value: "i'm a string, encode me!"},
},
labels.FromStrings("foo", "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh", "bar", "blop"),
labels.FromStrings("foo", "aSdtIGEgc3RyaW5nLCBlbmNvZGUgbWUh",
"bar", "i'm a string, encode me!",
),
},
}
@ -681,7 +644,6 @@ func Test_labelsFormatter_Format(t *testing.T) {
builder := NewBaseLabelsBuilder().ForLabels(tt.in, tt.in.Hash())
builder.Reset()
_, _ = tt.fmter.Process(1661518453244672570, []byte("test line"), builder)
sort.Sort(tt.want)
require.Equal(t, tt.want, builder.LabelsResult().Labels())
})
}

2
pkg/logql/log/ip_test.go
Unescape View File

@ -172,7 +172,7 @@ func Test_IPLabelFilterTy(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
lf := NewIPLabelFilter(c.pat, c.label, c.ty)
lbs := labels.Labels{labels.Label{Name: c.label, Value: string(c.val)}}
lbs := labels.FromStrings(c.label, string(c.val))
lbb := NewBaseLabelsBuilder().ForLabels(lbs, lbs.Hash())
_, ok := lf.Process(0, []byte("x"), lbb)
if c.fail {

224
pkg/logql/log/label_filter_test.go
Unescape View File

@ -2,7 +2,6 @@ package log
import (
"reflect"
"sort"
"strings"
"testing"
"time"
@ -24,51 +23,51 @@ func TestBinary_Filter(t *testing.T) {
}{
{
NewAndLabelFilter(NewNumericLabelFilter(LabelFilterEqual, "foo", 5), NewDurationLabelFilter(LabelFilterEqual, "bar", 1*time.Second)),
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "1s"}},
labels.FromStrings("foo", "5", "bar", "1s"),
true,
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "1s"}},
labels.FromStrings("foo", "5", "bar", "1s"),
},
{
NewAndLabelFilter(NewNumericLabelFilter(LabelFilterEqual, "foo", 5), NewBytesLabelFilter(LabelFilterEqual, "bar", 42000)),
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "42kB"}},
labels.FromStrings("foo", "5", "bar", "42kB"),
true,
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "42kB"}},
labels.FromStrings("foo", "5", "bar", "42kB"),
},
{
NewAndLabelFilter(
NewNumericLabelFilter(LabelFilterEqual, "foo", 5),
NewDurationLabelFilter(LabelFilterEqual, "bar", 1*time.Second),
),
labels.Labels{{Name: "foo", Value: "6"}, {Name: "bar", Value: "1s"}},
labels.FromStrings("foo", "6", "bar", "1s"),
false,
labels.Labels{{Name: "foo", Value: "6"}, {Name: "bar", Value: "1s"}},
labels.FromStrings("foo", "6", "bar", "1s"),
},
{
NewAndLabelFilter(
NewNumericLabelFilter(LabelFilterEqual, "foo", 5),
NewDurationLabelFilter(LabelFilterEqual, "bar", 1*time.Second),
),
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "2s"}},
labels.FromStrings("foo", "5", "bar", "2s"),
false,
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "2s"}},
labels.FromStrings("foo", "5", "bar", "2s"),
},
{
NewAndLabelFilter(
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchEqual, "foo", "5")),
NewDurationLabelFilter(LabelFilterEqual, "bar", 1*time.Second),
),
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "1s"}},
labels.FromStrings("foo", "5", "bar", "1s"),
true,
labels.Labels{{Name: "foo", Value: "5"}, {Name: "bar", Value: "1s"}},
labels.FromStrings("foo", "5", "bar", "1s"),
},
{
NewAndLabelFilter(
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchEqual, "foo", "5")),
NewDurationLabelFilter(LabelFilterEqual, "bar", 1*time.Second),
),
labels.Labels{{Name: "foo", Value: "6"}, {Name: "bar", Value: "1s"}},
labels.FromStrings("foo", "6", "bar", "1s"),
false,
labels.Labels{{Name: "foo", Value: "6"}, {Name: "bar", Value: "1s"}},
labels.FromStrings("foo", "6", "bar", "1s"),
},
{
NewAndLabelFilter(
@ -78,17 +77,15 @@ func TestBinary_Filter(t *testing.T) {
),
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotEqual, "method", "POST")),
),
labels.Labels{
{Name: "duration", Value: "2s"},
{Name: "status", Value: "200"},
{Name: "method", Value: "GET"},
},
labels.FromStrings("duration", "2s",
"status", "200",
"method", "GET",
),
true,
labels.Labels{
{Name: "duration", Value: "2s"},
{Name: "status", Value: "200"},
{Name: "method", Value: "GET"},
},
labels.FromStrings("duration", "2s",
"status", "200",
"method", "GET",
),
},
{
NewAndLabelFilter(
@ -98,17 +95,15 @@ func TestBinary_Filter(t *testing.T) {
),
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotEqual, "method", "POST")),
),
labels.Labels{
{Name: "duration", Value: "2s"},
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("duration", "2s",
"status", "200",
"method", "POST",
),
false,
labels.Labels{
{Name: "duration", Value: "2s"},
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("duration", "2s",
"status", "200",
"method", "POST",
),
},
{
NewAndLabelFilter(
@ -118,17 +113,15 @@ func TestBinary_Filter(t *testing.T) {
),
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotEqual, "method", "POST")),
),
labels.Labels{
{Name: "duration", Value: "2s"},
{Name: "status", Value: "500"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("duration", "2s",
"status", "500",
"method", "POST",
),
false,
labels.Labels{
{Name: "duration", Value: "2s"},
{Name: "status", Value: "500"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("duration", "2s",
"status", "500",
"method", "POST",
),
},
{
NewAndLabelFilter(
@ -138,63 +131,50 @@ func TestBinary_Filter(t *testing.T) {
),
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotEqual, "method", "POST")),
),
labels.Labels{
{Name: "duration", Value: "2s"},
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("duration", "2s",
"status", "200",
"method", "POST",
),
false,
labels.Labels{
{Name: "duration", Value: "2s"},
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("duration", "2s",
"status", "200",
"method", "POST",
),
},
{
NewDurationLabelFilter(LabelFilterGreaterThan, "duration", 3*time.Second),
labels.Labels{
{Name: "duration", Value: "2weeeeee"},
},
labels.FromStrings("duration", "2weeeeee"),
true,
labels.Labels{
{Name: "duration", Value: "2weeeeee"},
{Name: "__error__", Value: "LabelFilterErr"},
{Name: "__error_details__", Value: "time: unknown unit \"weeeeee\" in duration \"2weeeeee\""},
},
labels.FromStrings("duration", "2weeeeee",
"__error__", "LabelFilterErr",
"__error_details__", "time: unknown unit \"weeeeee\" in duration \"2weeeeee\"",
),
},
{
NewBytesLabelFilter(LabelFilterGreaterThan, "bytes", 100),
labels.Labels{
{Name: "bytes", Value: "2qb"},
},
labels.FromStrings("bytes", "2qb"),
true,
labels.Labels{
{Name: "bytes", Value: "2qb"},
{Name: "__error__", Value: "LabelFilterErr"},
{Name: "__error_details__", Value: "unhandled size name: qb"},
},
labels.FromStrings("bytes", "2qb",
"__error__", "LabelFilterErr",
"__error_details__", "unhandled size name: qb",
),
},
{
NewNumericLabelFilter(LabelFilterGreaterThan, "number", 100),
labels.Labels{
{Name: "number", Value: "not_a_number"},
},
labels.FromStrings("number", "not_a_number"),
true,
labels.Labels{
{Name: "number", Value: "not_a_number"},
{Name: "__error__", Value: "LabelFilterErr"},
{Name: "__error_details__", Value: "strconv.ParseFloat: parsing \"not_a_number\": invalid syntax"},
},
labels.FromStrings("number", "not_a_number",
"__error__", "LabelFilterErr",
"__error_details__", "strconv.ParseFloat: parsing \"not_a_number\": invalid syntax",
),
},
}
for _, tt := range tests {
t.Run(tt.f.String(), func(t *testing.T) {
sort.Sort(tt.lbs)
b := NewBaseLabelsBuilder().ForLabels(tt.lbs, tt.lbs.Hash())
b.Reset()
_, got := tt.f.Process(0, nil, b)
require.Equal(t, tt.want, got)
sort.Sort(tt.wantLbs)
require.Equal(t, tt.wantLbs, b.LabelsResult().Labels())
})
}
@ -222,13 +202,13 @@ func TestBytes_Filter(t *testing.T) {
}
for _, tt := range tests {
f := NewBytesLabelFilter(LabelFilterEqual, "bar", tt.expectedBytes)
lbs := labels.Labels{{Name: "bar", Value: tt.label}}
lbs := labels.FromStrings("bar", tt.label)
t.Run(f.String(), func(t *testing.T) {
b := NewBaseLabelsBuilder().ForLabels(lbs, lbs.Hash())
b.Reset()
_, got := f.Process(0, nil, b)
require.Equal(t, tt.want, got)
wantLbs := labels.Labels{{Name: "bar", Value: tt.wantLabel}}
wantLbs := labels.FromStrings("bar", tt.wantLabel)
require.Equal(t, wantLbs, b.LabelsResult().Labels())
})
}
@ -245,68 +225,58 @@ func TestErrorFiltering(t *testing.T) {
}{
{
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotEqual, logqlmodel.ErrorLabel, errJSON)),
labels.Labels{
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("status", "200",
"method", "POST",
),
errJSON,
false,
labels.Labels{
{Name: logqlmodel.ErrorLabel, Value: errJSON},
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings(logqlmodel.ErrorLabel, errJSON,
"status", "200",
"method", "POST",
),
},
{
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotRegexp, logqlmodel.ErrorLabel, ".+")),
labels.Labels{
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("status", "200",
"method", "POST",
),
"foo",
false,
labels.Labels{
{Name: logqlmodel.ErrorLabel, Value: "foo"},
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings(logqlmodel.ErrorLabel, "foo",
"status", "200",
"method", "POST",
),
},
{
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotRegexp, logqlmodel.ErrorLabel, ".+")),
labels.Labels{
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("status", "200",
"method", "POST",
),
"",
true,
labels.Labels{
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("status", "200",
"method", "POST",
),
},
{
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotEqual, logqlmodel.ErrorLabel, errJSON)),
labels.Labels{
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("status", "200",
"method", "POST",
),
"",
true,
labels.Labels{
{Name: "status", Value: "200"},
{Name: "method", Value: "POST"},
},
labels.FromStrings("status", "200",
"method", "POST",
),
},
}
for _, tt := range tests {
t.Run(tt.f.String(), func(t *testing.T) {
sort.Sort(tt.lbs)
b := NewBaseLabelsBuilder().ForLabels(tt.lbs, tt.lbs.Hash())
b.Reset()
b.SetErr(tt.err)
_, got := tt.f.Process(0, nil, b)
require.Equal(t, tt.want, got)
sort.Sort(tt.wantLbs)
require.Equal(t, tt.wantLbs, b.LabelsResult().Labels())
})
}
@ -365,51 +335,51 @@ func TestStringLabelFilter(t *testing.T) {
{
name: `logfmt|subqueries!="0" (without label)`,
filter: NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotEqual, "subqueries", "0")),
labels: labels.Labels{{Name: "msg", Value: "hello"}}, // no label `subqueries`
labels: labels.FromStrings("msg", "hello"), // no label `subqueries`
// without `subqueries` label, the value is assumed to be empty `subqueries=""` is matches the label filter `subqueries!="0"`.
shouldMatch: true,
},
{
name: `logfmt|subqueries!="0" (with label)`,
filter: NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotEqual, "subqueries", "0")),
labels: labels.Labels{{Name: "msg", Value: "hello"}, {Name: "subqueries", Value: "2"}}, // label `subqueries` exist
labels: labels.FromStrings("msg", "hello", "subqueries", "2"), // label `subqueries` exist
shouldMatch: true,
},
{
name: `logfmt|subqueries!~"0" (without label)`,
filter: NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotRegexp, "subqueries", "0")),
labels: labels.Labels{{Name: "msg", Value: "hello"}}, // no label `subqueries`
labels: labels.FromStrings("msg", "hello"), // no label `subqueries`
// without `subqueries` label, the value is assumed to be empty `subqueries=""` is matches the label filter `subqueries!="0"`.
shouldMatch: true,
},
{
name: `logfmt|subqueries!~"0" (with label)`,
filter: NewStringLabelFilter(labels.MustNewMatcher(labels.MatchNotRegexp, "subqueries", "0")),
labels: labels.Labels{{Name: "msg", Value: "hello"}, {Name: "subqueries", Value: "2"}}, // label `subqueries` exist
labels: labels.FromStrings("msg", "hello", "subqueries", "2"), // label `subqueries` exist
shouldMatch: true,
},
{
name: `logfmt|subqueries="0" (without label)`,
filter: NewStringLabelFilter(labels.MustNewMatcher(labels.MatchEqual, "subqueries", "")),
labels: labels.Labels{{Name: "msg", Value: "hello"}}, // no label `subqueries`
labels: labels.FromStrings("msg", "hello"), // no label `subqueries`
shouldMatch: true,
},
{
name: `logfmt|subqueries="0" (with label)`,
filter: NewStringLabelFilter(labels.MustNewMatcher(labels.MatchEqual, "subqueries", "")),
labels: labels.Labels{{Name: "msg", Value: "hello"}, {Name: "subqueries", Value: ""}}, // label `subqueries` exist
labels: labels.FromStrings("msg", "hello", "subqueries", ""), // label `subqueries` exist
shouldMatch: true,
},
{
name: `logfmt|subqueries=~"0" (without label)`,
filter: NewStringLabelFilter(labels.MustNewMatcher(labels.MatchRegexp, "subqueries", "")),
labels: labels.Labels{{Name: "msg", Value: "hello"}}, // no label `subqueries`
labels: labels.FromStrings("msg", "hello"), // no label `subqueries`
shouldMatch: true,
},
{
name: `logfmt|subqueries=~"0" (with label)`,
filter: NewStringLabelFilter(labels.MustNewMatcher(labels.MatchRegexp, "subqueries", "")),
labels: labels.Labels{{Name: "msg", Value: "hello"}, {Name: "subqueries", Value: ""}}, // label `subqueries` exist
labels: labels.FromStrings("msg", "hello", "subqueries", ""), // label `subqueries` exist
shouldMatch: true,
},
{
@ -454,9 +424,7 @@ func BenchmarkLineLabelFilters(b *testing.B) {
"foo", "foobar", "bar", "foobuzz", "buzz", "f", " ", "fba", "foofoofoo", "b", "foob", "bfoo", "FoO",
"foo, 世界", allunicode(), "fooÏbar",
}, ",")
lbl := NewBaseLabelsBuilder().ForLabels(labels.Labels{
{Name: "foo", Value: fixture},
}, 0)
lbl := NewBaseLabelsBuilder().ForLabels(labels.FromStrings("foo", fixture), 0)
for _, test := range []struct {
re string

91
pkg/logql/log/labels_test.go
Unescape View File

@ -1,7 +1,6 @@
package log
import (
"sort"
"testing"
"github.com/prometheus/prometheus/model/labels"
@ -11,7 +10,7 @@ import (
)
func TestLabelsBuilder_Get(t *testing.T) {
lbs := labels.Labels{labels.Label{Name: "already", Value: "in"}}
lbs := labels.FromStrings("already", "in")
b := NewBaseLabelsBuilder().ForLabels(lbs, lbs.Hash())
b.Reset()
b.Set("foo", "bar")
@ -34,68 +33,59 @@ func TestLabelsBuilder_Get(t *testing.T) {
}
func TestLabelsBuilder_LabelsError(t *testing.T) {
lbs := labels.Labels{labels.Label{Name: "already", Value: "in"}}
lbs := labels.FromStrings("already", "in")
b := NewBaseLabelsBuilder().ForLabels(lbs, lbs.Hash())
b.Reset()
b.SetErr("err")
lbsWithErr := b.LabelsResult().Labels()
require.Equal(
t,
labels.Labels{
labels.Label{Name: logqlmodel.ErrorLabel, Value: "err"},
labels.Label{Name: "already", Value: "in"},
},
labels.FromStrings(logqlmodel.ErrorLabel, "err",
"already", "in",
),
lbsWithErr,
)
// make sure the original labels is unchanged.
require.Equal(t, labels.Labels{labels.Label{Name: "already", Value: "in"}}, lbs)
require.Equal(t, labels.FromStrings("already", "in"), lbs)
}
func TestLabelsBuilder_LabelsResult(t *testing.T) {
lbs := labels.Labels{
labels.Label{Name: "namespace", Value: "loki"},
labels.Label{Name: "job", Value: "us-central1/loki"},
labels.Label{Name: "cluster", Value: "us-central1"},
}
sort.Sort(lbs)
strs := []string{"namespace", "loki",
"job", "us-central1/loki",
"cluster", "us-central1"}
lbs := labels.FromStrings(strs...)
b := NewBaseLabelsBuilder().ForLabels(lbs, lbs.Hash())
b.Reset()
assertLabelResult(t, lbs, b.LabelsResult())
b.SetErr("err")
withErr := append(lbs, labels.Label{Name: logqlmodel.ErrorLabel, Value: "err"})
sort.Sort(withErr)
withErr := labels.FromStrings(append(strs, logqlmodel.ErrorLabel, "err")...)
assertLabelResult(t, withErr, b.LabelsResult())
b.Set("foo", "bar")
b.Set("namespace", "tempo")
b.Set("buzz", "fuzz")
b.Del("job")
expected := labels.Labels{
labels.Label{Name: logqlmodel.ErrorLabel, Value: "err"},
labels.Label{Name: "namespace", Value: "tempo"},
labels.Label{Name: "cluster", Value: "us-central1"},
labels.Label{Name: "foo", Value: "bar"},
labels.Label{Name: "buzz", Value: "fuzz"},
}
sort.Sort(expected)
expected := labels.FromStrings(logqlmodel.ErrorLabel, "err",
"namespace", "tempo",
"cluster", "us-central1",
"foo", "bar",
"buzz", "fuzz",
)
assertLabelResult(t, expected, b.LabelsResult())
// cached.
assertLabelResult(t, expected, b.LabelsResult())
}
func TestLabelsBuilder_GroupedLabelsResult(t *testing.T) {
lbs := labels.Labels{
labels.Label{Name: "namespace", Value: "loki"},
labels.Label{Name: "job", Value: "us-central1/loki"},
labels.Label{Name: "cluster", Value: "us-central1"},
}
sort.Sort(lbs)
strs := []string{"namespace", "loki",
"job", "us-central1/loki",
"cluster", "us-central1"}
lbs := labels.FromStrings(strs...)
b := NewBaseLabelsBuilderWithGrouping([]string{"namespace"}, nil, false, false).ForLabels(lbs, lbs.Hash())
b.Reset()
assertLabelResult(t, labels.Labels{labels.Label{Name: "namespace", Value: "loki"}}, b.GroupedLabels())
assertLabelResult(t, labels.FromStrings("namespace", "loki"), b.GroupedLabels())
b.SetErr("err")
withErr := append(lbs, labels.Label{Name: logqlmodel.ErrorLabel, Value: "err"})
sort.Sort(withErr)
withErr := labels.FromStrings(append(strs, logqlmodel.ErrorLabel, "err")...)
assertLabelResult(t, withErr, b.GroupedLabels())
b.Reset()
@ -103,45 +93,38 @@ func TestLabelsBuilder_GroupedLabelsResult(t *testing.T) {
b.Set("namespace", "tempo")
b.Set("buzz", "fuzz")
b.Del("job")
expected := labels.Labels{
labels.Label{Name: "namespace", Value: "tempo"},
}
sort.Sort(expected)
expected := labels.FromStrings("namespace", "tempo")
assertLabelResult(t, expected, b.GroupedLabels())
// cached.
assertLabelResult(t, expected, b.GroupedLabels())
b = NewBaseLabelsBuilderWithGrouping([]string{"job"}, nil, false, false).ForLabels(lbs, lbs.Hash())
assertLabelResult(t, labels.Labels{labels.Label{Name: "job", Value: "us-central1/loki"}}, b.GroupedLabels())
assertLabelResult(t, labels.Labels{labels.Label{Name: "job", Value: "us-central1/loki"}}, b.GroupedLabels())
assertLabelResult(t, labels.FromStrings("job", "us-central1/loki"), b.GroupedLabels())
assertLabelResult(t, labels.FromStrings("job", "us-central1/loki"), b.GroupedLabels())
b.Del("job")
assertLabelResult(t, labels.Labels{}, b.GroupedLabels())
assertLabelResult(t, labels.EmptyLabels(), b.GroupedLabels())
b.Reset()
b.Set("namespace", "tempo")
assertLabelResult(t, labels.Labels{labels.Label{Name: "job", Value: "us-central1/loki"}}, b.GroupedLabels())
assertLabelResult(t, labels.FromStrings("job", "us-central1/loki"), b.GroupedLabels())
b = NewBaseLabelsBuilderWithGrouping([]string{"job"}, nil, true, false).ForLabels(lbs, lbs.Hash())
b.Del("job")
b.Set("foo", "bar")
b.Set("job", "something")
expected = labels.Labels{
labels.Label{Name: "namespace", Value: "loki"},
labels.Label{Name: "cluster", Value: "us-central1"},
labels.Label{Name: "foo", Value: "bar"},
}
sort.Sort(expected)
expected = labels.FromStrings("namespace", "loki",
"cluster", "us-central1",
"foo", "bar",
)
assertLabelResult(t, expected, b.GroupedLabels())
b = NewBaseLabelsBuilderWithGrouping(nil, nil, false, false).ForLabels(lbs, lbs.Hash())
b.Set("foo", "bar")
b.Set("job", "something")
expected = labels.Labels{
labels.Label{Name: "namespace", Value: "loki"},
labels.Label{Name: "job", Value: "something"},
labels.Label{Name: "cluster", Value: "us-central1"},
labels.Label{Name: "foo", Value: "bar"},
}
sort.Sort(expected)
expected = labels.FromStrings("namespace", "loki",
"job", "something",
"cluster", "us-central1",
"foo", "bar",
)
assertLabelResult(t, expected, b.GroupedLabels())
}

165
pkg/logql/log/metrics_extraction_test.go
Unescape View File

@ -1,7 +1,6 @@
package log
import (
"sort"
"testing"
"time"
@ -25,9 +24,9 @@ func Test_labelSampleExtractor_Extract(t *testing.T) {
ex: mustSampleExtractor(LabelExtractorWithStages(
"foo", ConvertFloat, nil, false, false, nil, NoopStage,
)),
in: labels.Labels{labels.Label{Name: "foo", Value: "15.0"}},
in: labels.FromStrings("foo", "15.0"),
want: 15,
wantLbs: labels.Labels{},
wantLbs: labels.EmptyLabels(),
wantOk: true,
},
{
@ -35,9 +34,9 @@ func Test_labelSampleExtractor_Extract(t *testing.T) {
ex: mustSampleExtractor(LabelExtractorWithStages(
"foo", ConvertFloat, nil, false, true, nil, NoopStage,
)),
in: labels.Labels{labels.Label{Name: "foo", Value: "15.0"}, labels.Label{Name: "bar", Value: "buzz"}},
in: labels.FromStrings("foo", "15.0", "bar", "buzz"),
want: 15,
wantLbs: labels.Labels{},
wantLbs: labels.EmptyLabels(),
wantOk: true,
},
{
@ -45,34 +44,29 @@ func Test_labelSampleExtractor_Extract(t *testing.T) {
ex: mustSampleExtractor(LabelExtractorWithStages(
"foo", ConvertFloat, []string{"bar", "buzz"}, true, false, nil, NoopStage,
)),
in: labels.Labels{
{Name: "foo", Value: "10"},
{Name: "bar", Value: "foo"},
{Name: "buzz", Value: "blip"},
{Name: "namespace", Value: "dev"},
},
want: 10,
wantLbs: labels.Labels{
{Name: "namespace", Value: "dev"},
},
wantOk: true,
in: labels.FromStrings("foo", "10",
"bar", "foo",
"buzz", "blip",
"namespace", "dev",
),
want: 10,
wantLbs: labels.FromStrings("namespace", "dev"),
wantOk: true,
},
{
name: "convert float with",
ex: mustSampleExtractor(LabelExtractorWithStages(
"foo", ConvertFloat, []string{"bar", "buzz"}, false, false, nil, NoopStage,
)),
in: labels.Labels{
{Name: "foo", Value: "0.6"},
{Name: "bar", Value: "foo"},
{Name: "buzz", Value: "blip"},
{Name: "namespace", Value: "dev"},
},
in: labels.FromStrings("foo", "0.6",
"bar", "foo",
"buzz", "blip",
"namespace", "dev",
),
want: 0.6,
wantLbs: labels.Labels{
{Name: "bar", Value: "foo"},
{Name: "buzz", Value: "blip"},
},
wantLbs: labels.FromStrings("bar", "foo",
"buzz", "blip",
),
wantOk: true,
},
{
@ -80,17 +74,15 @@ func Test_labelSampleExtractor_Extract(t *testing.T) {
ex: mustSampleExtractor(LabelExtractorWithStages(
"foo", ConvertDuration, []string{"bar", "buzz"}, false, false, nil, NoopStage,
)),
in: labels.Labels{
{Name: "foo", Value: "500ms"},
{Name: "bar", Value: "foo"},
{Name: "buzz", Value: "blip"},
{Name: "namespace", Value: "dev"},
},
in: labels.FromStrings("foo", "500ms",
"bar", "foo",
"buzz", "blip",
"namespace", "dev",
),
want: 0.5,
wantLbs: labels.Labels{
{Name: "bar", Value: "foo"},
{Name: "buzz", Value: "blip"},
},
wantLbs: labels.FromStrings("bar", "foo",
"buzz", "blip",
),
wantOk: true,
},
{
@ -98,17 +90,15 @@ func Test_labelSampleExtractor_Extract(t *testing.T) {
ex: mustSampleExtractor(LabelExtractorWithStages(
"foo", ConvertBytes, []string{"bar", "buzz"}, false, false, nil, NoopStage,
)),
in: labels.Labels{
{Name: "foo", Value: "13 MiB"},
{Name: "bar", Value: "foo"},
{Name: "buzz", Value: "blip"},
{Name: "namespace", Value: "dev"},
},
in: labels.FromStrings("foo", "13 MiB",
"bar", "foo",
"buzz", "blip",
"namespace", "dev",
),
want: 13 * 1024 * 1024,
wantLbs: labels.Labels{
{Name: "bar", Value: "foo"},
{Name: "buzz", Value: "blip"},
},
wantLbs: labels.FromStrings("bar", "foo",
"buzz", "blip",
),
wantOk: true,
},
{
@ -116,16 +106,14 @@ func Test_labelSampleExtractor_Extract(t *testing.T) {
ex: mustSampleExtractor(LabelExtractorWithStages(
"foo", ConvertFloat, []string{"bar", "buzz"}, false, false, nil, NoopStage,
)),
in: labels.Labels{
{Name: "foo", Value: "not_a_number"},
{Name: "bar", Value: "foo"},
},
wantLbs: labels.Labels{
{Name: "__error__", Value: "SampleExtractionErr"},
{Name: "__error_details__", Value: "strconv.ParseFloat: parsing \"not_a_number\": invalid syntax"},
{Name: "bar", Value: "foo"},
{Name: "foo", Value: "not_a_number"},
},
in: labels.FromStrings("foo", "not_a_number",
"bar", "foo",
),
wantLbs: labels.FromStrings("__error__", "SampleExtractionErr",
"__error_details__", "strconv.ParseFloat: parsing \"not_a_number\": invalid syntax",
"bar", "foo",
"foo", "not_a_number",
),
wantOk: true,
},
{
@ -133,38 +121,29 @@ func Test_labelSampleExtractor_Extract(t *testing.T) {
ex: mustSampleExtractor(LabelExtractorWithStages(
"foo", ConvertDuration, []string{"bar", "buzz"}, false, false, []Stage{NewLogfmtParser()}, NoopStage,
)),
in: labels.Labels{
{Name: "bar", Value: "foo"},
},
want: 0.1234,
wantLbs: labels.Labels{
{Name: "bar", Value: "foo"},
},
wantOk: true,
line: "foo=123.4ms",
in: labels.FromStrings("bar", "foo"),
want: 0.1234,
wantLbs: labels.FromStrings("bar", "foo"),
wantOk: true,
line: "foo=123.4ms",
},
{
name: "dynamic label, not convertable",
ex: mustSampleExtractor(LabelExtractorWithStages(
"foo", ConvertDuration, []string{"bar", "buzz"}, false, false, []Stage{NewLogfmtParser()}, NoopStage,
)),
in: labels.Labels{
{Name: "bar", Value: "foo"},
},
wantLbs: labels.Labels{
{Name: "__error__", Value: "SampleExtractionErr"},
{Name: "__error_details__", Value: "time: invalid duration \"not_a_number\""},
{Name: "bar", Value: "foo"},
{Name: "foo", Value: "not_a_number"},
},
in: labels.FromStrings("bar", "foo"),
wantLbs: labels.FromStrings("__error__", "SampleExtractionErr",
"__error_details__", "time: invalid duration \"not_a_number\"",
"bar", "foo",
"foo", "not_a_number",
),
wantOk: true,
line: "foo=not_a_number",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sort.Sort(tt.in)
outval, outlbs, ok := tt.ex.ForStream(tt.in).Process(0, []byte(tt.line))
require.Equal(t, tt.wantOk, ok)
require.Equal(t, tt.want, outval)
@ -181,10 +160,10 @@ func Test_labelSampleExtractor_Extract(t *testing.T) {
func Test_Extract_ExpectedLabels(t *testing.T) {
ex := mustSampleExtractor(LabelExtractorWithStages("duration", ConvertDuration, []string{"foo"}, false, false, []Stage{NewJSONParser()}, NoopStage))
f, lbs, ok := ex.ForStream(labels.Labels{{Name: "bar", Value: "foo"}}).ProcessString(0, `{"duration":"20ms","foo":"json"}`)
f, lbs, ok := ex.ForStream(labels.FromStrings("bar", "foo")).ProcessString(0, `{"duration":"20ms","foo":"json"}`)
require.True(t, ok)
require.Equal(t, (20 * time.Millisecond).Seconds(), f)
require.Equal(t, labels.Labels{{Name: "foo", Value: "json"}}, lbs.Labels())
require.Equal(t, labels.FromStrings("foo", "json"), lbs.Labels())
}
func TestLabelExtractorWithStages(t *testing.T) {
@ -232,7 +211,7 @@ func TestLabelExtractorWithStages(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
for _, line := range tc.checkLines {
v, lbs, ok := tc.extractor.ForStream(labels.Labels{{Name: "bar", Value: "foo"}}).ProcessString(0, line.logLine)
v, lbs, ok := tc.extractor.ForStream(labels.FromStrings("bar", "foo")).ProcessString(0, line.logLine)
skipped := !ok
assert.Equal(t, line.skip, skipped, "line", line.logLine)
if !skipped {
@ -261,11 +240,9 @@ func TestNewLineSampleExtractor(t *testing.T) {
se, err := NewLineSampleExtractor(CountExtractor, nil, nil, false, false)
require.NoError(t, err)
lbs := labels.Labels{
{Name: "namespace", Value: "dev"},
{Name: "cluster", Value: "us-central1"},
}
sort.Sort(lbs)
lbs := labels.FromStrings("namespace", "dev",
"cluster", "us-central1",
)
sse := se.ForStream(lbs)
f, l, ok := sse.Process(0, []byte(`foo`))
@ -286,7 +263,7 @@ func TestNewLineSampleExtractor(t *testing.T) {
f, l, ok = sse.Process(0, []byte(`foo`))
require.True(t, ok)
require.Equal(t, 3., f)
assertLabelResult(t, labels.Labels{labels.Label{Name: "namespace", Value: "dev"}}, l)
assertLabelResult(t, labels.FromStrings("namespace", "dev"), l)
sse = se.ForStream(lbs)
_, _, ok = sse.Process(0, []byte(`nope`))
@ -295,8 +272,8 @@ func TestNewLineSampleExtractor(t *testing.T) {
func TestFilteringSampleExtractor(t *testing.T) {
se := NewFilteringSampleExtractor([]PipelineFilter{
newPipelineFilter(2, 4, labels.Labels{{Name: "foo", Value: "bar"}, {Name: "bar", Value: "baz"}}, "e"),
newPipelineFilter(3, 5, labels.Labels{{Name: "baz", Value: "foo"}}, "e"),
newPipelineFilter(2, 4, labels.FromStrings("foo", "bar", "bar", "baz"), "e"),
newPipelineFilter(3, 5, labels.FromStrings("baz", "foo"), "e"),
}, newStubExtractor())
tt := []struct {
@ -306,13 +283,13 @@ func TestFilteringSampleExtractor(t *testing.T) {
labels labels.Labels
ok bool
}{
{"it is after the timerange", 6, "line", labels.Labels{{Name: "baz", Value: "foo"}}, true},
{"it is before the timerange", 1, "line", labels.Labels{{Name: "baz", Value: "foo"}}, true},
{"it doesn't match the filter", 3, "all good", labels.Labels{{Name: "baz", Value: "foo"}}, true},
{"it doesn't match all the selectors", 3, "line", labels.Labels{{Name: "foo", Value: "bar"}}, true},
{"it doesn't match any selectors", 3, "line", labels.Labels{{Name: "beep", Value: "boop"}}, true},
{"it matches all selectors", 3, "line", labels.Labels{{Name: "foo", Value: "bar"}, {Name: "bar", Value: "baz"}}, false},
{"it tries all the filters", 5, "line", labels.Labels{{Name: "baz", Value: "foo"}}, false},
{"it is after the timerange", 6, "line", labels.FromStrings("baz", "foo"), true},
{"it is before the timerange", 1, "line", labels.FromStrings("baz", "foo"), true},
{"it doesn't match the filter", 3, "all good", labels.FromStrings("baz", "foo"), true},
{"it doesn't match all the selectors", 3, "line", labels.FromStrings("foo", "bar"), true},
{"it doesn't match any selectors", 3, "line", labels.FromStrings("beep", "boop"), true},
{"it matches all selectors", 3, "line", labels.FromStrings("foo", "bar", "bar", "baz"), false},
{"it tries all the filters", 5, "line", labels.FromStrings("baz", "foo"), false},
}
for _, test := range tt {

8
pkg/logql/log/parser_hints_test.go
Unescape View File

@ -43,7 +43,7 @@ var (
)
func Test_ParserHints(t *testing.T) {
lbs := labels.Labels{{Name: "app", Value: "nginx"}, {Name: "cluster", Value: "us-central-west"}}
lbs := labels.FromStrings("app", "nginx", "cluster", "us-central-west")
t.Parallel()
for _, tt := range []struct {
@ -258,7 +258,7 @@ func TestLabelFiltersInParseHints(t *testing.T) {
s := []log.Stage{log.NewStringLabelFilter(labels.MustNewMatcher(labels.MatchEqual, "protocol", "nothing"))}
h := log.NewParserHint(nil, nil, true, true, "metric", s)
lb := log.NewBaseLabelsBuilder().ForLabels(labels.Labels{{Name: "protocol", Value: "HTTP/2.0"}}, 0)
lb := log.NewBaseLabelsBuilder().ForLabels(labels.FromStrings("protocol", "HTTP/2.0"), 0)
require.False(t, h.ShouldContinueParsingLine("protocol", lb))
})
@ -266,7 +266,7 @@ func TestLabelFiltersInParseHints(t *testing.T) {
s := []log.Stage{log.NewStringLabelFilter(labels.MustNewMatcher(labels.MatchEqual, "protocol", "nothing"))}
h := log.NewParserHint(nil, nil, true, true, "metric", s)
lb := log.NewBaseLabelsBuilder().ForLabels(labels.Labels{{Name: "response", Value: "200"}}, 0)
lb := log.NewBaseLabelsBuilder().ForLabels(labels.FromStrings("response", "200"), 0)
require.True(t, h.ShouldContinueParsingLine("response", lb))
})
@ -279,7 +279,7 @@ func TestLabelFiltersInParseHints(t *testing.T) {
}
h := log.NewParserHint(nil, nil, true, true, "metric", s)
lb := log.NewBaseLabelsBuilder().ForLabels(labels.Labels{{Name: "protocol", Value: "HTTP/2.0"}}, 0)
lb := log.NewBaseLabelsBuilder().ForLabels(labels.FromStrings("protocol", "HTTP/2.0"), 0)
require.True(t, h.ShouldContinueParsingLine("protocol", lb))
})
}

738
pkg/logql/log/parser_test.go
View File

File diff suppressed because it is too large Load Diff

129
pkg/logql/log/pipeline_test.go
Unescape View File

@ -1,7 +1,6 @@
package log
import (
"sort"
"testing"
"time"
@ -12,7 +11,7 @@ import (
)
func TestNoopPipeline(t *testing.T) {
lbs := labels.Labels{{Name: "foo", Value: "bar"}}
lbs := labels.FromStrings("foo", "bar")
l, lbr, matches := NewNoopPipeline().ForStream(lbs).Process(0, []byte(""))
require.Equal(t, []byte(""), l)
require.Equal(t, NewLabelsResult(lbs, lbs.Hash()), lbr)
@ -25,7 +24,7 @@ func TestNoopPipeline(t *testing.T) {
}
func TestPipeline(t *testing.T) {
lbs := labels.Labels{{Name: "foo", Value: "bar"}}
lbs := labels.FromStrings("foo", "bar")
p := NewPipeline([]Stage{
NewStringLabelFilter(labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")),
newMustLineFormatter("lbs {{.foo}}"),
@ -40,12 +39,12 @@ func TestPipeline(t *testing.T) {
require.Equal(t, NewLabelsResult(lbs, lbs.Hash()), lbr)
require.Equal(t, true, matches)
l, lbr, matches = p.ForStream(labels.Labels{}).Process(0, []byte("line"))
l, lbr, matches = p.ForStream(labels.EmptyLabels()).Process(0, []byte("line"))
require.Equal(t, []byte(nil), l)
require.Equal(t, nil, lbr)
require.Equal(t, false, matches)
ls, lbr, matches = p.ForStream(labels.Labels{}).ProcessString(0, "line")
ls, lbr, matches = p.ForStream(labels.EmptyLabels()).ProcessString(0, "line")
require.Equal(t, "", ls)
require.Equal(t, nil, lbr)
require.Equal(t, false, matches)
@ -53,8 +52,8 @@ func TestPipeline(t *testing.T) {
func TestFilteringPipeline(t *testing.T) {
p := NewFilteringPipeline([]PipelineFilter{
newPipelineFilter(2, 4, labels.Labels{{Name: "foo", Value: "bar"}, {Name: "bar", Value: "baz"}}, "e"),
newPipelineFilter(3, 5, labels.Labels{{Name: "baz", Value: "foo"}}, "e"),
newPipelineFilter(2, 4, labels.FromStrings("foo", "bar", "bar", "baz"), "e"),
newPipelineFilter(3, 5, labels.FromStrings("baz", "foo"), "e"),
}, newStubPipeline())
tt := []struct {
@ -64,13 +63,13 @@ func TestFilteringPipeline(t *testing.T) {
inputStreamLabels labels.Labels
ok bool
}{
{"it is before the timerange", 1, "line", labels.Labels{{Name: "baz", Value: "foo"}}, true},
{"it is after the timerange", 6, "line", labels.Labels{{Name: "baz", Value: "foo"}}, true},
{"it doesn't match the filter", 3, "all good", labels.Labels{{Name: "baz", Value: "foo"}}, true},
{"it doesn't match all the selectors", 3, "line", labels.Labels{{Name: "foo", Value: "bar"}}, true},
{"it doesn't match any selectors", 3, "line", labels.Labels{{Name: "beep", Value: "boop"}}, true},
{"it matches all selectors", 3, "line", labels.Labels{{Name: "foo", Value: "bar"}, {Name: "bar", Value: "baz"}}, false},
{"it tries all the filters", 5, "line", labels.Labels{{Name: "baz", Value: "foo"}}, false},
{"it is before the timerange", 1, "line", labels.FromStrings("baz", "foo"), true},
{"it is after the timerange", 6, "line", labels.FromStrings("baz", "foo"), true},
{"it doesn't match the filter", 3, "all good", labels.FromStrings("baz", "foo"), true},
{"it doesn't match all the selectors", 3, "line", labels.FromStrings("foo", "bar"), true},
{"it doesn't match any selectors", 3, "line", labels.FromStrings("beep", "boop"), true},
{"it matches all selectors", 3, "line", labels.FromStrings("foo", "bar", "bar", "baz"), false},
{"it tries all the filters", 5, "line", labels.FromStrings("baz", "foo"), false},
}
for _, test := range tt {
@ -88,10 +87,10 @@ func TestFilteringPipeline(t *testing.T) {
func newPipelineFilter(start, end int64, lbls labels.Labels, filter string) PipelineFilter {
var stages []Stage
var matchers []*labels.Matcher
for _, l := range lbls {
lbls.Range(func(l labels.Label) {
m := labels.MustNewMatcher(labels.MatchEqual, l.Name, l.Value)
matchers = append(matchers, m)
}
})
stages = append(stages, mustFilter(NewFilter(filter, labels.MatchEqual)).ToStage())
return PipelineFilter{start, end, matchers, NewPipeline(stages)}
@ -168,18 +167,16 @@ func TestDropLabelsPipeline(t *testing.T) {
[]byte(`{"app":"foo","namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar"}}}`),
},
[]labels.Labels{
{
{Name: "level", Value: "info"},
{Name: "ts", Value: "2020-10-18T18:04:22.147378997Z"},
{Name: "caller", Value: "metrics.go:81"},
{Name: "status", Value: "200"},
},
{
{Name: "app", Value: "foo"},
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
{Name: "pod_deployment_ref", Value: "foobar"},
},
labels.FromStrings("level", "info",
"ts", "2020-10-18T18:04:22.147378997Z",
"caller", "metrics.go:81",
"status", "200",
),
labels.FromStrings("app", "foo",
"namespace", "prod",
"pod_uuid", "foo",
"pod_deployment_ref", "foobar",
),
},
},
{
@ -211,28 +208,25 @@ func TestDropLabelsPipeline(t *testing.T) {
[]byte(`{"app":"foo","namespace":"prod","pod":{"uuid":"foo","deployment":{"ref":"foobar"}}}`),
},
[]labels.Labels{
{
{Name: "level", Value: "info"},
{Name: "ts", Value: "2020-10-18T18:04:22.147378997Z"},
{Name: "caller", Value: "metrics.go:81"},
{Name: logqlmodel.ErrorLabel, Value: errJSON},
{Name: logqlmodel.ErrorDetailsLabel, Value: "Value looks like object, but can't find closing '}' symbol"},
},
{
{Name: "namespace", Value: "prod"},
{Name: "pod_uuid", Value: "foo"},
{Name: "pod_deployment_ref", Value: "foobar"},
{Name: logqlmodel.ErrorDetailsLabel, Value: "logfmt syntax error at pos 2 : unexpected '\"'"},
},
labels.FromStrings("level", "info",
"ts", "2020-10-18T18:04:22.147378997Z",
"caller", "metrics.go:81",
logqlmodel.ErrorLabel, errJSON,
logqlmodel.ErrorDetailsLabel, "Value looks like object, but can't find closing '}' symbol",
),
labels.FromStrings("namespace", "prod",
"pod_uuid", "foo",
"pod_deployment_ref", "foobar",
logqlmodel.ErrorDetailsLabel, "logfmt syntax error at pos 2 : unexpected '\"'",
),
},
},
}
for _, tt := range tests {
p := NewPipeline(tt.stages)
sp := p.ForStream(labels.Labels{})
sp := p.ForStream(labels.EmptyLabels())
for i, line := range tt.lines {
_, finalLbs, _ := sp.Process(0, line)
sort.Sort(tt.wantLabels[i])
require.Equal(t, tt.wantLabels[i], finalLbs.Labels())
}
}
@ -256,16 +250,15 @@ func Benchmark_Pipeline(b *testing.B) {
p := NewPipeline(stages)
line := []byte(`level=info ts=2020-10-18T18:04:22.147378997Z caller=metrics.go:81 org_id=29 traceID=29a0f088b047eb8c latency=fast query="{stream=\"stdout\",pod=\"loki-canary-xmjzp\"}" query_type=limited range_type=range length=20s step=1s duration=58.126671ms status=200 throughput_mb=2.496547 total_bytes_mb=0.145116`)
lineString := string(line)
lbs := labels.Labels{
{Name: "cluster", Value: "ops-tool1"},
{Name: "name", Value: "querier"},
{Name: "pod", Value: "querier-5896759c79-q7q9h"},
{Name: "stream", Value: "stderr"},
{Name: "container", Value: "querier"},
{Name: "namespace", Value: "loki-dev"},
{Name: "job", Value: "loki-dev/querier"},
{Name: "pod_template_hash", Value: "5896759c79"},
}
lbs := labels.FromStrings("cluster", "ops-tool1",
"name", "querier",
"pod", "querier-5896759c79-q7q9h",
"stream", "stderr",
"container", "querier",
"namespace", "loki-dev",
"job", "loki-dev/querier",
"pod_template_hash", "5896759c79",
)
sp := p.ForStream(lbs)
@ -331,16 +324,15 @@ func jsonBenchmark(b *testing.B, parser Stage) {
parser,
})
line := []byte(`{"ts":"2020-12-27T09:15:54.333026285Z","error":"action could not be completed", "context":{"file": "metrics.go"}}`)
lbs := labels.Labels{
{Name: "cluster", Value: "ops-tool1"},
{Name: "name", Value: "querier"},
{Name: "pod", Value: "querier-5896759c79-q7q9h"},
{Name: "stream", Value: "stderr"},
{Name: "container", Value: "querier"},
{Name: "namespace", Value: "loki-dev"},
{Name: "job", Value: "loki-dev/querier"},
{Name: "pod_template_hash", Value: "5896759c79"},
}
lbs := labels.FromStrings("cluster", "ops-tool1",
"name", "querier",
"pod", "querier-5896759c79-q7q9h",
"stream", "stderr",
"container", "querier",
"namespace", "loki-dev",
"job", "loki-dev/querier",
"pod_template_hash", "5896759c79",
)
b.ResetTimer()
sp := p.ForStream(lbs)
for n := 0; n < b.N; n++ {
@ -365,7 +357,7 @@ func invalidJSONBenchmark(b *testing.B, parser Stage) {
})
line := []byte(`invalid json`)
b.ResetTimer()
sp := p.ForStream(labels.Labels{})
sp := p.ForStream(labels.EmptyLabels())
for n := 0; n < b.N; n++ {
resLine, resLbs, resMatches = sp.Process(0, line)
@ -418,11 +410,10 @@ func logfmtBenchmark(b *testing.B, parser Stage) {
})
line := []byte(`level=info ts=2020-10-18T18:04:22.147378997Z caller=metrics.go:81 org_id=29 traceID=29a0f088b047eb8c latency=fast query="{stream=\"stdout\",pod=\"loki-canary-xmjzp\"}" query_type=limited range_type=range length=20s step=1s duration=58.126671ms status=200 throughput_mb=2.496547 total_bytes_mb=0.145116`)
lbs := labels.Labels{
{Name: "cluster", Value: "ops-tool1"},
{Name: "name", Value: "querier"},
{Name: "ts", Value: "2020-10-18T18:04:22.147378997Z"},
}
lbs := labels.FromStrings("cluster", "ops-tool1",
"name", "querier",
"ts", "2020-10-18T18:04:22.147378997Z",
)
b.ResetTimer()
sp := p.ForStream(lbs)
for n := 0; n < b.N; n++ {

Write Preview
Loading…
Cancel
Save