apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Improve LogQL format stages requireLabel (#4769)

Browse Source 
* Improve LogQL format stages.

This only includes required labels.

```
➜ go test -benchmem -run=^$  -bench ^Benchmark_Pipeline ./pkg/logql/log -v -count 5  > after.txt && benchstat before.txt after.txt
name                                 old time/op    new time/op    delta
_Pipeline/pipeline_bytes-16            13.8µs ± 6%    11.5µs ± 1%  -16.42%  (p=0.008 n=5+5)
_Pipeline/pipeline_string-16           13.9µs ± 2%    11.5µs ± 2%  -17.27%  (p=0.008 n=5+5)
_Pipeline/line_extractor_bytes-16      14.0µs ± 4%    11.6µs ± 0%  -17.45%  (p=0.016 n=5+4)
_Pipeline/line_extractor_string-16     13.5µs ± 0%    11.5µs ± 0%  -14.67%  (p=0.008 n=5+5)
_Pipeline/label_extractor_bytes-16     13.7µs ± 1%    11.7µs ± 1%  -14.69%  (p=0.008 n=5+5)
_Pipeline/label_extractor_string-16    14.1µs ±11%    11.6µs ± 0%  -17.89%  (p=0.008 n=5+5)

name                                 old alloc/op   new alloc/op   delta
_Pipeline/pipeline_bytes-16            9.51kB ± 0%    7.50kB ± 0%  -21.19%  (p=0.008 n=5+5)
_Pipeline/pipeline_string-16           9.51kB ± 0%    7.50kB ± 0%  -21.19%  (p=0.008 n=5+5)
_Pipeline/line_extractor_bytes-16      9.51kB ± 0%    7.50kB ± 0%  -21.19%  (p=0.008 n=5+5)
_Pipeline/line_extractor_string-16     9.51kB ± 0%    7.50kB ± 0%     ~     (p=0.079 n=4+5)
_Pipeline/label_extractor_bytes-16     9.51kB ± 0%    7.50kB ± 0%  -21.18%  (p=0.008 n=5+5)
_Pipeline/label_extractor_string-16    9.51kB ± 0%    7.50kB ± 0%  -21.18%  (p=0.008 n=5+5)

name                                 old allocs/op  new allocs/op  delta
_Pipeline/pipeline_bytes-16              46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/pipeline_string-16             46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/line_extractor_bytes-16        46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/line_extractor_string-16       46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/label_extractor_bytes-16       46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
_Pipeline/label_extractor_string-16      46.0 ± 0%      45.0 ± 0%   -2.17%  (p=0.008 n=5+5)
```

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

* rollback model change

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

* remove unused variable

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
pull/4833/head
Cyril Tovena 4 years ago committed by GitHub
parent 347e3e37cf
commit 0a944670ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 23 additions and 18 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 37
      pkg/logql/log/fmt.go

37
pkg/logql/log/fmt.go
Unescape View File

@ -101,7 +101,7 @@ type LineFormatter struct {
func NewFormatter(tmpl string) (*LineFormatter, error) {
t, err := template.New("line").Option("missingkey=zero").Funcs(functionMap).Parse(tmpl)
if err != nil {
return nil, fmt.Errorf("invalid line template: %s", err)
return nil, fmt.Errorf("invalid line template: %w", err)
}
return &LineFormatter{
Template: t,
@ -111,6 +111,7 @@ func NewFormatter(tmpl string) (*LineFormatter, error) {
func (lf *LineFormatter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) {
lf.buf.Reset()
if err := lf.Template.Execute(lf.buf, lbs.Labels().Map()); err != nil {
lbs.SetErr(errTemplateFormat)
return line, true
@ -122,18 +123,25 @@ func (lf *LineFormatter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool)
}
func (lf *LineFormatter) RequiredLabelNames() []string {
return uniqueString(listNodeFields(lf.Root))
return uniqueString(listNodeFields([]parse.Node{lf.Root}))
}
func listNodeFields(node parse.Node) []string {
func listNodeFields(nodes []parse.Node) []string {
var res []string
if node.Type() == parse.NodeAction {
for _, node := range nodes {
switch node.Type() {
case parse.NodePipe:
res = append(res, listNodeFieldsFromPipe(node.(*parse.PipeNode))...)
case parse.NodeAction:
res = append(res, listNodeFieldsFromPipe(node.(*parse.ActionNode).Pipe)...)
}
case parse.NodeList:
res = append(res, listNodeFields(node.(*parse.ListNode).Nodes)...)
case parse.NodeCommand:
res = append(res, listNodeFields(node.(*parse.CommandNode).Args)...)
case parse.NodeIf, parse.NodeWith, parse.NodeRange:
res = append(res, listNodeFieldsFromBranch(node)...)
if ln, ok := node.(*parse.ListNode); ok {
for _, n := range ln.Nodes {
res = append(res, listNodeFields(n)...)
case parse.NodeField:
res = append(res, node.(*parse.FieldNode).Ident...)
}
}
return res
@ -156,10 +164,10 @@ func listNodeFieldsFromBranch(node parse.Node) []string {
res = append(res, listNodeFieldsFromPipe(b.Pipe)...)
}
if b.List != nil {
res = append(res, listNodeFields(b.List)...)
res = append(res, listNodeFields(b.List.Nodes)...)
}
if b.ElseList != nil {
res = append(res, listNodeFields(b.ElseList)...)
res = append(res, listNodeFields(b.ElseList.Nodes)...)
}
return res
}
@ -167,11 +175,7 @@ func listNodeFieldsFromBranch(node parse.Node) []string {
func listNodeFieldsFromPipe(p *parse.PipeNode) []string {
var res []string
for _, c := range p.Cmds {
for _, a := range c.Args {
if f, ok := a.(*parse.FieldNode); ok {
res = append(res, f.Ident...)
}
}
res = append(res, listNodeFields(c.Args)...)
}
return res
}
@ -220,6 +224,7 @@ func NewLabelsFormatter(fmts []LabelFmt) (*LabelsFormatter, error) {
return nil, err
}
formats := make([]labelFormatter, 0, len(fmts))
for _, fm := range fmts {
toAdd := labelFormatter{LabelFmt: fm}
if !fm.Rename {
@ -284,7 +289,7 @@ func (lf *LabelsFormatter) RequiredLabelNames() []string {
names = append(names, fm.Value)
continue
}
names = append(names, listNodeFields(fm.tmpl.Root)...)
names = append(names, listNodeFields([]parse.Node{fm.tmpl.Root})...)
}
return uniqueString(names)
}

Write Preview
Loading…
Cancel
Save