apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

test: Add new tests for Drain pattern matching (#12655)

Browse Source
pull/12693/head
benclive 1 year ago committed by GitHub
parent c1415c5268
commit 5ae33d19ff
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 1467 additions and 49 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 42
      pkg/pattern/drain/drain.go
  2. 117
      pkg/pattern/drain/drain_test.go
  3. 1000
      pkg/pattern/drain/testdata/agent-logfmt.txt
  4. 0
      pkg/pattern/drain/testdata/custom.txt
  5. 210
      pkg/pattern/drain/testdata/drone-json.txt
  6. 100
      pkg/pattern/drain/testdata/ingester-logfmt.txt
  7. 2
      pkg/pattern/ingester_querier.go
  8. 45
      pkg/pattern/stream.go

42
pkg/pattern/drain/drain.go
Unescape View File

@ -103,12 +103,46 @@ type Node struct {
}
func DefaultConfig() *Config {
// TODO(kolesnikovae):
//
// This is crucial for Drain to ensure that the first LogClusterDepth tokens
// are constant (see https://jiemingzhu.github.io/pub/pjhe_icws2017.pdf).
// We should remove any variables such as timestamps, IDs, IPs, counters, etc.
// from these tokens.
//
// Moreover, Drain is not designed for structured logs. Therefore, we should
// handle logfmt (and, probably, JSON) logs in a special way:
//
// The parse tree should have a fixed length, and the depth should be
// determined by the number of fields in the logfmt message.
// A parsing tree should be maintained for each unique field set.
return &Config{
// At training, if at the depth of LogClusterDepth there is a cluster with
// similarity coefficient greater that SimTh, then the log message is added
// to that cluster. Otherwise, a new cluster is created.
//
// LogClusterDepth should be equal to the number of constant tokens from
// the beginning of the message that likely determine the message contents.
//
// > In this step, Drain traverses from a 1-st layer node, which
// > is searched in step 2, to a leaf node. This step is based on
// > the assumption that tokens in the beginning positions of a log
// > message are more likely to be constants. Specifically, Drain
// > selects the next internal node by the tokens in the beginning
// > positions of the log message
LogClusterDepth: 8,
SimTh: 0.3,
MaxChildren: 100,
ParamString: "<_>",
MaxClusters: 0,
// SimTh is basically a ratio of matching/total in the cluster.
// Cluster tokens: "foo <*> bar fred"
// Log line: "foo bar baz qux"
// * * * x
// Similarity of these sequences is 0.75 (the distance)
// Both SimTh and MaxClusterDepth impact branching factor: the greater
// MaxClusterDepth and SimTh, the less the chance that there will be
// "similar" clusters, but the greater the footprint.
SimTh: 0.3,
MaxChildren: 100,
ParamString: `<_>`,
MaxClusters: 300,
}
}

117
pkg/pattern/drain/drain_test.go
Unescape View File

@ -1 +1,118 @@
package drain
import (
"bufio"
"os"
"testing"
"github.com/stretchr/testify/require"
)
func TestDrain_TrainExtractsPatterns(t *testing.T) {
tests := []struct {
name string
drain *Drain
inputFile string
patterns []string
}{
{
// High variation leads to many patterns including some that are too generic (many tokens matched) and some that are too specific (too few matchers)
name: `Generate patterns on high variation logfmt logs`,
drain: New(DefaultConfig()),
inputFile: `testdata/agent-logfmt.txt`,
patterns: []string{
`ts=2024-04-16T15:10:43.192290389Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/*.log:{batch_kubernetes_io_controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", batch_kubernetes_io_job_name=\"testcoordinator-job-2665838\", container=\"testcoordinator\", controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", job=\"k6-cloud/testcoordinator\", job_name=\"testcoordinator-job-2665838\", name=\"testcoordinator\", namespace=\"k6-cloud\", pod=\"testcoordinator-job-2665838-9g8ds\"}"`,
`<_> <_> level=info component=logs logs_config=default <_> target" <_> <_> <_> <_> <_> <_>`,
`<_> caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" <_>`,
`<_> caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" <_> reason=null`,
`<_> caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" <_>`,
`<_> caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened <_>`,
`<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"`,
`<_> caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" <_> op=CREATE`,
`<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos <_> on line 1: unexpected '\"'"`,
`<_> <_> level=info component=logs logs_config=default <_> <_> <_> <_> <_>`,
`<_> caller=log.go:168 component=logs logs_config=default level=info <_> <_> <_> <_> <_>`,
`<_> caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" <_>`,
`<_> <_> level=info component=logs logs_config=default <_> target" <_> conprof=\"true\", <_> <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`,
`<_> level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines <_> <_>`,
`2024-04-16 15:10:42.555 ts=2024-04-16T15:10:42.555230437Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=38.545339ms`,
},
},
{
// Lower variation leads to fewer patterns including some with limited value (single lines, no matchers)
name: `Generate patterns on low variation logfmt logs`,
drain: New(DefaultConfig()),
inputFile: `testdata/ingester-logfmt.txt`,
patterns: []string{
`<_> caller=head.go:216 level=debug tenant=987678 msg="profile is empty after delta computation" metricName=memory`,
`ts=2024-04-17T09:52:46.363974185Z caller=http.go:194 level=debug traceID=1b48f5156a61ca69 msg="GET /debug/pprof/delta_mutex (200) 1.161082ms"`,
`<_> caller=http.go:194 level=debug <_> <_> msg="POST /ingester.v1.IngesterService/Push (200) <_>`, // A perfect log line: Abstracted the variable part but kept the constants.
},
},
{
// Lower variation logs in json leads to a high number of patterns with very few matchers
name: `Generate patterns on json formatted logs`,
drain: New(DefaultConfig()),
inputFile: `testdata/drone-json.txt`,
patterns: []string{
`<_> capacity <_>`,
`<_> capacity changes <_>`,
`{"id":"D4Oh1ivB6cdLWa08","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:48:52Z"}`,
`{"id":"q62wCcIkEOueqFKF","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:03:28Z"}`,
`{"id":"m6SpYHzdXrDAFqDR","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:58Z"}`,
`{"id":"T0I8Dsnw3uSi3Gal","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:28Z"}`,
`{"id":"9eA72xOtx8kzMhXn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:58Z"}`,
`{"id":"pet7QVfO1yE8fk56","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:28Z"}`,
`{"id":"15eSzaEG0enf86Kl","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:57Z"}`,
`{"id":"JO1OT5ADoNA8NYqr","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:27Z"}`,
`{"id":"Xz2OCJhgeBSRFyoN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:57Z"}`,
`{"id":"pPc2ORUhHAhFgBg3","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:27Z"}`,
`{"id":"4G6Srn6lSwzYrx19","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:57Z"}`,
`{"id":"1Lu90T1fWzsWOKlc","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:27Z"}`,
`{"id":"4XjwwNoOwZFaWePQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:57Z"}`,
`{"id":"IQy23J3NON0BV10V","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:26Z"}`,
`{"id":"FQ8wCQfaR9W387cH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:56Z"}`,
`{"id":"Hhwn7ecXjxF67DG6","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:26Z"}`,
`{"id":"luflyGZvZnLzhQEH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:56Z"}`,
`{"id":"q20GZcvyzMwrTGx5","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:26Z"}`,
`{"id":"3K61Yf6ImKYexoFx","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:56Z"}`,
`{"id":"SmbOO0l5aADX9BaQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:23Z"}`,
`{"id":"96TvvsMzSkkaW8oW","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:53Z"}`,
`{"id":"C7aYn8cb4NCrkkYI","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:23Z"}`,
`{"id":"CMG7ZwwYqNPBonAn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:53Z"}`,
`{"id":"focV9BzODwRbWwKE","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:23Z"}`,
`{"id":"HphRnJOM8uYohf1p","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:53Z"}`,
`{"id":"m3n8GndhG45uGIQA","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:23Z"}`,
`{"id":"nTO38tWtnvRWRl1G","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:52Z"}`,
`{"id":"5qEIzErDfiALVPAN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:22Z"}`,
`{"id":"q61oHTtF4MMiQVGH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:52Z"}`,
`{"id":"4rNxIlhDKxGgzBHe","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:22Z"}`,
`<_> server <_>`,
`<_> unfinished <_>`,
`<_> <_> (flow; linux; helm)"}`,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
file, err := os.Open(tt.inputFile)
require.NoError(t, err)
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
tt.drain.Train(line, 0)
}
var output []string
clusters := tt.drain.Clusters()
for _, cluster := range clusters {
output = append(output, cluster.String())
}
require.Equal(t, tt.patterns, output)
})
}
}

1000
pkg/pattern/drain/testdata/agent-logfmt.txt vendored
View File

File diff suppressed because it is too large Load Diff

0
pkg/pattern/drain/testdata/custom.txt vendored
Unescape View File

210
pkg/pattern/drain/testdata/drone-json.txt vendored
Unescape View File

@ -0,0 +1,210 @@
{"duration":20950259,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:03:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"q62wCcIkEOueqFKF","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:03:28Z"}
{"id":"q62wCcIkEOueqFKF","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:03:28Z"}
{"id":"q62wCcIkEOueqFKF","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:03:28Z"}
{"id":"q62wCcIkEOueqFKF","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:03:28Z"}
{"id":"q62wCcIkEOueqFKF","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:03:28Z"}
{"duration":24479472,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:03:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":56560748,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:03:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"m6SpYHzdXrDAFqDR","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:02:58Z"}
{"id":"m6SpYHzdXrDAFqDR","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:02:58Z"}
{"id":"m6SpYHzdXrDAFqDR","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:58Z"}
{"id":"m6SpYHzdXrDAFqDR","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:02:58Z"}
{"id":"m6SpYHzdXrDAFqDR","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:02:58Z"}
{"duration":36437134,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:02:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":36704414,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:02:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"T0I8Dsnw3uSi3Gal","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:02:28Z"}
{"id":"T0I8Dsnw3uSi3Gal","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:02:28Z"}
{"id":"T0I8Dsnw3uSi3Gal","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:28Z"}
{"id":"T0I8Dsnw3uSi3Gal","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:02:28Z"}
{"id":"T0I8Dsnw3uSi3Gal","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:02:28Z"}
{"duration":49120197,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:02:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":69764560,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:02:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"9eA72xOtx8kzMhXn","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:01:58Z"}
{"id":"9eA72xOtx8kzMhXn","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:01:58Z"}
{"id":"9eA72xOtx8kzMhXn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:58Z"}
{"id":"9eA72xOtx8kzMhXn","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:01:58Z"}
{"id":"9eA72xOtx8kzMhXn","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:01:58Z"}
{"duration":93395725,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:01:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":27353687,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:01:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"pet7QVfO1yE8fk56","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:01:28Z"}
{"id":"pet7QVfO1yE8fk56","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:01:28Z"}
{"id":"pet7QVfO1yE8fk56","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:28Z"}
{"id":"pet7QVfO1yE8fk56","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:01:27Z"}
{"id":"pet7QVfO1yE8fk56","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:01:27Z"}
{"duration":21783864,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:01:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":27700228,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:01:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"15eSzaEG0enf86Kl","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:00:57Z"}
{"id":"15eSzaEG0enf86Kl","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:00:57Z"}
{"id":"15eSzaEG0enf86Kl","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:57Z"}
{"id":"15eSzaEG0enf86Kl","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:00:57Z"}
{"id":"15eSzaEG0enf86Kl","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:00:57Z"}
{"duration":20800183,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:00:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":114305939,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:00:33Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"JO1OT5ADoNA8NYqr","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:00:27Z"}
{"id":"JO1OT5ADoNA8NYqr","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:00:27Z"}
{"id":"JO1OT5ADoNA8NYqr","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:27Z"}
{"id":"JO1OT5ADoNA8NYqr","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:00:27Z"}
{"id":"JO1OT5ADoNA8NYqr","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:00:27Z"}
{"duration":118009488,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:00:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":33663734,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:00:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"Xz2OCJhgeBSRFyoN","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:59:57Z"}
{"id":"Xz2OCJhgeBSRFyoN","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:59:57Z"}
{"id":"Xz2OCJhgeBSRFyoN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:57Z"}
{"id":"Xz2OCJhgeBSRFyoN","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:59:57Z"}
{"id":"Xz2OCJhgeBSRFyoN","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:59:57Z"}
{"duration":48520224,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:59:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":25172573,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:59:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"pPc2ORUhHAhFgBg3","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:59:27Z"}
{"id":"pPc2ORUhHAhFgBg3","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:59:27Z"}
{"id":"pPc2ORUhHAhFgBg3","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:27Z"}
{"id":"pPc2ORUhHAhFgBg3","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:59:27Z"}
{"id":"pPc2ORUhHAhFgBg3","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:59:27Z"}
{"duration":215980677,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:59:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":58239998,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:59:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"4G6Srn6lSwzYrx19","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:58:57Z"}
{"id":"4G6Srn6lSwzYrx19","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:58:57Z"}
{"id":"4G6Srn6lSwzYrx19","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:57Z"}
{"id":"4G6Srn6lSwzYrx19","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:58:57Z"}
{"id":"4G6Srn6lSwzYrx19","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:58:57Z"}
{"duration":33198841,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:58:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":38097220,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:58:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"1Lu90T1fWzsWOKlc","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:58:27Z"}
{"id":"1Lu90T1fWzsWOKlc","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:58:27Z"}
{"id":"1Lu90T1fWzsWOKlc","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:27Z"}
{"id":"1Lu90T1fWzsWOKlc","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:58:27Z"}
{"id":"1Lu90T1fWzsWOKlc","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:58:27Z"}
{"duration":63956469,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:58:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":125762940,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:58:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"4XjwwNoOwZFaWePQ","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:57:57Z"}
{"id":"4XjwwNoOwZFaWePQ","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:57:57Z"}
{"id":"4XjwwNoOwZFaWePQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:57Z"}
{"id":"4XjwwNoOwZFaWePQ","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:57:57Z"}
{"id":"4XjwwNoOwZFaWePQ","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:57:56Z"}
{"duration":33644691,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:57:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":24328424,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:57:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"IQy23J3NON0BV10V","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:57:26Z"}
{"id":"IQy23J3NON0BV10V","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:57:26Z"}
{"id":"IQy23J3NON0BV10V","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:26Z"}
{"id":"IQy23J3NON0BV10V","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:57:26Z"}
{"id":"IQy23J3NON0BV10V","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:57:26Z"}
{"duration":20257078,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:57:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":28190860,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:57:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"FQ8wCQfaR9W387cH","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:56:56Z"}
{"id":"FQ8wCQfaR9W387cH","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:56:56Z"}
{"id":"FQ8wCQfaR9W387cH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:56Z"}
{"id":"FQ8wCQfaR9W387cH","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:56:56Z"}
{"id":"FQ8wCQfaR9W387cH","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:56:56Z"}
{"duration":124540195,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:56:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":33392901,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:56:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"Hhwn7ecXjxF67DG6","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:56:26Z"}
{"id":"Hhwn7ecXjxF67DG6","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:56:26Z"}
{"id":"Hhwn7ecXjxF67DG6","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:26Z"}
{"id":"Hhwn7ecXjxF67DG6","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:56:26Z"}
{"id":"Hhwn7ecXjxF67DG6","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:56:26Z"}
{"duration":38704940,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:56:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":41929762,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:56:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"luflyGZvZnLzhQEH","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:55:56Z"}
{"id":"luflyGZvZnLzhQEH","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:55:56Z"}
{"id":"luflyGZvZnLzhQEH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:56Z"}
{"id":"luflyGZvZnLzhQEH","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:55:56Z"}
{"id":"luflyGZvZnLzhQEH","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:55:56Z"}
{"duration":20307958,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:55:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":20185884,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:55:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"q20GZcvyzMwrTGx5","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:55:26Z"}
{"id":"q20GZcvyzMwrTGx5","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:55:26Z"}
{"id":"q20GZcvyzMwrTGx5","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:26Z"}
{"id":"q20GZcvyzMwrTGx5","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:55:26Z"}
{"id":"q20GZcvyzMwrTGx5","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:55:26Z"}
{"duration":62732272,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:55:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":233876906,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:55:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"3K61Yf6ImKYexoFx","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:54:56Z"}
{"id":"3K61Yf6ImKYexoFx","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:54:56Z"}
{"id":"3K61Yf6ImKYexoFx","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:56Z"}
{"id":"3K61Yf6ImKYexoFx","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:54:56Z"}
{"id":"3K61Yf6ImKYexoFx","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:54:53Z"}
{"duration":91919807,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:54:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":123141860,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:54:33Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"SmbOO0l5aADX9BaQ","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:54:23Z"}
{"id":"SmbOO0l5aADX9BaQ","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:54:23Z"}
{"id":"SmbOO0l5aADX9BaQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:23Z"}
{"id":"SmbOO0l5aADX9BaQ","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:54:23Z"}
{"id":"SmbOO0l5aADX9BaQ","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:54:23Z"}
{"duration":127118328,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:54:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":154732993,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:54:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"96TvvsMzSkkaW8oW","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:53:53Z"}
{"id":"96TvvsMzSkkaW8oW","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:53:53Z"}
{"id":"96TvvsMzSkkaW8oW","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:53Z"}
{"id":"96TvvsMzSkkaW8oW","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:53:53Z"}
{"id":"96TvvsMzSkkaW8oW","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:53:53Z"}
{"duration":118361504,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:53:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":137758190,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:53:33Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"C7aYn8cb4NCrkkYI","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:53:23Z"}
{"id":"C7aYn8cb4NCrkkYI","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:53:23Z"}
{"id":"C7aYn8cb4NCrkkYI","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:23Z"}
{"id":"C7aYn8cb4NCrkkYI","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:53:23Z"}
{"id":"C7aYn8cb4NCrkkYI","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:53:23Z"}
{"duration":68794582,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:53:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":74435188,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:53:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"CMG7ZwwYqNPBonAn","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:52:53Z"}
{"id":"CMG7ZwwYqNPBonAn","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:52:53Z"}
{"id":"CMG7ZwwYqNPBonAn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:53Z"}
{"id":"CMG7ZwwYqNPBonAn","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:52:53Z"}
{"id":"CMG7ZwwYqNPBonAn","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:52:53Z"}
{"duration":50623103,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:52:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":55970830,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:52:33Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"focV9BzODwRbWwKE","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:52:23Z"}
{"id":"focV9BzODwRbWwKE","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:52:23Z"}
{"id":"focV9BzODwRbWwKE","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:23Z"}
{"id":"focV9BzODwRbWwKE","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:52:23Z"}
{"id":"focV9BzODwRbWwKE","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:52:23Z"}
{"duration":33285760,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:52:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":18306142,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:52:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"HphRnJOM8uYohf1p","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:51:53Z"}
{"id":"HphRnJOM8uYohf1p","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:51:53Z"}
{"id":"HphRnJOM8uYohf1p","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:53Z"}
{"id":"HphRnJOM8uYohf1p","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:51:53Z"}
{"id":"HphRnJOM8uYohf1p","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:51:53Z"}
{"duration":174783742,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:51:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":31536414,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:51:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"m3n8GndhG45uGIQA","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:51:23Z"}
{"id":"m3n8GndhG45uGIQA","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:51:23Z"}
{"id":"m3n8GndhG45uGIQA","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:23Z"}
{"id":"m3n8GndhG45uGIQA","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:51:22Z"}
{"id":"m3n8GndhG45uGIQA","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:51:22Z"}
{"duration":119507962,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:51:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":430410510,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:51:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"nTO38tWtnvRWRl1G","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:50:52Z"}
{"id":"nTO38tWtnvRWRl1G","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:50:52Z"}
{"id":"nTO38tWtnvRWRl1G","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:52Z"}
{"id":"nTO38tWtnvRWRl1G","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:50:52Z"}
{"id":"nTO38tWtnvRWRl1G","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:50:52Z"}
{"duration":23638914,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:50:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":20300837,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:50:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"5qEIzErDfiALVPAN","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:50:22Z"}
{"id":"5qEIzErDfiALVPAN","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:50:22Z"}
{"id":"5qEIzErDfiALVPAN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:22Z"}
{"id":"5qEIzErDfiALVPAN","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:50:22Z"}
{"id":"5qEIzErDfiALVPAN","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:50:22Z"}
{"duration":33318175,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:50:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":19013978,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:50:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"q61oHTtF4MMiQVGH","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:49:52Z"}
{"id":"q61oHTtF4MMiQVGH","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:49:52Z"}
{"id":"q61oHTtF4MMiQVGH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:52Z"}
{"id":"q61oHTtF4MMiQVGH","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:49:52Z"}
{"id":"q61oHTtF4MMiQVGH","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:49:52Z"}
{"duration":19815267,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:49:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":21037324,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:49:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"4rNxIlhDKxGgzBHe","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:49:22Z"}
{"id":"4rNxIlhDKxGgzBHe","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:49:22Z"}
{"id":"4rNxIlhDKxGgzBHe","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:22Z"}
{"id":"4rNxIlhDKxGgzBHe","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:49:22Z"}
{"id":"4rNxIlhDKxGgzBHe","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:49:22Z"}
{"duration":20219085,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:49:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"duration":24085452,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:49:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}
{"id":"D4Oh1ivB6cdLWa08","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:48:52Z"}
{"id":"D4Oh1ivB6cdLWa08","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:48:52Z"}
{"id":"D4Oh1ivB6cdLWa08","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:48:52Z"}
{"id":"D4Oh1ivB6cdLWa08","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:48:52Z"}
{"id":"D4Oh1ivB6cdLWa08","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:48:52Z"}
{"duration":51986975,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:48:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"}

100
pkg/pattern/drain/testdata/ingester-logfmt.txt vendored
Unescape View File

@ -0,0 +1,100 @@
ts=2024-04-17T09:52:46.364803306Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.368313ms"
ts=2024-04-17T09:52:46.364804708Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.619763ms"
ts=2024-04-17T09:52:46.364730588Z caller=http.go:194 level=debug traceID=116a1a427c28ef7c orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 944.101µs"
ts=2024-04-17T09:52:46.364808055Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 595.34µs"
ts=2024-04-17T09:52:46.364790582Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 845.635µs"
ts=2024-04-17T09:52:46.36478937Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 677.201µs"
ts=2024-04-17T09:52:46.36470023Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.702772ms"
ts=2024-04-17T09:52:46.364312973Z caller=http.go:194 level=debug traceID=241f9222b132d55f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.305663ms"
ts=2024-04-17T09:52:46.364492298Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 672.113µs"
ts=2024-04-17T09:52:46.36471937Z caller=http.go:194 level=debug traceID=6ff6d0cc340ca6e7 orgID=32133 msg="POST /ingester.v1.IngesterService/Push (200) 1.822532ms"
ts=2024-04-17T09:52:46.364561196Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.468961ms"
ts=2024-04-17T09:52:46.36477128Z caller=head.go:216 level=debug tenant=987678 msg="profile is empty after delta computation" metricName=memory
ts=2024-04-17T09:52:46.364749251Z caller=http.go:194 level=debug traceID=6b72cb766bf5191f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 8.508724ms"
ts=2024-04-17T09:52:46.364726948Z caller=head.go:216 level=debug tenant=987678 msg="profile is empty after delta computation" metricName=memory
ts=2024-04-17T09:52:46.364753592Z caller=http.go:194 level=debug traceID=4d4de2711f38c05c orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.564507ms"
ts=2024-04-17T09:52:46.364737024Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 967.106µs"
ts=2024-04-17T09:52:46.364734092Z caller=http.go:194 level=debug traceID=345399fe6745a50c orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 10.63998ms"
ts=2024-04-17T09:52:46.364684284Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 839.55µs"
ts=2024-04-17T09:52:46.364649689Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 783.914µs"
ts=2024-04-17T09:52:46.364650688Z caller=http.go:194 level=debug traceID=568f48b07fd4c3ef orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 874.107µs"
ts=2024-04-17T09:52:46.364616184Z caller=http.go:194 level=debug traceID=2b1bf144cf74bf28 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 17.261341ms"
ts=2024-04-17T09:52:46.364595982Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 914.739µs"
ts=2024-04-17T09:52:46.364584929Z caller=http.go:194 level=debug traceID=5b8bca4cdba21f43 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.142724ms"
ts=2024-04-17T09:52:46.364590656Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 895.782µs"
ts=2024-04-17T09:52:46.364657596Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.368331ms"
ts=2024-04-17T09:52:46.364622325Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 660.247µs"
ts=2024-04-17T09:52:46.364594116Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 467.285µs"
ts=2024-04-17T09:52:46.364545621Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 530.452µs"
ts=2024-04-17T09:52:46.36450376Z caller=http.go:194 level=debug traceID=74f2e8cf7e0c7fc7 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.135674ms"
ts=2024-04-17T09:52:46.36443397Z caller=http.go:194 level=debug traceID=6fb0c4a951fa6f0f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.398122ms"
ts=2024-04-17T09:52:46.364464147Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.131431ms"
ts=2024-04-17T09:52:46.364440826Z caller=http.go:194 level=debug traceID=34467dc44e314b02 orgID=325123 msg="POST /ingester.v1.IngesterService/Push (200) 1.023612ms"
ts=2024-04-17T09:52:46.36432296Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.408666ms"
ts=2024-04-17T09:52:46.364405022Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.341907ms"
ts=2024-04-17T09:52:46.36444769Z caller=http.go:194 level=debug traceID=568f48b07fd4c3ef orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 632.404µs"
ts=2024-04-17T09:52:46.364429591Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 478.714µs"
ts=2024-04-17T09:52:46.364341351Z caller=http.go:194 level=debug traceID=4d4de2711f38c05c orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.09688ms"
ts=2024-04-17T09:52:46.364357027Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 763.325µs"
ts=2024-04-17T09:52:46.364363024Z caller=http.go:194 level=debug traceID=6ff6d0cc340ca6e7 orgID=32133 msg="POST /ingester.v1.IngesterService/Push (200) 1.501374ms"
ts=2024-04-17T09:52:46.364308133Z caller=http.go:194 level=debug traceID=6fb0c4a951fa6f0f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.102861ms"
ts=2024-04-17T09:52:46.36423841Z caller=http.go:194 level=debug traceID=071fd5ad1d0d0515 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.527951ms"
ts=2024-04-17T09:52:46.364312232Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 858.218µs"
ts=2024-04-17T09:52:46.36428579Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 492.16µs"
ts=2024-04-17T09:52:46.361802864Z caller=http.go:194 level=debug traceID=744100a66cc00de4 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.011408ms"
ts=2024-04-17T09:52:46.364289781Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 502.847µs"
ts=2024-04-17T09:52:46.364200562Z caller=http.go:194 level=debug traceID=62968dddfc09bbbe orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.494135ms"
ts=2024-04-17T09:52:46.364282918Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 704.283µs"
ts=2024-04-17T09:52:46.364279378Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 764.795µs"
ts=2024-04-17T09:52:46.364217907Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 485.357µs"
ts=2024-04-17T09:52:46.364219229Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 837.055µs"
ts=2024-04-17T09:52:46.364112209Z caller=http.go:194 level=debug traceID=635516613bdc1d25 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 679.644µs"
ts=2024-04-17T09:52:46.364155917Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.279766ms"
ts=2024-04-17T09:52:46.364081008Z caller=http.go:194 level=debug traceID=64b8a1a256934690 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 3.267758ms"
ts=2024-04-17T09:52:46.364189774Z caller=http.go:194 level=debug traceID=34467dc44e314b02 orgID=325123 msg="POST /ingester.v1.IngesterService/Push (200) 880.75µs"
ts=2024-04-17T09:52:46.364145376Z caller=http.go:194 level=debug traceID=70121da20778353f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.053066ms"
ts=2024-04-17T09:52:46.364093541Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 660.622µs"
ts=2024-04-17T09:52:46.364068342Z caller=http.go:194 level=debug traceID=3dc61701564a23f7 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 5.097204ms"
ts=2024-04-17T09:52:46.36353869Z caller=http.go:194 level=debug traceID=4a7fddbe653f5299 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.804943ms"
ts=2024-04-17T09:52:46.362372357Z caller=http.go:194 level=debug traceID=0f348e7d30d12cff orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.009358ms"
ts=2024-04-17T09:52:46.364063967Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.368053ms"
ts=2024-04-17T09:52:46.364085113Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 918.778µs"
ts=2024-04-17T09:52:46.364063374Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 370.137µs"
ts=2024-04-17T09:52:46.358938595Z caller=http.go:194 level=debug traceID=694c1939d3e1fbc7 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 926.871µs"
ts=2024-04-17T09:52:46.364006323Z caller=http.go:194 level=debug traceID=241f9222b132d55f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.061869ms"
ts=2024-04-17T09:52:46.363969935Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 773.863µs"
ts=2024-04-17T09:52:46.36401476Z caller=http.go:194 level=debug traceID=46ecede10c2603f5 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 3.392645ms"
ts=2024-04-17T09:52:46.363873777Z caller=http.go:194 level=debug traceID=71aeed5818159731 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 805.376µs"
ts=2024-04-17T09:52:46.363974185Z caller=http.go:194 level=debug traceID=1b48f5156a61ca69 msg="GET /debug/pprof/delta_mutex (200) 1.161082ms"
ts=2024-04-17T09:52:46.363789152Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 764.051µs"
ts=2024-04-17T09:52:46.362529898Z caller=http.go:194 level=debug traceID=635516613bdc1d25 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 790.475µs"
ts=2024-04-17T09:52:46.364031457Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 476.579µs"
ts=2024-04-17T09:52:46.361323896Z caller=http.go:194 level=debug traceID=599c62a3ac7de78a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.331755ms"
ts=2024-04-17T09:52:46.363943545Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 582.609µs"
ts=2024-04-17T09:52:46.363907661Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 844.336µs"
ts=2024-04-17T09:52:46.363979594Z caller=http.go:194 level=debug traceID=6ff6d0cc340ca6e7 orgID=32133 msg="POST /ingester.v1.IngesterService/Push (200) 1.315545ms"
ts=2024-04-17T09:52:46.363939502Z caller=http.go:194 level=debug traceID=34467dc44e314b02 orgID=325123 msg="POST /ingester.v1.IngesterService/Push (200) 758.566µs"
ts=2024-04-17T09:52:46.363924229Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 443.231µs"
ts=2024-04-17T09:52:46.363882565Z caller=http.go:194 level=debug traceID=71aeed5818159731 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 746.182µs"
ts=2024-04-17T09:52:46.363850696Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 487.255µs"
ts=2024-04-17T09:52:46.363769122Z caller=http.go:194 level=debug traceID=04ece9ea52939418 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 14.805487ms"
ts=2024-04-17T09:52:46.363816282Z caller=http.go:194 level=debug traceID=0052911cab18a35d orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 4.99224ms"
ts=2024-04-17T09:52:46.363754816Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 931.934µs"
ts=2024-04-17T09:52:46.363705611Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 744.005µs"
ts=2024-04-17T09:52:46.36377788Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 956.91µs"
ts=2024-04-17T09:52:46.363732729Z caller=http.go:194 level=debug traceID=6e6e579dac547044 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.036865ms"
ts=2024-04-17T09:52:46.36376503Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 851.996µs"
ts=2024-04-17T09:52:46.36375519Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 845.405µs"
ts=2024-04-17T09:52:46.363674007Z caller=http.go:194 level=debug traceID=0b0e675689e15909 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.177299ms"
ts=2024-04-17T09:52:46.363672895Z caller=http.go:194 level=debug traceID=554713b69cd1473a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.611307ms"
ts=2024-04-17T09:52:46.36371749Z caller=http.go:194 level=debug traceID=70121da20778353f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 846.904µs"
ts=2024-04-17T09:52:46.363695016Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 529.001µs"
ts=2024-04-17T09:52:46.363644321Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 440.298µs"
ts=2024-04-17T09:52:46.363603421Z caller=http.go:194 level=debug traceID=748df0ddf0b0f22d orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.562106ms"
ts=2024-04-17T09:52:46.363685875Z caller=http.go:194 level=debug traceID=71aeed5818159731 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 701.457µs"
ts=2024-04-17T09:52:46.363659588Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 729.084µs"
ts=2024-04-17T09:52:46.363561079Z caller=http.go:194 level=debug traceID=635516613bdc1d25 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 870.913µs"
ts=2024-04-17T09:52:46.363625435Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 560.428µs"
ts=2024-04-17T09:52:46.36352246Z caller=http.go:194 level=debug traceID=0650b8c486d05d36 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.104667ms"
ts=2024-04-17T09:52:46.363534538Z caller=http.go:194 level=debug traceID=5269eebfe73cfedb orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.110652ms"
ts=2024-04-17T09:52:46.363579783Z caller=http.go:194 level=debug traceID=64b8a1a256934690 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 3.154879ms"

2
pkg/pattern/ingester_querier.go
Unescape View File

@ -67,7 +67,7 @@ func (q *IngesterQuerier) Patterns(ctx context.Context, req *logproto.QueryPatte
}
func prunePatterns(resp *logproto.QueryPatternsResponse, minClusterSize int) *logproto.QueryPatternsResponse {
d := drain.New(drainConfig)
d := drain.New(drain.DefaultConfig())
for _, p := range resp.Series {
d.TrainPattern(p.Pattern, p.Samples)
}

45
pkg/pattern/stream.go
Unescape View File

@ -13,49 +13,6 @@ import (
"github.com/prometheus/prometheus/model/labels"
)
// TODO(kolesnikovae):
//
// This is crucial for Drain to ensure that the first LogClusterDepth tokens
// are constant (see https://jiemingzhu.github.io/pub/pjhe_icws2017.pdf).
// We should remove any variables such as timestamps, IDs, IPs, counters, etc.
// from these tokens.
//
// Moreover, Drain is not designed for structured logs. Therefore, we should
// handle logfmt (and, probably, JSON) logs in a special way:
//
// The parse tree should have a fixed length, and the depth should be
// determined by the number of fields in the logfmt message.
// A parsing tree should be maintained for each unique field set.
var drainConfig = &drain.Config{
// At training, if at the depth of LogClusterDepth there is a cluster with
// similarity coefficient greater that SimTh, then the log message is added
// to that cluster. Otherwise, a new cluster is created.
//
// LogClusterDepth should be equal to the number of constant tokens from
// the beginning of the message that likely determine the message contents.
//
// > In this step, Drain traverses from a 1-st layer node, which
// > is searched in step 2, to a leaf node. This step is based on
// > the assumption that tokens in the beginning positions of a log
// > message are more likely to be constants. Specifically, Drain
// > selects the next internal node by the tokens in the beginning
// > positions of the log message
LogClusterDepth: 8,
// SimTh is basically a ratio of matching/total in the cluster.
// Cluster tokens: "foo <*> bar fred"
// Log line: "foo bar baz qux"
// * * * x
// Similarity of these sequences is 0.75 (the distance)
// Both SimTh and MaxClusterDepth impact branching factor: the greater
// MaxClusterDepth and SimTh, the less the chance that there will be
// "similar" clusters, but the greater the footprint.
SimTh: 0.3,
MaxChildren: 100,
ParamString: "<_>",
MaxClusters: 300,
}
type stream struct {
fp model.Fingerprint
labels labels.Labels
@ -76,7 +33,7 @@ func newStream(
labels: labels,
labelsString: labels.String(),
labelHash: labels.Hash(),
patterns: drain.New(drainConfig),
patterns: drain.New(drain.DefaultConfig()),
}, nil
}

Write Preview
Loading…
Cancel
Save