mirror of https://github.com/grafana/loki
chore(level_detection): Make log level detection configurable (#14784)
Shantanu Alshi
7 months ago
committed by
GitHub
parent
126ebeda0d
commit
867ce3d711
@ -0,0 +1,226 @@ |
||||
package distributor |
||||
|
||||
import ( |
||||
"bytes" |
||||
"strconv" |
||||
"strings" |
||||
"unicode" |
||||
"unsafe" |
||||
|
||||
"github.com/buger/jsonparser" |
||||
"github.com/prometheus/prometheus/model/labels" |
||||
"go.opentelemetry.io/collector/pdata/plog" |
||||
|
||||
"github.com/grafana/loki/v3/pkg/loghttp/push" |
||||
"github.com/grafana/loki/v3/pkg/logproto" |
||||
"github.com/grafana/loki/v3/pkg/logql/log/logfmt" |
||||
"github.com/grafana/loki/v3/pkg/util/constants" |
||||
) |
||||
|
||||
var ( |
||||
trace = []byte("trace") |
||||
traceAbbrv = []byte("trc") |
||||
debug = []byte("debug") |
||||
debugAbbrv = []byte("dbg") |
||||
info = []byte("info") |
||||
infoAbbrv = []byte("inf") |
||||
warn = []byte("warn") |
||||
warnAbbrv = []byte("wrn") |
||||
warning = []byte("warning") |
||||
errorStr = []byte("error") |
||||
errorAbbrv = []byte("err") |
||||
critical = []byte("critical") |
||||
fatal = []byte("fatal") |
||||
) |
||||
|
||||
func allowedLabelsForLevel(allowedFields []string) map[string]struct{} { |
||||
if len(allowedFields) == 0 { |
||||
return map[string]struct{}{ |
||||
"level": {}, "LEVEL": {}, "Level": {}, |
||||
"severity": {}, "SEVERITY": {}, "Severity": {}, |
||||
"lvl": {}, "LVL": {}, "Lvl": {}, |
||||
} |
||||
} |
||||
allowedFieldsMap := make(map[string]struct{}, len(allowedFields)) |
||||
for _, field := range allowedFields { |
||||
allowedFieldsMap[field] = struct{}{} |
||||
} |
||||
return allowedFieldsMap |
||||
} |
||||
|
||||
type LevelDetector struct { |
||||
validationContext validationContext |
||||
allowedLabels map[string]struct{} |
||||
} |
||||
|
||||
func newLevelDetector(validationContext validationContext) *LevelDetector { |
||||
logLevelFields := validationContext.logLevelFields |
||||
return &LevelDetector{ |
||||
validationContext: validationContext, |
||||
allowedLabels: allowedLabelsForLevel(logLevelFields), |
||||
} |
||||
} |
||||
|
||||
func (l *LevelDetector) shouldDiscoverLogLevels() bool { |
||||
return l.validationContext.allowStructuredMetadata && l.validationContext.discoverLogLevels |
||||
} |
||||
|
||||
func (l *LevelDetector) extractLogLevel(labels labels.Labels, structuredMetadata labels.Labels, entry logproto.Entry) (logproto.LabelAdapter, bool) { |
||||
levelFromLabel, hasLevelLabel := l.hasAnyLevelLabels(labels) |
||||
var logLevel string |
||||
if hasLevelLabel { |
||||
logLevel = levelFromLabel |
||||
} else if levelFromMetadata, ok := l.hasAnyLevelLabels(structuredMetadata); ok { |
||||
logLevel = levelFromMetadata |
||||
} else { |
||||
logLevel = l.detectLogLevelFromLogEntry(entry, structuredMetadata) |
||||
} |
||||
|
||||
if logLevel == "" { |
||||
return logproto.LabelAdapter{}, false |
||||
} |
||||
return logproto.LabelAdapter{ |
||||
Name: constants.LevelLabel, |
||||
Value: logLevel, |
||||
}, true |
||||
} |
||||
|
||||
func (l *LevelDetector) hasAnyLevelLabels(labels labels.Labels) (string, bool) { |
||||
for lbl := range l.allowedLabels { |
||||
if labels.Has(lbl) { |
||||
return labels.Get(lbl), true |
||||
} |
||||
} |
||||
return "", false |
||||
} |
||||
|
||||
func (l *LevelDetector) detectLogLevelFromLogEntry(entry logproto.Entry, structuredMetadata labels.Labels) string { |
||||
// otlp logs have a severity number, using which we are defining the log levels.
|
||||
// Significance of severity number is explained in otel docs here https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
|
||||
if otlpSeverityNumberTxt := structuredMetadata.Get(push.OTLPSeverityNumber); otlpSeverityNumberTxt != "" { |
||||
otlpSeverityNumber, err := strconv.Atoi(otlpSeverityNumberTxt) |
||||
if err != nil { |
||||
return constants.LogLevelInfo |
||||
} |
||||
if otlpSeverityNumber == int(plog.SeverityNumberUnspecified) { |
||||
return constants.LogLevelUnknown |
||||
} else if otlpSeverityNumber <= int(plog.SeverityNumberTrace4) { |
||||
return constants.LogLevelTrace |
||||
} else if otlpSeverityNumber <= int(plog.SeverityNumberDebug4) { |
||||
return constants.LogLevelDebug |
||||
} else if otlpSeverityNumber <= int(plog.SeverityNumberInfo4) { |
||||
return constants.LogLevelInfo |
||||
} else if otlpSeverityNumber <= int(plog.SeverityNumberWarn4) { |
||||
return constants.LogLevelWarn |
||||
} else if otlpSeverityNumber <= int(plog.SeverityNumberError4) { |
||||
return constants.LogLevelError |
||||
} else if otlpSeverityNumber <= int(plog.SeverityNumberFatal4) { |
||||
return constants.LogLevelFatal |
||||
} |
||||
return constants.LogLevelUnknown |
||||
} |
||||
|
||||
return l.extractLogLevelFromLogLine(entry.Line) |
||||
} |
||||
|
||||
func (l *LevelDetector) extractLogLevelFromLogLine(log string) string { |
||||
logSlice := unsafe.Slice(unsafe.StringData(log), len(log)) |
||||
var v []byte |
||||
if isJSON(log) { |
||||
v = l.getValueUsingJSONParser(logSlice) |
||||
} else if isLogFmt(logSlice) { |
||||
v = l.getValueUsingLogfmtParser(logSlice) |
||||
} else { |
||||
return detectLevelFromLogLine(log) |
||||
} |
||||
|
||||
switch { |
||||
case bytes.EqualFold(v, trace), bytes.EqualFold(v, traceAbbrv): |
||||
return constants.LogLevelTrace |
||||
case bytes.EqualFold(v, debug), bytes.EqualFold(v, debugAbbrv): |
||||
return constants.LogLevelDebug |
||||
case bytes.EqualFold(v, info), bytes.EqualFold(v, infoAbbrv): |
||||
return constants.LogLevelInfo |
||||
case bytes.EqualFold(v, warn), bytes.EqualFold(v, warnAbbrv), bytes.EqualFold(v, warning): |
||||
return constants.LogLevelWarn |
||||
case bytes.EqualFold(v, errorStr), bytes.EqualFold(v, errorAbbrv): |
||||
return constants.LogLevelError |
||||
case bytes.EqualFold(v, critical): |
||||
return constants.LogLevelCritical |
||||
case bytes.EqualFold(v, fatal): |
||||
return constants.LogLevelFatal |
||||
default: |
||||
return detectLevelFromLogLine(log) |
||||
} |
||||
} |
||||
|
||||
func (l *LevelDetector) getValueUsingLogfmtParser(line []byte) []byte { |
||||
d := logfmt.NewDecoder(line) |
||||
for !d.EOL() && d.ScanKeyval() { |
||||
if _, ok := l.allowedLabels[string(d.Key())]; ok { |
||||
return d.Value() |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (l *LevelDetector) getValueUsingJSONParser(log []byte) []byte { |
||||
for allowedLabel := range l.allowedLabels { |
||||
l, _, _, err := jsonparser.Get(log, allowedLabel) |
||||
if err == nil { |
||||
return l |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func isLogFmt(line []byte) bool { |
||||
equalIndex := bytes.Index(line, []byte("=")) |
||||
if len(line) == 0 || equalIndex == -1 { |
||||
return false |
||||
} |
||||
return true |
||||
} |
||||
|
||||
func isJSON(line string) bool { |
||||
var firstNonSpaceChar rune |
||||
for _, char := range line { |
||||
if !unicode.IsSpace(char) { |
||||
firstNonSpaceChar = char |
||||
break |
||||
} |
||||
} |
||||
|
||||
var lastNonSpaceChar rune |
||||
for i := len(line) - 1; i >= 0; i-- { |
||||
char := rune(line[i]) |
||||
if !unicode.IsSpace(char) { |
||||
lastNonSpaceChar = char |
||||
break |
||||
} |
||||
} |
||||
|
||||
return firstNonSpaceChar == '{' && lastNonSpaceChar == '}' |
||||
} |
||||
|
||||
func detectLevelFromLogLine(log string) string { |
||||
if strings.Contains(log, "info:") || strings.Contains(log, "INFO:") || |
||||
strings.Contains(log, "info") || strings.Contains(log, "INFO") { |
||||
return constants.LogLevelInfo |
||||
} |
||||
if strings.Contains(log, "err:") || strings.Contains(log, "ERR:") || |
||||
strings.Contains(log, "error") || strings.Contains(log, "ERROR") { |
||||
return constants.LogLevelError |
||||
} |
||||
if strings.Contains(log, "warn:") || strings.Contains(log, "WARN:") || |
||||
strings.Contains(log, "warning") || strings.Contains(log, "WARNING") { |
||||
return constants.LogLevelWarn |
||||
} |
||||
if strings.Contains(log, "CRITICAL:") || strings.Contains(log, "critical:") { |
||||
return constants.LogLevelCritical |
||||
} |
||||
if strings.Contains(log, "debug:") || strings.Contains(log, "DEBUG:") { |
||||
return constants.LogLevelDebug |
||||
} |
||||
return constants.LogLevelUnknown |
||||
} |
||||
@ -0,0 +1,436 @@ |
||||
package distributor |
||||
|
||||
import ( |
||||
"fmt" |
||||
"testing" |
||||
|
||||
"github.com/grafana/dskit/flagext" |
||||
ring_client "github.com/grafana/dskit/ring/client" |
||||
"github.com/stretchr/testify/require" |
||||
"go.opentelemetry.io/collector/pdata/plog" |
||||
|
||||
loghttp_push "github.com/grafana/loki/v3/pkg/loghttp/push" |
||||
"github.com/grafana/loki/v3/pkg/logproto" |
||||
"github.com/grafana/loki/v3/pkg/util/constants" |
||||
"github.com/grafana/loki/v3/pkg/validation" |
||||
|
||||
"github.com/grafana/loki/pkg/push" |
||||
) |
||||
|
||||
func Test_DetectLogLevels(t *testing.T) { |
||||
setup := func(discoverLogLevels bool) (*validation.Limits, *mockIngester) { |
||||
limits := &validation.Limits{} |
||||
flagext.DefaultValues(limits) |
||||
|
||||
limits.DiscoverLogLevels = discoverLogLevels |
||||
limits.AllowStructuredMetadata = true |
||||
return limits, &mockIngester{} |
||||
} |
||||
|
||||
t.Run("log level detection disabled", func(t *testing.T) { |
||||
limits, ingester := setup(false) |
||||
distributors, _ := prepare(t, 1, 5, limits, func(_ string) (ring_client.PoolClient, error) { return ingester, nil }) |
||||
|
||||
writeReq := makeWriteRequestWithLabels(1, 10, []string{`{foo="bar"}`}) |
||||
_, err := distributors[0].Push(ctx, writeReq) |
||||
require.NoError(t, err) |
||||
topVal := ingester.Peek() |
||||
require.Equal(t, `{foo="bar"}`, topVal.Streams[0].Labels) |
||||
require.Len(t, topVal.Streams[0].Entries[0].StructuredMetadata, 0) |
||||
}) |
||||
|
||||
t.Run("log level detection enabled but level cannot be detected", func(t *testing.T) { |
||||
limits, ingester := setup(true) |
||||
distributors, _ := prepare(t, 1, 5, limits, func(_ string) (ring_client.PoolClient, error) { return ingester, nil }) |
||||
|
||||
writeReq := makeWriteRequestWithLabels(1, 10, []string{`{foo="bar"}`}) |
||||
_, err := distributors[0].Push(ctx, writeReq) |
||||
require.NoError(t, err) |
||||
topVal := ingester.Peek() |
||||
require.Equal(t, `{foo="bar"}`, topVal.Streams[0].Labels) |
||||
require.Len(t, topVal.Streams[0].Entries[0].StructuredMetadata, 1) |
||||
}) |
||||
|
||||
t.Run("log level detection enabled and warn logs", func(t *testing.T) { |
||||
for _, level := range []string{"warn", "Wrn", "WARNING"} { |
||||
limits, ingester := setup(true) |
||||
distributors, _ := prepare( |
||||
t, |
||||
1, |
||||
5, |
||||
limits, |
||||
func(_ string) (ring_client.PoolClient, error) { return ingester, nil }, |
||||
) |
||||
|
||||
writeReq := makeWriteRequestWithLabelsWithLevel(1, 10, []string{`{foo="bar"}`}, level) |
||||
_, err := distributors[0].Push(ctx, writeReq) |
||||
require.NoError(t, err) |
||||
topVal := ingester.Peek() |
||||
require.Equal(t, `{foo="bar"}`, topVal.Streams[0].Labels) |
||||
require.Equal(t, push.LabelsAdapter{ |
||||
{ |
||||
Name: constants.LevelLabel, |
||||
Value: constants.LogLevelWarn, |
||||
}, |
||||
}, topVal.Streams[0].Entries[0].StructuredMetadata, fmt.Sprintf("level: %s", level)) |
||||
} |
||||
}) |
||||
|
||||
t.Run("log level detection enabled but log level already present in stream", func(t *testing.T) { |
||||
limits, ingester := setup(true) |
||||
distributors, _ := prepare(t, 1, 5, limits, func(_ string) (ring_client.PoolClient, error) { return ingester, nil }) |
||||
|
||||
writeReq := makeWriteRequestWithLabels(1, 10, []string{`{foo="bar", level="debug"}`}) |
||||
_, err := distributors[0].Push(ctx, writeReq) |
||||
require.NoError(t, err) |
||||
topVal := ingester.Peek() |
||||
require.Equal(t, `{foo="bar", level="debug"}`, topVal.Streams[0].Labels) |
||||
sm := topVal.Streams[0].Entries[0].StructuredMetadata |
||||
require.Len(t, sm, 1) |
||||
require.Equal(t, sm[0].Name, constants.LevelLabel) |
||||
require.Equal(t, sm[0].Value, constants.LogLevelDebug) |
||||
}) |
||||
|
||||
t.Run("log level detection enabled but log level already present as structured metadata", func(t *testing.T) { |
||||
limits, ingester := setup(true) |
||||
distributors, _ := prepare(t, 1, 5, limits, func(_ string) (ring_client.PoolClient, error) { return ingester, nil }) |
||||
|
||||
writeReq := makeWriteRequestWithLabels(1, 10, []string{`{foo="bar"}`}) |
||||
writeReq.Streams[0].Entries[0].StructuredMetadata = push.LabelsAdapter{ |
||||
{ |
||||
Name: "severity", |
||||
Value: constants.LogLevelWarn, |
||||
}, |
||||
} |
||||
_, err := distributors[0].Push(ctx, writeReq) |
||||
require.NoError(t, err) |
||||
topVal := ingester.Peek() |
||||
require.Equal(t, `{foo="bar"}`, topVal.Streams[0].Labels) |
||||
sm := topVal.Streams[0].Entries[0].StructuredMetadata |
||||
require.Equal(t, push.LabelsAdapter{ |
||||
{ |
||||
Name: "severity", |
||||
Value: constants.LogLevelWarn, |
||||
}, { |
||||
Name: constants.LevelLabel, |
||||
Value: constants.LogLevelWarn, |
||||
}, |
||||
}, sm) |
||||
}) |
||||
} |
||||
|
||||
func Test_detectLogLevelFromLogEntry(t *testing.T) { |
||||
ld := newLevelDetector( |
||||
validationContext{ |
||||
discoverLogLevels: true, |
||||
allowStructuredMetadata: true, |
||||
logLevelFields: []string{"level", "LEVEL", "Level", "severity", "SEVERITY", "Severity", "lvl", "LVL", "Lvl"}, |
||||
}) |
||||
|
||||
for _, tc := range []struct { |
||||
name string |
||||
entry logproto.Entry |
||||
expectedLogLevel string |
||||
}{ |
||||
{ |
||||
name: "use severity number from otlp logs", |
||||
entry: logproto.Entry{ |
||||
Line: "error", |
||||
StructuredMetadata: push.LabelsAdapter{ |
||||
{ |
||||
Name: loghttp_push.OTLPSeverityNumber, |
||||
Value: fmt.Sprintf("%d", plog.SeverityNumberDebug3), |
||||
}, |
||||
}, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelDebug, |
||||
}, |
||||
{ |
||||
name: "invalid severity number should not cause any issues", |
||||
entry: logproto.Entry{ |
||||
StructuredMetadata: push.LabelsAdapter{ |
||||
{ |
||||
Name: loghttp_push.OTLPSeverityNumber, |
||||
Value: "foo", |
||||
}, |
||||
}, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelInfo, |
||||
}, |
||||
{ |
||||
name: "non otlp without any of the log level keywords in log line", |
||||
entry: logproto.Entry{ |
||||
Line: "foo", |
||||
}, |
||||
expectedLogLevel: constants.LogLevelUnknown, |
||||
}, |
||||
{ |
||||
name: "non otlp with log level keywords in log line", |
||||
entry: logproto.Entry{ |
||||
Line: "this is a warning log", |
||||
}, |
||||
expectedLogLevel: constants.LogLevelWarn, |
||||
}, |
||||
{ |
||||
name: "json log line with an error", |
||||
entry: logproto.Entry{ |
||||
Line: `{"foo":"bar","msg":"message with keyword error but it should not get picked up","level":"critical"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelCritical, |
||||
}, |
||||
{ |
||||
name: "json log line with an error", |
||||
entry: logproto.Entry{ |
||||
Line: `{"FOO":"bar","MSG":"message with keyword error but it should not get picked up","LEVEL":"Critical"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelCritical, |
||||
}, |
||||
{ |
||||
name: "json log line with an warning", |
||||
entry: logproto.Entry{ |
||||
Line: `{"foo":"bar","msg":"message with keyword warn but it should not get picked up","level":"warn"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelWarn, |
||||
}, |
||||
{ |
||||
name: "json log line with an warning", |
||||
entry: logproto.Entry{ |
||||
Line: `{"foo":"bar","msg":"message with keyword warn but it should not get picked up","SEVERITY":"FATAL"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelFatal, |
||||
}, |
||||
{ |
||||
name: "json log line with an error in block case", |
||||
entry: logproto.Entry{ |
||||
Line: `{"foo":"bar","msg":"message with keyword warn but it should not get picked up","level":"ERR"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelError, |
||||
}, |
||||
{ |
||||
name: "json log line with an INFO in block case", |
||||
entry: logproto.Entry{ |
||||
Line: `{"foo":"bar","msg":"message with keyword INFO get picked up"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelInfo, |
||||
}, |
||||
{ |
||||
name: "logfmt log line with an INFO and not level returns info log level", |
||||
entry: logproto.Entry{ |
||||
Line: `foo=bar msg="message with info and not level should get picked up"`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelInfo, |
||||
}, |
||||
{ |
||||
name: "logfmt log line with a warn", |
||||
entry: logproto.Entry{ |
||||
Line: `foo=bar msg="message with keyword error but it should not get picked up" level=warn`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelWarn, |
||||
}, |
||||
{ |
||||
name: "logfmt log line with a warn with camel case", |
||||
entry: logproto.Entry{ |
||||
Line: `foo=bar msg="message with keyword error but it should not get picked up" level=Warn`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelWarn, |
||||
}, |
||||
{ |
||||
name: "logfmt log line with a trace", |
||||
entry: logproto.Entry{ |
||||
Line: `foo=bar msg="message with keyword error but it should not get picked up" level=Trace`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelTrace, |
||||
}, |
||||
{ |
||||
name: "logfmt log line with some other level returns unknown log level", |
||||
entry: logproto.Entry{ |
||||
Line: `foo=bar msg="message with keyword but it should not get picked up" level=NA`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelUnknown, |
||||
}, |
||||
{ |
||||
name: "logfmt log line with label Severity is allowed for level detection", |
||||
entry: logproto.Entry{ |
||||
Line: `foo=bar msg="message with keyword but it should not get picked up" severity=critical`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelCritical, |
||||
}, |
||||
{ |
||||
name: "logfmt log line with label Severity with camelcase is allowed for level detection", |
||||
entry: logproto.Entry{ |
||||
Line: `Foo=bar MSG="Message with keyword but it should not get picked up" Severity=critical`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelCritical, |
||||
}, |
||||
{ |
||||
name: "logfmt log line with a info with non standard case", |
||||
entry: logproto.Entry{ |
||||
Line: `foo=bar msg="message with keyword error but it should not get picked up" level=inFO`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelInfo, |
||||
}, |
||||
{ |
||||
name: "logfmt log line with a info with non block case for level", |
||||
entry: logproto.Entry{ |
||||
Line: `FOO=bar MSG="message with keyword error but it should not get picked up" LEVEL=inFO`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelInfo, |
||||
}, |
||||
} { |
||||
t.Run(tc.name, func(t *testing.T) { |
||||
detectedLogLevel := ld.detectLogLevelFromLogEntry(tc.entry, logproto.FromLabelAdaptersToLabels(tc.entry.StructuredMetadata)) |
||||
require.Equal(t, tc.expectedLogLevel, detectedLogLevel) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_detectLogLevelFromLogEntryWithCustomLabels(t *testing.T) { |
||||
ld := newLevelDetector( |
||||
validationContext{ |
||||
discoverLogLevels: true, |
||||
allowStructuredMetadata: true, |
||||
logLevelFields: []string{"log_level", "logging_level", "LOGGINGLVL", "lvl"}, |
||||
}) |
||||
|
||||
for _, tc := range []struct { |
||||
name string |
||||
entry logproto.Entry |
||||
expectedLogLevel string |
||||
}{ |
||||
{ |
||||
name: "use severity number from otlp logs", |
||||
entry: logproto.Entry{ |
||||
Line: "error", |
||||
StructuredMetadata: push.LabelsAdapter{ |
||||
{ |
||||
Name: loghttp_push.OTLPSeverityNumber, |
||||
Value: fmt.Sprintf("%d", plog.SeverityNumberDebug3), |
||||
}, |
||||
}, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelDebug, |
||||
}, |
||||
{ |
||||
name: "invalid severity number should not cause any issues", |
||||
entry: logproto.Entry{ |
||||
StructuredMetadata: push.LabelsAdapter{ |
||||
{ |
||||
Name: loghttp_push.OTLPSeverityNumber, |
||||
Value: "foo", |
||||
}, |
||||
}, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelInfo, |
||||
}, |
||||
{ |
||||
name: "non otlp without any of the log level keywords in log line", |
||||
entry: logproto.Entry{ |
||||
Line: "foo", |
||||
}, |
||||
expectedLogLevel: constants.LogLevelUnknown, |
||||
}, |
||||
{ |
||||
name: "non otlp with log level keywords in log line", |
||||
entry: logproto.Entry{ |
||||
Line: "this is a warning log", |
||||
}, |
||||
expectedLogLevel: constants.LogLevelWarn, |
||||
}, |
||||
{ |
||||
name: "json log line with an error", |
||||
entry: logproto.Entry{ |
||||
Line: `{"foo":"bar","msg":"message with keyword error but it should not get picked up","log_level":"critical"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelCritical, |
||||
}, |
||||
{ |
||||
name: "json log line with an error", |
||||
entry: logproto.Entry{ |
||||
Line: `{"FOO":"bar","MSG":"message with keyword error but it should not get picked up","LOGGINGLVL":"Critical"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelCritical, |
||||
}, |
||||
{ |
||||
name: "json log line with an warning", |
||||
entry: logproto.Entry{ |
||||
Line: `{"foo":"bar","msg":"message with keyword warn but it should not get picked up","lvl":"warn"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelWarn, |
||||
}, |
||||
{ |
||||
name: "json log line with an warning", |
||||
entry: logproto.Entry{ |
||||
Line: `{"foo":"bar","msg":"message with keyword warn but it should not get picked up","LOGGINGLVL":"FATAL"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelFatal, |
||||
}, |
||||
{ |
||||
name: "json log line with an error in block case", |
||||
entry: logproto.Entry{ |
||||
Line: `{"foo":"bar","msg":"message with keyword warn but it should not get picked up","logging_level":"ERR"}`, |
||||
}, |
||||
expectedLogLevel: constants.LogLevelError, |
||||
}, |
||||
} { |
||||
t.Run(tc.name, func(t *testing.T) { |
||||
detectedLogLevel := ld.detectLogLevelFromLogEntry(tc.entry, logproto.FromLabelAdaptersToLabels(tc.entry.StructuredMetadata)) |
||||
require.Equal(t, tc.expectedLogLevel, detectedLogLevel) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Benchmark_extractLogLevelFromLogLine(b *testing.B) { |
||||
// looks scary, but it is some random text of about 1000 chars from charset a-zA-Z0-9
|
||||
logLine := "dGzJ6rKk Zj U04SWEqEK4Uwho8 DpNyLz0 Nfs61HJ fz5iKVigg 44 kabOz7ghviGmVONriAdz4lA 7Kis1OTvZGT3 " + |
||||
"ZB6ioK4fgJLbzm AuIcbnDZKx3rZ aeZJQzRb3zhrn vok8Efav6cbyzbRUQ PYsEdQxCpdCDcGNsKG FVwe61 nhF06t9hXSNySEWa " + |
||||
"gBAXP1J8oEL grep1LfeKjA23ntszKA A772vNyxjQF SjWfJypwI7scxk oLlqRzrDl ostO4CCwx01wDB7Utk0 64A7p5eQDITE6zc3 " + |
||||
"rGL DrPnD K2oj Vro2JEvI2YScstnMx SVu H o GUl8fxZJJ1HY0 C QOA HNJr5XtsCNRrLi 0w C0Pd8XWbVZyQkSlsRm zFw1lW " + |
||||
"c8j6JFQuQnnB EyL20z0 2Duo0dvynnAGD 45ut2Z Jrz8Nd7Pmg 5oQ09r9vnmy U2 mKHO5uBfndPnbjbr mzOvQs9bM1 9e " + |
||||
"yvNSfcbPyhuWvB VKJt2kp8IoTVc XCe Uva5mp9NrGh3TEbjQu1 C Zvdk uPr7St2m kwwMRcS9eC aS6ZuL48eoQUiKo VBPd4m49ymr " + |
||||
"eQZ0fbjWpj6qA A6rYs4E 58dqh9ntu8baziDJ4c 1q6aVEig YrMXTF hahrlt 6hKVHfZLFZ V 9hEVN0WKgcpu6L zLxo6YC57 XQyfAGpFM " + |
||||
"Wm3 S7if5qCXPzvuMZ2 gNHdst Z39s9uNc58QBDeYRW umyIF BDqEdqhE tAs2gidkqee3aux8b NLDb7 ZZLekc0cQZ GUKQuBg2pL2y1S " + |
||||
"RJtBuW ABOqQHLSlNuUw ZlM2nGS2 jwA7cXEOJhY 3oPv4gGAz Uqdre16MF92C06jOH dayqTCK8XmIilT uvgywFSfNadYvRDQa " + |
||||
"iUbswJNcwqcr6huw LAGrZS8NGlqqzcD2wFU rm Uqcrh3TKLUCkfkwLm 5CIQbxMCUz boBrEHxvCBrUo YJoF2iyif4xq3q yk " |
||||
ld := &LevelDetector{ |
||||
validationContext: validationContext{ |
||||
discoverLogLevels: true, |
||||
allowStructuredMetadata: true, |
||||
logLevelFields: []string{"level", "LEVEL", "Level", "severity", "SEVERITY", "Severity", "lvl", "LVL", "Lvl"}, |
||||
}, |
||||
} |
||||
for i := 0; i < b.N; i++ { |
||||
level := ld.extractLogLevelFromLogLine(logLine) |
||||
require.Equal(b, constants.LogLevelUnknown, level) |
||||
} |
||||
} |
||||
|
||||
func Benchmark_optParseExtractLogLevelFromLogLineJson(b *testing.B) { |
||||
logLine := `{"msg": "something" , "level": "error", "id": "1"}` |
||||
|
||||
ld := newLevelDetector( |
||||
validationContext{ |
||||
discoverLogLevels: true, |
||||
allowStructuredMetadata: true, |
||||
logLevelFields: []string{"level", "LEVEL", "Level", "severity", "SEVERITY", "Severity", "lvl", "LVL", "Lvl"}, |
||||
}) |
||||
|
||||
for i := 0; i < b.N; i++ { |
||||
level := ld.extractLogLevelFromLogLine(logLine) |
||||
require.Equal(b, constants.LogLevelError, level) |
||||
} |
||||
} |
||||
|
||||
func Benchmark_optParseExtractLogLevelFromLogLineLogfmt(b *testing.B) { |
||||
logLine := `FOO=bar MSG="message with keyword error but it should not get picked up" LEVEL=inFO` |
||||
ld := newLevelDetector( |
||||
validationContext{ |
||||
discoverLogLevels: true, |
||||
allowStructuredMetadata: true, |
||||
logLevelFields: []string{"level", "LEVEL", "Level", "severity", "SEVERITY", "Severity", "lvl", "LVL", "Lvl"}, |
||||
}) |
||||
|
||||
for i := 0; i < b.N; i++ { |
||||
level := ld.extractLogLevelFromLogLine(logLine) |
||||
require.Equal(b, constants.LogLevelInfo, level) |
||||
} |
||||
} |
||||
Loading…
Reference in new issue