mirror of https://github.com/grafana/loki
renamed `non indexed labels` feature to `structured metadata` (#10432)
Loki team has made a decision to rename the feature because the new name is more precise. --------- Signed-off-by: Vladyslav Diachenko <vlad.diachenko@grafana.com> Co-authored-by: Salva Corts <salva.corts@grafana.com>pull/10437/head
Vladyslav Diachenko
2 years ago
committed by
GitHub
parent
37ec68eebb
commit
00a3c5b026
@ -1,132 +0,0 @@ |
||||
package stages |
||||
|
||||
import ( |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/prometheus/client_golang/prometheus" |
||||
"github.com/prometheus/common/model" |
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/grafana/loki/pkg/push" |
||||
util_log "github.com/grafana/loki/pkg/util/log" |
||||
) |
||||
|
||||
var pipelineStagesNonIndexedLabelsUsingMatch = ` |
||||
pipeline_stages: |
||||
- match: |
||||
selector: '{source="test"}' |
||||
stages: |
||||
- logfmt: |
||||
mapping: |
||||
app: |
||||
- non_indexed_labels: |
||||
app: |
||||
` |
||||
|
||||
var pipelineStagesNonIndexedLabelsFromLogfmt = ` |
||||
pipeline_stages: |
||||
- logfmt: |
||||
mapping: |
||||
app: |
||||
- non_indexed_labels: |
||||
app: |
||||
` |
||||
|
||||
var pipelineStagesNonIndexedLabelsFromJSON = ` |
||||
pipeline_stages: |
||||
- json: |
||||
expressions: |
||||
app: |
||||
- non_indexed_labels: |
||||
app: |
||||
` |
||||
|
||||
var pipelineStagesNonIndexedLabelsWithRegexParser = ` |
||||
pipeline_stages: |
||||
- regex: |
||||
expression: "^(?s)(?P<time>\\S+?) (?P<stream>stdout|stderr) (?P<flags>\\S+?) (?P<content>.*)$" |
||||
- non_indexed_labels: |
||||
stream: |
||||
` |
||||
|
||||
var pipelineStagesNonIndexedLabelsFromJSONWithTemplate = ` |
||||
pipeline_stages: |
||||
- json: |
||||
expressions: |
||||
app: |
||||
- template: |
||||
source: app |
||||
template: '{{ ToUpper .Value }}' |
||||
- non_indexed_labels: |
||||
app: |
||||
` |
||||
|
||||
var pipelineStagesNonIndexedAndRegularLabelsFromJSON = ` |
||||
pipeline_stages: |
||||
- json: |
||||
expressions: |
||||
app: |
||||
component: |
||||
- non_indexed_labels: |
||||
app: |
||||
- labels: |
||||
component:
|
||||
` |
||||
|
||||
func Test_NonIndexedLabelsStage(t *testing.T) { |
||||
tests := map[string]struct { |
||||
pipelineStagesYaml string |
||||
logLine string |
||||
streamLabels model.LabelSet |
||||
expectedNonIndexedLabels push.LabelsAdapter |
||||
expectedLabels model.LabelSet |
||||
}{ |
||||
"expected non-indexed labels to be extracted with logfmt parser and to be added to entry": { |
||||
pipelineStagesYaml: pipelineStagesNonIndexedLabelsFromLogfmt, |
||||
logLine: "app=loki component=ingester", |
||||
expectedNonIndexedLabels: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}}, |
||||
}, |
||||
"expected non-indexed labels to be extracted with json parser and to be added to entry": { |
||||
pipelineStagesYaml: pipelineStagesNonIndexedLabelsFromJSON, |
||||
logLine: `{"app":"loki" ,"component":"ingester"}`, |
||||
expectedNonIndexedLabels: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}}, |
||||
}, |
||||
"expected non-indexed labels to be extracted with regexp parser and to be added to entry": { |
||||
pipelineStagesYaml: pipelineStagesNonIndexedLabelsWithRegexParser, |
||||
logLine: `2019-01-01T01:00:00.000000001Z stderr P i'm a log message!`, |
||||
expectedNonIndexedLabels: push.LabelsAdapter{push.LabelAdapter{Name: "stream", Value: "stderr"}}, |
||||
}, |
||||
"expected non-indexed labels to be extracted with json parser and to be added to entry after rendering the template": { |
||||
pipelineStagesYaml: pipelineStagesNonIndexedLabelsFromJSONWithTemplate, |
||||
logLine: `{"app":"loki" ,"component":"ingester"}`, |
||||
expectedNonIndexedLabels: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "LOKI"}}, |
||||
}, |
||||
"expected non-indexed and regular labels to be extracted with json parser and to be added to entry": { |
||||
pipelineStagesYaml: pipelineStagesNonIndexedAndRegularLabelsFromJSON, |
||||
logLine: `{"app":"loki" ,"component":"ingester"}`, |
||||
expectedNonIndexedLabels: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}}, |
||||
expectedLabels: model.LabelSet{model.LabelName("component"): model.LabelValue("ingester")}, |
||||
}, |
||||
"expected non-indexed to be extracted using match stage": { |
||||
pipelineStagesYaml: pipelineStagesNonIndexedLabelsUsingMatch, |
||||
logLine: `app=loki component=ingester`, |
||||
expectedNonIndexedLabels: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}}, |
||||
expectedLabels: model.LabelSet{model.LabelName("source"): model.LabelValue("test")}, |
||||
streamLabels: model.LabelSet{model.LabelName("source"): model.LabelValue("test")}, |
||||
}, |
||||
} |
||||
for name, test := range tests { |
||||
t.Run(name, func(t *testing.T) { |
||||
pl, err := NewPipeline(util_log.Logger, loadConfig(test.pipelineStagesYaml), nil, prometheus.DefaultRegisterer) |
||||
require.NoError(t, err) |
||||
result := processEntries(pl, newEntry(nil, test.streamLabels, test.logLine, time.Now()))[0] |
||||
require.Equal(t, test.expectedNonIndexedLabels, result.NonIndexedLabels) |
||||
if test.expectedLabels != nil { |
||||
require.Equal(t, test.expectedLabels, result.Labels) |
||||
} else { |
||||
require.Empty(t, result.Labels) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
@ -0,0 +1,146 @@ |
||||
package stages |
||||
|
||||
import ( |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/prometheus/client_golang/prometheus" |
||||
"github.com/prometheus/common/model" |
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/grafana/loki/pkg/push" |
||||
util_log "github.com/grafana/loki/pkg/util/log" |
||||
) |
||||
|
||||
var pipelineStagesStructuredMetadataUsingMatch = ` |
||||
pipeline_stages: |
||||
- match: |
||||
selector: '{source="test"}' |
||||
stages: |
||||
- logfmt: |
||||
mapping: |
||||
app: |
||||
- structured_metadata: |
||||
app: |
||||
` |
||||
|
||||
var pipelineStagesStructuredMetadataFromLogfmt = ` |
||||
pipeline_stages: |
||||
- logfmt: |
||||
mapping: |
||||
app: |
||||
- structured_metadata: |
||||
app: |
||||
` |
||||
|
||||
var pipelineStagesStructuredMetadataFromJSON = ` |
||||
pipeline_stages: |
||||
- json: |
||||
expressions: |
||||
app: |
||||
- structured_metadata: |
||||
app: |
||||
` |
||||
|
||||
var pipelineStagesStructuredMetadataWithRegexParser = ` |
||||
pipeline_stages: |
||||
- regex: |
||||
expression: "^(?s)(?P<time>\\S+?) (?P<stream>stdout|stderr) (?P<flags>\\S+?) (?P<content>.*)$" |
||||
- structured_metadata: |
||||
stream: |
||||
` |
||||
|
||||
var pipelineStagesStructuredMetadataFromJSONWithTemplate = ` |
||||
pipeline_stages: |
||||
- json: |
||||
expressions: |
||||
app: |
||||
- template: |
||||
source: app |
||||
template: '{{ ToUpper .Value }}' |
||||
- structured_metadata: |
||||
app: |
||||
` |
||||
|
||||
var pipelineStagesStructuredMetadataAndRegularLabelsFromJSON = ` |
||||
pipeline_stages: |
||||
- json: |
||||
expressions: |
||||
app: |
||||
component: |
||||
- structured_metadata: |
||||
app: |
||||
- labels: |
||||
component:
|
||||
` |
||||
|
||||
var deprecatedPipelineStagesStructuredMetadataFromJSON = ` |
||||
pipeline_stages: |
||||
- json: |
||||
expressions: |
||||
app: |
||||
- non_indexed_labels: |
||||
app: |
||||
` |
||||
|
||||
func Test_StructuredMetadataStage(t *testing.T) { |
||||
tests := map[string]struct { |
||||
pipelineStagesYaml string |
||||
logLine string |
||||
streamLabels model.LabelSet |
||||
expectedStructuredMetadata push.LabelsAdapter |
||||
expectedLabels model.LabelSet |
||||
}{ |
||||
"expected structured metadata to be extracted with logfmt parser and to be added to entry": { |
||||
pipelineStagesYaml: pipelineStagesStructuredMetadataFromLogfmt, |
||||
logLine: "app=loki component=ingester", |
||||
expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}}, |
||||
}, |
||||
"expected structured metadata to be extracted with json parser and to be added to entry": { |
||||
pipelineStagesYaml: pipelineStagesStructuredMetadataFromJSON, |
||||
logLine: `{"app":"loki" ,"component":"ingester"}`, |
||||
expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}}, |
||||
}, |
||||
"expected structured metadata to be extracted with json parser and to be added to entry even if deprecated stage name is used": { |
||||
pipelineStagesYaml: deprecatedPipelineStagesStructuredMetadataFromJSON, |
||||
logLine: `{"app":"loki" ,"component":"ingester"}`, |
||||
expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}}, |
||||
}, |
||||
"expected structured metadata to be extracted with regexp parser and to be added to entry": { |
||||
pipelineStagesYaml: pipelineStagesStructuredMetadataWithRegexParser, |
||||
logLine: `2019-01-01T01:00:00.000000001Z stderr P i'm a log message!`, |
||||
expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "stream", Value: "stderr"}}, |
||||
}, |
||||
"expected structured metadata to be extracted with json parser and to be added to entry after rendering the template": { |
||||
pipelineStagesYaml: pipelineStagesStructuredMetadataFromJSONWithTemplate, |
||||
logLine: `{"app":"loki" ,"component":"ingester"}`, |
||||
expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "LOKI"}}, |
||||
}, |
||||
"expected structured metadata and regular labels to be extracted with json parser and to be added to entry": { |
||||
pipelineStagesYaml: pipelineStagesStructuredMetadataAndRegularLabelsFromJSON, |
||||
logLine: `{"app":"loki" ,"component":"ingester"}`, |
||||
expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}}, |
||||
expectedLabels: model.LabelSet{model.LabelName("component"): model.LabelValue("ingester")}, |
||||
}, |
||||
"expected structured metadata to be extracted using match stage": { |
||||
pipelineStagesYaml: pipelineStagesStructuredMetadataUsingMatch, |
||||
logLine: `app=loki component=ingester`, |
||||
expectedStructuredMetadata: push.LabelsAdapter{push.LabelAdapter{Name: "app", Value: "loki"}}, |
||||
expectedLabels: model.LabelSet{model.LabelName("source"): model.LabelValue("test")}, |
||||
streamLabels: model.LabelSet{model.LabelName("source"): model.LabelValue("test")}, |
||||
}, |
||||
} |
||||
for name, test := range tests { |
||||
t.Run(name, func(t *testing.T) { |
||||
pl, err := NewPipeline(util_log.Logger, loadConfig(test.pipelineStagesYaml), nil, prometheus.DefaultRegisterer) |
||||
require.NoError(t, err) |
||||
result := processEntries(pl, newEntry(nil, test.streamLabels, test.logLine, time.Now()))[0] |
||||
require.Equal(t, test.expectedStructuredMetadata, result.StructuredMetadata) |
||||
if test.expectedLabels != nil { |
||||
require.Equal(t, test.expectedLabels, result.Labels) |
||||
} else { |
||||
require.Empty(t, result.Labels) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
Loading…
Reference in new issue