apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix regression in lambda-promtail tool that yields incorrect label names (#8547)

Browse Source 
Commit 4d5678aa17 (diff-c898e72f3b15b46b32f9405b243b7d5b56b68db7bf03541fefd587dc9e49af8eL70-R90) introduced a regression that caused labels to have duplicate `_lb` parts in the label names, as described in https://github.com/grafana/loki/issues/8470

Fixes #8470

Also updated Loki to the latest commit on `main`. Previously the dependency pointed to a Git commit that was out of the tree of this repository.
pull/8548/head
Christian Haudum 3 years ago committed by GitHub
parent 42522d794a
commit ff39654a99
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 364 additions and 1338 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 9
      tools/lambda-promtail/Makefile
  2. 140
      tools/lambda-promtail/go.mod
  3. 1520
      tools/lambda-promtail/go.sum
  4. 6
      tools/lambda-promtail/lambda-promtail/s3.go
  5. 27
      tools/lambda-promtail/lambda-promtail/s3_test.go

9
tools/lambda-promtail/Makefile
Unescape View File

@ -1,7 +1,12 @@
all: build
all: clean test build
GOTEST ?= go test
build:
GOOS=linux CGO_ENABLED=0 go build -o ./main lambda-promtail/*.go
test:
$(GOTEST) ./...
clean:
rm main
rm -f main

140
tools/lambda-promtail/go.mod
Unescape View File

@ -1,113 +1,119 @@
module main
go 1.17
go 1.19
require (
github.com/aws/aws-lambda-go v1.26.0
github.com/aws/aws-sdk-go-v2 v1.11.2
github.com/aws/aws-sdk-go-v2/config v1.11.1
github.com/aws/aws-sdk-go-v2 v1.16.0
github.com/aws/aws-sdk-go-v2/config v1.15.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.22.0
github.com/gogo/protobuf v1.3.2
github.com/golang/snappy v0.0.4
github.com/grafana/dskit v0.0.0-20220105080720-01ce9286d7d5
github.com/grafana/loki v1.6.2-0.20220128102010-431d018ec64f
github.com/prometheus/common v0.32.1
github.com/grafana/dskit v0.0.0-20230201083518-528d8a7d52f2
github.com/grafana/loki v1.6.2-0.20230216091802-4e4359e67c6c
github.com/prometheus/common v0.39.0
github.com/stretchr/testify v1.8.1
)
require (
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
github.com/armon/go-metrics v0.3.9 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/armon/go-metrics v0.4.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.6.5 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.11.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.7 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.12.0 // indirect
github.com/aws/smithy-go v1.9.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 // indirect
github.com/aws/smithy-go v1.11.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/coreos/etcd v3.3.25+incompatible // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
github.com/coreos/go-systemd/v22 v22.4.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dennwc/varint v1.0.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/go-kit/log v0.2.0 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/fatih/color v1.14.1 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/gogo/googleapis v1.4.0 // indirect
github.com/gogo/status v1.1.0 // indirect
github.com/gogo/status v1.1.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grafana/loki/pkg/push v0.0.0-20230127102416-571f88bc5765 // indirect
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/hashicorp/consul/api v1.12.0 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/consul/api v1.18.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-hclog v0.16.2 // indirect
github.com/hashicorp/go-hclog v1.2.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/go-multierror v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/memberlist v0.3.0 // indirect
github.com/hashicorp/serf v0.9.6 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/hashicorp/memberlist v0.5.0 // indirect
github.com/hashicorp/serf v0.10.1 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mattn/go-colorable v0.1.9 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.45 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/miekg/dns v1.1.50 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/prometheus/prometheus v1.8.2-0.20211119115433-692a54649ed7 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/exporter-toolkit v0.8.2 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/prometheus/prometheus v0.41.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/sercand/kuberesolver v2.4.0+incompatible // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/stretchr/testify v1.7.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/weaveworks/common v0.0.0-20211015155308-ebe5bdc2c89e // indirect
github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d // indirect
github.com/weaveworks/promrus v1.2.0 // indirect
go.etcd.io/etcd v3.3.25+incompatible // indirect
go.etcd.io/etcd/api/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/v3 v3.5.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/mod v0.5.1 // indirect
golang.org/x/net v0.0.0-20220105145211-5b0dc2dfae98 // indirect
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
golang.org/x/tools v0.1.8 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
google.golang.org/grpc v1.40.1 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
go.etcd.io/etcd/client/v3 v3.5.4 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/goleak v1.2.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/crypto v0.5.0 // indirect
golang.org/x/exp v0.0.0-20221212164502-fae10dda9338 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/oauth2 v0.4.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/text v0.6.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.4.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
google.golang.org/grpc v1.52.3 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
replace k8s.io/client-go => k8s.io/client-go v0.21.0

1520
tools/lambda-promtail/go.sum
View File

File diff suppressed because it is too large Load Diff

6
tools/lambda-promtail/lambda-promtail/s3.go
Unescape View File

@ -85,9 +85,9 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io.
}
ls := model.LabelSet{
model.LabelName("__aws_log_type"): model.LabelValue(logType),
model.LabelName(fmt.Sprintf("__aws_%s_lb", logType)): model.LabelValue(labels["src"]),
model.LabelName(fmt.Sprintf("__aws_%s_lb_owner", logType)): model.LabelValue(labels["account_id"]),
model.LabelName("__aws_log_type"): model.LabelValue(logType),
model.LabelName(fmt.Sprintf("__aws_%s", logType)): model.LabelValue(labels["src"]),
model.LabelName(fmt.Sprintf("__aws_%s_owner", logType)): model.LabelValue(labels["account_id"]),
}
ls = applyExtraLabels(ls)

27
tools/lambda-promtail/lambda-promtail/s3_test.go
Unescape View File

@ -9,6 +9,7 @@ import (
"github.com/aws/aws-lambda-go/events"
"github.com/grafana/loki/pkg/logproto"
"github.com/stretchr/testify/require"
)
func Test_getLabels(t *testing.T) {
@ -111,9 +112,10 @@ func Test_parseS3Log(t *testing.T) {
batchSize int
}
tests := []struct {
name string
args args
wantErr bool
name string
args args
wantErr bool
expectedStream string
}{
{
name: "vpcflowlogs",
@ -124,10 +126,13 @@ func Test_parseS3Log(t *testing.T) {
streams: map[string]*logproto.Stream{},
},
labels: map[string]string{
"type": FLOW_LOG_TYPE,
"type": FLOW_LOG_TYPE,
"src": "source",
"account_id": "123456789",
},
},
wantErr: false,
expectedStream: `{__aws_log_type="s3_vpc_flow", __aws_s3_vpc_flow="source", __aws_s3_vpc_flow_owner="123456789"}`,
wantErr: false,
},
{
name: "albaccesslogs",
@ -138,10 +143,13 @@ func Test_parseS3Log(t *testing.T) {
streams: map[string]*logproto.Stream{},
},
labels: map[string]string{
"type": LB_LOG_TYPE,
"type": LB_LOG_TYPE,
"src": "source",
"account_id": "123456789",
},
},
wantErr: false,
expectedStream: `{__aws_log_type="s3_lb", __aws_s3_lb="source", __aws_s3_lb_owner="123456789"}`,
wantErr: false,
},
}
for _, tt := range tests {
@ -156,6 +164,11 @@ func Test_parseS3Log(t *testing.T) {
if err := parseS3Log(context.Background(), tt.args.b, tt.args.labels, tt.args.obj); (err != nil) != tt.wantErr {
t.Errorf("parseS3Log() error = %v, wantErr %v", err, tt.wantErr)
}
require.Len(t, tt.args.b.streams, 1)
stream, ok := tt.args.b.streams[tt.expectedStream]
require.True(t, ok, "batch does not contain stream: %s", tt.expectedStream)
require.NotNil(t, stream)
})
}
}

Write Preview
Loading…
Cancel
Save