Set unique timestamp in lambda-promtail. (#9560)

pull/9592/head
Karsten Jeschkies 3 years ago committed by GitHub
parent fd0efe0799
commit c9a3ff53e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      tools/lambda-promtail/lambda-promtail/cloudtrail_test.go
  2. 3
      tools/lambda-promtail/lambda-promtail/cw.go
  3. 2
      tools/lambda-promtail/lambda-promtail/json_stream.go
  4. 5
      tools/lambda-promtail/lambda-promtail/kinesis.go
  5. 3
      tools/lambda-promtail/lambda-promtail/kinesis_test.go
  6. 2
      tools/lambda-promtail/lambda-promtail/main_test.go
  7. 3
      tools/lambda-promtail/lambda-promtail/promtail.go
  8. 5
      tools/lambda-promtail/lambda-promtail/s3.go
  9. 3
      tools/lambda-promtail/lambda-promtail/s3_test.go

@ -13,7 +13,7 @@ func TestParseJson(t *testing.T) {
if err != nil {
t.Error(err)
}
gzipReader,err := gzip.NewReader(file)
gzipReader, err := gzip.NewReader(file)
if err != nil {
t.Error(err)
}

@ -6,8 +6,9 @@ import (
"time"
"github.com/aws/aws-lambda-go/events"
"github.com/grafana/loki/pkg/logproto"
"github.com/prometheus/common/model"
"github.com/grafana/loki/pkg/logproto"
)
func parseCWEvent(ctx context.Context, b *batch, ev *events.CloudwatchLogsEvent) error {

@ -6,7 +6,7 @@ import (
"io"
)
// Stream helps transmit each recordss withing a channel.
// Stream helps transmit each recordss within a channel.
type Stream struct {
records chan Record
}

@ -5,8 +5,9 @@ import (
"time"
"github.com/aws/aws-lambda-go/events"
"github.com/grafana/loki/pkg/logproto"
"github.com/prometheus/common/model"
"github.com/grafana/loki/pkg/logproto"
)
func parseKinesisEvent(ctx context.Context, b batchIf, ev *events.KinesisEvent) error {
@ -15,7 +16,7 @@ func parseKinesisEvent(ctx context.Context, b batchIf, ev *events.KinesisEvent)
}
for _, record := range ev.Records {
timestamp := time.Unix(record.Kinesis.ApproximateArrivalTimestamp.Unix(),0)
timestamp := time.Unix(record.Kinesis.ApproximateArrivalTimestamp.Unix(), 0)
labels := model.LabelSet{
model.LabelName("__aws_log_type"): model.LabelValue("kinesis"),

@ -7,8 +7,9 @@ import (
"testing"
"github.com/aws/aws-lambda-go/events"
"github.com/grafana/loki/pkg/logproto"
"github.com/stretchr/testify/require"
"github.com/grafana/loki/pkg/logproto"
)
type MockBatch struct {

@ -18,7 +18,7 @@ func TestLambdaPromtail_ExtraLabelsValid(t *testing.T) {
}
func TestLambdaPromtail_ExtraLabelsMissingValue(t *testing.T) {
extraLabels, err := parseExtraLabels("A,a,B,b,C,c,D",false)
extraLabels, err := parseExtraLabels("A,a,B,b,C,c,D", false)
require.Nil(t, extraLabels)
require.Errorf(t, err, invalidExtraLabelsError)
}

@ -15,8 +15,9 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/golang/snappy"
"github.com/grafana/dskit/backoff"
"github.com/grafana/loki/pkg/logproto"
"github.com/prometheus/common/model"
"github.com/grafana/loki/pkg/logproto"
)
const (

@ -12,9 +12,10 @@ import (
"github.com/aws/aws-lambda-go/events"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/grafana/loki/pkg/logproto"
"github.com/prometheus/common/model"
"github.com/grafana/loki/pkg/logproto"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
@ -97,7 +98,6 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io.
ls = applyExtraLabels(ls)
timestamp := time.Now()
// extract the timestamp of the nested event and sends the rest as raw json
if labels["type"] == CLOUDTRAIL_LOG_TYPE {
records := make(chan Record)
@ -131,6 +131,7 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io.
}
match := timestampRegex.FindStringSubmatch(log_line)
timestamp := time.Now()
if len(match) > 0 {
timestamp, err = time.Parse(time.RFC3339, match[1])
if err != nil {

@ -8,8 +8,9 @@ import (
"testing"
"github.com/aws/aws-lambda-go/events"
"github.com/grafana/loki/pkg/logproto"
"github.com/stretchr/testify/require"
"github.com/grafana/loki/pkg/logproto"
)
func Test_getLabels(t *testing.T) {

Loading…
Cancel
Save