mirror of https://github.com/grafana/loki
Lambda-promtail: Add support for processing SQS messages, add promtailClient Type, add logger, upgrade dependencies and fix unexpected flushing behaviors (#8231)
Christophe Collot
2 years ago
committed by
GitHub
parent
c6542e61fa
commit
a013e9f342
@ -0,0 +1,15 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"os" |
||||
|
||||
"github.com/go-kit/log" |
||||
"github.com/go-kit/log/level" |
||||
) |
||||
|
||||
func NewLogger(logLevel string) *log.Logger { |
||||
logger := log.NewLogfmtLogger(os.Stderr) |
||||
logger = level.NewFilter(logger, level.Allow(level.ParseDefault(logLevel, level.DebugValue()))) |
||||
logger = log.With(logger, "caller", log.DefaultCaller) |
||||
return &logger |
||||
} |
||||
@ -0,0 +1,53 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"crypto/tls" |
||||
"net/http" |
||||
"net/url" |
||||
"time" |
||||
|
||||
"github.com/go-kit/log" |
||||
"github.com/grafana/dskit/backoff" |
||||
) |
||||
|
||||
type Client interface { |
||||
sendToPromtail(ctx context.Context, b *batch) error |
||||
} |
||||
|
||||
// Implements Client
|
||||
type promtailClient struct { |
||||
config *promtailClientConfig |
||||
http *http.Client |
||||
log *log.Logger |
||||
} |
||||
|
||||
type promtailClientConfig struct { |
||||
backoff *backoff.Config |
||||
http *httpClientConfig |
||||
url *url.URL |
||||
} |
||||
|
||||
type httpClientConfig struct { |
||||
timeout time.Duration |
||||
skipTlsVerify bool |
||||
} |
||||
|
||||
func NewPromtailClient(cfg *promtailClientConfig, log *log.Logger) *promtailClient { |
||||
return &promtailClient{ |
||||
config: cfg, |
||||
http: NewHTTPClient(cfg.http), |
||||
log: log, |
||||
} |
||||
} |
||||
|
||||
func NewHTTPClient(cfg *httpClientConfig) *http.Client { |
||||
transport := http.DefaultTransport |
||||
if cfg.skipTlsVerify { |
||||
transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} |
||||
} |
||||
return &http.Client{ |
||||
Timeout: cfg.timeout, |
||||
Transport: transport, |
||||
} |
||||
} |
||||
@ -0,0 +1,32 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"encoding/json" |
||||
|
||||
"github.com/aws/aws-lambda-go/events" |
||||
) |
||||
|
||||
func processSQSEvent(ctx context.Context, evt *events.SQSEvent) error { |
||||
for _, record := range evt.Records { |
||||
// retrieve nested
|
||||
event, err := stringToRawEvent(record.Body) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
err = handler(ctx, event) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func stringToRawEvent(body string) (map[string]interface{}, error) { |
||||
result := make(map[string]interface{}) |
||||
err := json.Unmarshal([]byte(body), &result) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return result, nil |
||||
} |
||||
@ -0,0 +1,68 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"testing" |
||||
|
||||
"github.com/aws/aws-lambda-go/events" |
||||
) |
||||
|
||||
func TestStringToRawEvent(t *testing.T) { |
||||
tc := &events.SQSEvent{ |
||||
Records: []events.SQSMessage{ |
||||
{ |
||||
AWSRegion: "eu-west-3", |
||||
MessageId: "someID", |
||||
Body: `{ |
||||
"Records": [ |
||||
{ |
||||
"eventVersion": "2.1", |
||||
"eventSource": "aws:s3", |
||||
"awsRegion": "us-east-1", |
||||
"eventTime": "2023-01-18T01:52:46.432Z", |
||||
"eventName": "ObjectCreated:Put", |
||||
"userIdentity": { |
||||
"principalId": "AWS:SOMEID:AWS.INTERNAL.URL" |
||||
}, |
||||
"requestParameters": { |
||||
"sourceIPAddress": "172.15.15.15" |
||||
}, |
||||
"responseElements": { |
||||
"x-amz-request-id": "SOMEID", |
||||
"x-amz-id-2": "SOMEID" |
||||
}, |
||||
"s3": { |
||||
"s3SchemaVersion": "1.0", |
||||
"configurationId": "tf-s3-queue-SOMEID", |
||||
"bucket": { |
||||
"name": "some-bucket-name", |
||||
"ownerIdentity": { |
||||
"principalId": "SOMEID" |
||||
|
||||
}, |
||||
"arn": "arn:aws:s3:::SOME-BUCKET-ARN" |
||||
}, |
||||
"object": { |
||||
"key": "SOME-PREFIX/AWSLogs/ACCOUNT-ID/vpcflowlogs/us-east-1/2023/01/18/end-of-filename.log.gz", |
||||
"size": 1042577, |
||||
"eTag": "SOMEID", |
||||
"versionId": "SOMEID", |
||||
"sequencer": "SOMEID" |
||||
} |
||||
} |
||||
} |
||||
] |
||||
}`, |
||||
}, |
||||
}, |
||||
} |
||||
for _, record := range tc.Records { |
||||
event, err := stringToRawEvent(record.Body) |
||||
if err != nil { |
||||
t.Error(err) |
||||
} |
||||
_, err = checkEventType(event) |
||||
if err != nil { |
||||
t.Error(err) |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue