From 89985399817dde28fe5b94de990133d904bc3e5b Mon Sep 17 00:00:00 2001 From: Justin Roberson <1909600+sapslaj@users.noreply.github.com> Date: Thu, 3 Aug 2023 20:19:29 -0400 Subject: [PATCH] lambda-promtail: Support AWS Organization CloudTrail (#10147) **What this PR does / why we need it**: The S3 object key for Organization CloudTrail events includes the org's ID at the beginning. Currently, trying to ingest Organization CloudTrail events with lambda-promtail fails due to the object key not matching the regex. This PR adds support for both matching and parsing out that org ID. **Which issue(s) this PR fixes**: Fixes # **Special notes for your reviewer**: - instead of adding a new regex I just amended the existing regex just to avoid too heavy of a refactor - I also threw in error logging on the handler because that was _invaluable_ debugging why lambda-promtail wasn't working in our environment. I figure others might get some benefit from it. **Checklist** - [x] Reviewed the [`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md) guide (**required**) - [ ] Documentation added - [x] Tests updated - [ ] `CHANGELOG.md` updated - [ ] If the change is worth mentioning in the release notes, add `add-to-release-notes` label - [ ] Changes that require user attention or interaction to upgrade are documented in `docs/sources/setup/upgrade/_index.md` - [ ] For Helm chart changes bump the Helm chart version in `production/helm/loki/Chart.yaml` and update `production/helm/loki/CHANGELOG.md` and `production/helm/loki/README.md`. [Example PR](https://github.com/grafana/loki/commit/d10549e3ece02120974929894ee333d07755d213) --- tools/lambda-promtail/lambda-promtail/main.go | 14 +++++--- tools/lambda-promtail/lambda-promtail/s3.go | 4 +-- .../lambda-promtail/s3_test.go | 34 +++++++++++++++++++ 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/tools/lambda-promtail/lambda-promtail/main.go b/tools/lambda-promtail/lambda-promtail/main.go index 2adae30847..e140fb8426 100644 --- a/tools/lambda-promtail/lambda-promtail/main.go +++ b/tools/lambda-promtail/lambda-promtail/main.go @@ -186,19 +186,23 @@ func handler(ctx context.Context, ev map[string]interface{}) error { switch evt := event.(type) { case *events.S3Event: - return processS3Event(ctx, evt, pClient, pClient.log) + err = processS3Event(ctx, evt, pClient, pClient.log) case *events.CloudwatchLogsEvent: - return processCWEvent(ctx, evt, pClient) + err = processCWEvent(ctx, evt, pClient) case *events.KinesisEvent: - return processKinesisEvent(ctx, evt, pClient) + err = processKinesisEvent(ctx, evt, pClient) case *events.SQSEvent: - return processSQSEvent(ctx, evt, handler) + err = processSQSEvent(ctx, evt, handler) case *events.SNSEvent: - return processSNSEvent(ctx, evt, handler) + err = processSNSEvent(ctx, evt, handler) // When setting up S3 Notification on a bucket, a test event is first sent, see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/notification-content-structure.html case *events.S3TestEvent: return nil } + + if err != nil { + level.Error(*pClient.log).Log("err", fmt.Errorf("error processing event: %v", err)) + } return err } diff --git a/tools/lambda-promtail/lambda-promtail/s3.go b/tools/lambda-promtail/lambda-promtail/s3.go index e6f146afc0..73e984d2c5 100644 --- a/tools/lambda-promtail/lambda-promtail/s3.go +++ b/tools/lambda-promtail/lambda-promtail/s3.go @@ -62,7 +62,7 @@ var ( // example: example-prefix/EMLARXS9EXAMPLE.2019-11-14-20.RT4KCN4SGK9.gz defaultFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/\d+\_(?:elasticloadbalancing|vpcflowlogs)\_\w+-\w+-\d_(?:(?:app|nlb|net)\.*?)?(?P[a-zA-Z0-9\-]+)`) defaultTimestampRegex = regexp.MustCompile(`\w+ (?P\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z)`) - cloudtrailFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)\_\w+-\w+-\d_(?:(?:app|nlb|net)\.*?)?.+_(?P[a-zA-Z0-9\-]+)`) + cloudtrailFilenameRegex = regexp.MustCompile(`AWSLogs\/(?Po-[a-z0-9]{10,32})?\/?(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)\_\w+-\w+-\d_(?:(?:app|nlb|net)\.*?)?.+_(?P[a-zA-Z0-9\-]+)`) cloudfrontFilenameRegex = regexp.MustCompile(`(?P.*)\/(?P[A-Z0-9]+)\.(?P\d+)-(?P\d+)-(?P\d+)-(.+)`) cloudfrontTimestampRegex = regexp.MustCompile(`(?P\d+-\d+-\d+\s\d+:\d+:\d+)`) parsers = map[string]parserConfig{ @@ -204,7 +204,7 @@ func getLabels(record events.S3EventRecord) (map[string]string, error) { } match := p.filenameRegex.FindStringSubmatch(labels["key"]) for i, name := range p.filenameRegex.SubexpNames() { - if i != 0 && name != "" { + if i != 0 && name != "" && match[i] != "" { labels[name] = match[i] } } diff --git a/tools/lambda-promtail/lambda-promtail/s3_test.go b/tools/lambda-promtail/lambda-promtail/s3_test.go index 704c0ffbaf..6543099cd9 100644 --- a/tools/lambda-promtail/lambda-promtail/s3_test.go +++ b/tools/lambda-promtail/lambda-promtail/s3_test.go @@ -155,6 +155,40 @@ func Test_getLabels(t *testing.T) { }, wantErr: false, }, + { + name: "organization_cloudtrail_logs", + args: args{ + record: events.S3EventRecord{ + AWSRegion: "us-east-1", + S3: events.S3Entity{ + Bucket: events.S3Bucket{ + Name: "cloudtrail_logs_test", + OwnerIdentity: events.S3UserIdentity{ + PrincipalID: "test", + }, + }, + Object: events.S3Object{ + Key: "my-bucket/AWSLogs/o-test123456/123456789012/CloudTrail/us-east-1/2022/01/24/123456789012_CloudTrail_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", + }, + }, + }, + }, + want: map[string]string{ + "account_id": "123456789012", + "bucket": "cloudtrail_logs_test", + "bucket_owner": "test", + "bucket_region": "us-east-1", + "day": "24", + "key": "my-bucket/AWSLogs/o-test123456/123456789012/CloudTrail/us-east-1/2022/01/24/123456789012_CloudTrail_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz", + "month": "01", + "organization_id": "o-test123456", + "region": "us-east-1", + "src": "4jhzXFO2Jlvu2b3y", + "type": CLOUDTRAIL_LOG_TYPE, + "year": "2022", + }, + wantErr: false, + }, { name: "s3_cloudfront", args: args{