lambda-promtail: Support AWS Organization CloudTrail (#10147)

**What this PR does / why we need it**:

The S3 object key for Organization CloudTrail events includes the org's
ID at the beginning. Currently, trying to ingest Organization CloudTrail
events with lambda-promtail fails due to the object key not matching the
regex. This PR adds support for both matching and parsing out that org
ID.

**Which issue(s) this PR fixes**:
Fixes #<issue number>

**Special notes for your reviewer**:

- instead of adding a new regex I just amended the existing regex just
to avoid too heavy of a refactor
- I also threw in error logging on the handler because that was
_invaluable_ debugging why lambda-promtail wasn't working in our
environment. I figure others might get some benefit from it.

**Checklist**
- [x] Reviewed the
[`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md)
guide (**required**)
- [ ] Documentation added
- [x] Tests updated
- [ ] `CHANGELOG.md` updated
- [ ] If the change is worth mentioning in the release notes, add
`add-to-release-notes` label
- [ ] Changes that require user attention or interaction to upgrade are
documented in `docs/sources/setup/upgrade/_index.md`
- [ ] For Helm chart changes bump the Helm chart version in
`production/helm/loki/Chart.yaml` and update
`production/helm/loki/CHANGELOG.md` and
`production/helm/loki/README.md`. [Example
PR](d10549e3ec)

tools/lambda-promtail/lambda-promtail/main.go

@@ -186,19 +186,23 @@ func handler(ctx context.Context, ev map[string]interface{}) error {
 
 	switch evt := event.(type) {
 	case *events.S3Event:
-		return processS3Event(ctx, evt, pClient, pClient.log)
+		err = processS3Event(ctx, evt, pClient, pClient.log)
 	case *events.CloudwatchLogsEvent:
-		return processCWEvent(ctx, evt, pClient)
+		err = processCWEvent(ctx, evt, pClient)
 	case *events.KinesisEvent:
-		return processKinesisEvent(ctx, evt, pClient)
+		err = processKinesisEvent(ctx, evt, pClient)
 	case *events.SQSEvent:
-		return processSQSEvent(ctx, evt, handler)
+		err = processSQSEvent(ctx, evt, handler)
 	case *events.SNSEvent:
-		return processSNSEvent(ctx, evt, handler)
+		err = processSNSEvent(ctx, evt, handler)
 	// When setting up S3 Notification on a bucket, a test event is first sent, see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/notification-content-structure.html
 	case *events.S3TestEvent:
 		return nil
 	}
+
+	if err != nil {
+		level.Error(*pClient.log).Log("err", fmt.Errorf("error processing event: %v", err))
+	}
 	return err
 }
 

tools/lambda-promtail/lambda-promtail/s3.go

@@ -62,7 +62,7 @@ var (
 	// example: example-prefix/EMLARXS9EXAMPLE.2019-11-14-20.RT4KCN4SGK9.gz
 	defaultFilenameRegex     = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:elasticloadbalancing|vpcflowlogs)\_\w+-\w+-\d_(?:(?:app|nlb|net)\.*?)?(?P<src>[a-zA-Z0-9\-]+)`)
 	defaultTimestampRegex    = regexp.MustCompile(`\w+ (?P<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z)`)
-	cloudtrailFilenameRegex  = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)\_\w+-\w+-\d_(?:(?:app|nlb|net)\.*?)?.+_(?P<src>[a-zA-Z0-9\-]+)`)
+	cloudtrailFilenameRegex  = regexp.MustCompile(`AWSLogs\/(?P<organization_id>o-[a-z0-9]{10,32})?\/?(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)\_\w+-\w+-\d_(?:(?:app|nlb|net)\.*?)?.+_(?P<src>[a-zA-Z0-9\-]+)`)
 	cloudfrontFilenameRegex  = regexp.MustCompile(`(?P<prefix>.*)\/(?P<src>[A-Z0-9]+)\.(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+)-(.+)`)
 	cloudfrontTimestampRegex = regexp.MustCompile(`(?P<timestamp>\d+-\d+-\d+\s\d+:\d+:\d+)`)
 	parsers                  = map[string]parserConfig{
@@ -204,7 +204,7 @@ func getLabels(record events.S3EventRecord) (map[string]string, error) {
 			}
 			match := p.filenameRegex.FindStringSubmatch(labels["key"])
 			for i, name := range p.filenameRegex.SubexpNames() {
-				if i != 0 && name != "" {
+				if i != 0 && name != "" && match[i] != "" {
 					labels[name] = match[i]
 				}
 			}

tools/lambda-promtail/lambda-promtail/s3_test.go

@@ -155,6 +155,40 @@ func Test_getLabels(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "organization_cloudtrail_logs",
+			args: args{
+				record: events.S3EventRecord{
+					AWSRegion: "us-east-1",
+					S3: events.S3Entity{
+						Bucket: events.S3Bucket{
+							Name: "cloudtrail_logs_test",
+							OwnerIdentity: events.S3UserIdentity{
+								PrincipalID: "test",
+							},
+						},
+						Object: events.S3Object{
+							Key: "my-bucket/AWSLogs/o-test123456/123456789012/CloudTrail/us-east-1/2022/01/24/123456789012_CloudTrail_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz",
+						},
+					},
+				},
+			},
+			want: map[string]string{
+				"account_id":      "123456789012",
+				"bucket":          "cloudtrail_logs_test",
+				"bucket_owner":    "test",
+				"bucket_region":   "us-east-1",
+				"day":             "24",
+				"key":             "my-bucket/AWSLogs/o-test123456/123456789012/CloudTrail/us-east-1/2022/01/24/123456789012_CloudTrail_us-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz",
+				"month":           "01",
+				"organization_id": "o-test123456",
+				"region":          "us-east-1",
+				"src":             "4jhzXFO2Jlvu2b3y",
+				"type":            CLOUDTRAIL_LOG_TYPE,
+				"year":            "2022",
+			},
+			wantErr: false,
+		},
 		{
 			name: "s3_cloudfront",
 			args: args{