lambda-promtail: Add option to omit extra labels prefix `__extra_` (#8548)

If you want to add an extra label env, lambda-promtail would add the __extra_ prefix, so that the resulting label is __extra_env. However, if you already have the env variable in other stream and you want to have consistent label naming, you would want to omit the prefix so the resulting label is env.

Closes https://github.com/grafana/loki/issues/8471

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>

tools/lambda-promtail/lambda-promtail/main.go

@@ -51,8 +51,9 @@ func setupArguments() {
 
 	fmt.Println("write address: ", writeAddress.String())
 
+	omitExtraLabelsPrefix := os.Getenv("OMIT_EXTRA_LABELS_PREFIX")
 	extraLabelsRaw = os.Getenv("EXTRA_LABELS")
-	extraLabels, err = parseExtraLabels(extraLabelsRaw)
+	extraLabels, err = parseExtraLabels(extraLabelsRaw, strings.EqualFold(omitExtraLabelsPrefix, "true"))
 	if err != nil {
 		panic(err)
 	}
@@ -100,7 +101,11 @@ func setupArguments() {
 	s3Clients = make(map[string]*s3.Client)
 }
 
-func parseExtraLabels(extraLabelsRaw string) (model.LabelSet, error) {
+func parseExtraLabels(extraLabelsRaw string, omitPrefix bool) (model.LabelSet, error) {
+	prefix := "__extra_"
+	if omitPrefix {
+		prefix = ""
+	}
 	var extractedLabels = model.LabelSet{}
 	extraLabelsSplit := strings.Split(extraLabelsRaw, ",")
 
@@ -112,7 +117,7 @@ func parseExtraLabels(extraLabelsRaw string) (model.LabelSet, error) {
 		return nil, fmt.Errorf(invalidExtraLabelsError)
 	}
 	for i := 0; i < len(extraLabelsSplit); i += 2 {
-		extractedLabels[model.LabelName("__extra_"+extraLabelsSplit[i])] = model.LabelValue(extraLabelsSplit[i+1])
+		extractedLabels[model.LabelName(prefix+extraLabelsSplit[i])] = model.LabelValue(extraLabelsSplit[i+1])
 	}
 	err := extractedLabels.Validate()
 	if err != nil {

tools/lambda-promtail/main.tf

@@ -106,16 +106,17 @@ resource "aws_lambda_function" "lambda_promtail" {
 
   environment {
     variables = {
-      WRITE_ADDRESS   = var.write_address
-      USERNAME        = var.username
-      PASSWORD        = var.password
-      BEARER_TOKEN    = var.bearer_token
-      KEEP_STREAM     = var.keep_stream
-      BATCH_SIZE      = var.batch_size
-      EXTRA_LABELS    = var.extra_labels
-      TENANT_ID       = var.tenant_id
-      SKIP_TLS_VERIFY = var.skip_tls_verify
-      PRINT_LOG_LINE = var.print_log_line
+      WRITE_ADDRESS            = var.write_address
+      USERNAME                 = var.username
+      PASSWORD                 = var.password
+      BEARER_TOKEN             = var.bearer_token
+      KEEP_STREAM              = var.keep_stream
+      BATCH_SIZE               = var.batch_size
+      EXTRA_LABELS             = var.extra_labels
+      OMIT_EXTRA_LABELS_PREFIX = var.omit_extra_labels_prefix
+      TENANT_ID                = var.tenant_id
+      SKIP_TLS_VERIFY          = var.skip_tls_verify
+      PRINT_LOG_LINE           = var.print_log_line
     }
   }
 

tools/lambda-promtail/template.yaml

@@ -39,6 +39,10 @@ Parameters:
     Description: Comma separated list of extra labels, in the format 'name1,value1,name2,value2,...,nameN,valueN' to add to entries forwarded by lambda-promtail.
     Type: String
     Default: ""
+  OmitExtraLabelsPrefix:
+    Description: Whether or not to omit the prefix `__extra_` from extra labels defined in `ExtraLabels`.
+    Type: String
+    Default: "false"
   TenantID:
     Description: Tenant ID to be added when writing logs from lambda-promtail.
     Type: String
@@ -94,6 +98,7 @@ Resources:
           BEARER_TOKEN: !Ref BearerToken
           KEEP_STREAM: !Ref KeepStream
           EXTRA_LABELS: !Ref ExtraLabels
+          OMIT_EXTRA_LABELS_PREFIX: !Ref OmitExtraLabelsPrefix
           TENANT_ID: !Ref TenantID
           SKIP_TLS_VERIFY: !Ref SkipTlsVerify
   LambdaPromtailVersion:

tools/lambda-promtail/variables.tf

@@ -66,6 +66,12 @@ variable "extra_labels" {
   default     = ""
 }
 
+variable "omit_extra_labels_prefix" {
+  type        = string
+  description = "Whether or not to omit the prefix `__extra_` from extra labels defined in the variable `extra_labels`."
+  default     = "false"
+}
+
 variable "batch_size" {
   type        = string
   description = "Determines when to flush the batch of logs (bytes)."