diff --git a/CHANGELOG.md b/CHANGELOG.md index 09cc2ab461..01c5e75866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -407,6 +407,7 @@ to include only the most relevant. #### Lambda-Promtail * [5065](https://github.com/grafana/loki/pull/5065) **AndreZiviani**: lambda-promtail: Add ability to ingest logs from S3 +* [7632](https://github.com/grafana/loki/pull/7632) **changhyuni**: lambda-promtail: Add kinesis data stream to use in terraform #### Fluent Bit * [5223](https://github.com/grafana/loki/pull/5223) **cyriltovena**: fluent-bit: Attempt to unmarshal nested json. diff --git a/docs/sources/clients/lambda-promtail/_index.md b/docs/sources/clients/lambda-promtail/_index.md index 13144dea70..777952cc46 100644 --- a/docs/sources/clients/lambda-promtail/_index.md +++ b/docs/sources/clients/lambda-promtail/_index.md @@ -30,9 +30,15 @@ In an effort to make deployment of lambda-promtail as simple as possible, we've Terraform: ``` +## use cloudwatch log group terraform apply -var "lambda_promtail_image=" -var "write_address=https://logs-prod-us-central1.grafana.net/loki/api/v1/push" -var "password=" -var "username=" -var 'log_group_names=["/aws/lambda/log-group-1", "/aws/lambda/log-group-2"]' -var 'bucket_names=["bucket-a", "bucket-b"]' -var 'batch_size=131072' ``` +``` +## use kinesis data stream +terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=" +``` + The first few lines of `main.tf` define the AWS region to deploy to. Modify as desired, or remove and deploy to ``` diff --git a/tools/lambda-promtail/README.md b/tools/lambda-promtail/README.md index 0dc56d0855..d912fc4f11 100644 --- a/tools/lambda-promtail/README.md +++ b/tools/lambda-promtail/README.md @@ -45,16 +45,21 @@ This is the [Loki Write API](https://grafana.com/docs/loki/latest/api/#post-loki The `lambda-promtail` code picks this value up via an environment variable. -Also, if your deployment requires a [VPC configuration](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config), make sure to edit the `vpc_config` field in `main.tf` manually. Additonal documentation for the Lambda specific Terraform configuration is [here](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config). +Also, if your deployment requires a [VPC configuration](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config), make sure to edit the `vpc_config` field in `main.tf` manually. Additonal documentation for the Lambda specific Terraform configuration is [here](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config). If you want to link kinesis data stream to Lambda as event source, see [here](https://docs.aws.amazon.com/ko_kr/lambda/latest/dg/with-kinesis.html). `lambda-promtail` supports authentication either using HTTP Basic Auth or using Bearer Token. Then use Terraform to deploy: ```bash +## use cloudwatch log group terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'bearer_token=' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=" ``` +```bash +## use kinesis data stream +terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=" + or CloudFormation: ```bash diff --git a/tools/lambda-promtail/main.tf b/tools/lambda-promtail/main.tf index 4cfa37a010..19d36de97a 100644 --- a/tools/lambda-promtail/main.tf +++ b/tools/lambda-promtail/main.tf @@ -61,6 +61,13 @@ resource "aws_iam_role_policy" "logs" { ], "Effect" : "Allow", "Resource": "*", + }, + { + "Action" : [ + "kinesis:*", + ], + "Effect" : "Allow", + "Resource" : "*" } ] }) @@ -149,6 +156,30 @@ resource "aws_lambda_permission" "allow-s3-invoke-lambda-promtail" { source_arn = "arn:aws:s3:::${each.value}" } +resource "aws_kinesis_stream" "kinesis_stream" { + for_each = toset(var.kinesis_stream_name) + name = each.value + shard_count = 1 + retention_period = 48 + + shard_level_metrics = [ + "IncomingBytes", + "OutgoingBytes", + ] + + stream_mode_details { + stream_mode = "PROVISIONED" + } +} + +resource "aws_lambda_event_source_mapping" "kinesis_event_source" { + for_each = toset(var.kinesis_stream_name) + event_source_arn = aws_kinesis_stream.kinesis_stream[each.key].arn + function_name = aws_lambda_function.lambda_promtail.arn + starting_position = "LATEST" + depends_on = [aws_kinesis_stream.kinesis_stream] +} + resource "aws_s3_bucket_notification" "push-to-lambda-promtail" { for_each = toset(var.bucket_names) bucket = each.value diff --git a/tools/lambda-promtail/variables.tf b/tools/lambda-promtail/variables.tf index 0af5785180..4546930c53 100644 --- a/tools/lambda-promtail/variables.tf +++ b/tools/lambda-promtail/variables.tf @@ -83,3 +83,9 @@ variable "kms_key_arn" { description = "kms key arn for encryp env vars." default = "" } + +variable "kinesis_stream_name" { + type = list(string) + description = "Enter kinesis name if kinesis stream is configured as event source in lambda." + default = [] +} \ No newline at end of file