feat(lambda-promtail): build lambda with zip file (#13787)

Co-authored-by: Vladyslav Diachenko <[email protected]>

.gitignore

@@ -34,6 +34,7 @@ dist
 .aws-sam
 .idea
 pkg/loki/wal
+tools/lambda-promtail/main
 
 # Workspaces
 *.work

tools/lambda-promtail/Dockerfile

@@ -15,4 +15,4 @@ RUN go build -o /main -tags lambda.norpc -ldflags="-s -w" lambda-promtail/*.go
 # copy artifacts to a clean image
 FROM public.ecr.aws/lambda/provided:al2
 COPY --from=build-image /main /main
-ENTRYPOINT [ "/main" ]       
+ENTRYPOINT [ "/main" ]

tools/lambda-promtail/README.md

@@ -16,7 +16,7 @@ This is a sample deployment for lambda-promtail - Below is a brief explanation o
 * AWS CLI already configured with Administrator permission
 * [Terraform](https://www.terraform.io/downloads.html)
 
-If you want to modify the lambda-promtail code you will also need: 
+If you want to modify the lambda-promtail code you will also need:
 * [Docker installed](https://www.docker.com/community-edition)
 * [Golang](https://golang.org)
 
@@ -26,17 +26,11 @@ If you want to modify the lambda-promtail code you will also need:
 
 The provided Makefile has targets `build`, and `clean`.
 
-`build` builds the lambda-promtail as a Go static binary. To build the container image properly you should run `docker build . -f tools/lambda-promtail/Dockerfile` from the root of the Loki repository,you can upload this image to your AWS ECR and use via Lambda. `clean` will remove the built Go binary.
+`build` builds the lambda-promtail as a Go static binary. To build the container image properly you should run `docker build . -f tools/lambda-promtail/Dockerfile` from the root of the Loki repository, you can upload this image to your AWS ECR and use via Lambda or if you don't pass a `lambda_promtail_image` value, the terraform will build it from the Loki reposiroty, zip it and use it via Lambda. `clean` will remove the built Go binary.
 
 ### Packaging and deployment
 
-The easiest way to deploy to AWS Lambda using the Golang runtime is to use the `lambda-promtail` image by uploading it to your ECR.
-
-Alternatively you can build the Go binary and upload it to Lambda as a zip:
-```bash
-GOOS=linux CGO_ENABLED=0 go build main.go
-zip function.zip main
-```
+The easiest way to deploy to AWS Lambda using the Golang runtime is to build the `lambda-promtail` go file, zip it and upload it to the lambda function with terraform.
 
 To deploy your application for the first time, first make sure you've set the following value in the Terraform file:
 - `WRITE_ADDRESS`
@@ -47,19 +41,24 @@ The `lambda-promtail` code picks this value up via an environment variable.
 
 Also, if your deployment requires a [VPC configuration](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config), make sure to edit the `vpc_config` field in `main.tf` manually. Additonal documentation for the Lambda specific Terraform configuration is [here](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config). If you want to link kinesis data stream to Lambda as event source, see [here](https://docs.aws.amazon.com/ko_kr/lambda/latest/dg/with-kinesis.html).
 
-`lambda-promtail` supports authentication either using HTTP Basic Auth or using Bearer Token.  
-For development purposes you can set the environment variable SKIP_TLS_VERIFY to `true`, so you can use self signed certificates, but this is not recommended in production. Default is `false`.
+`lambda-promtail` supports authentication either using HTTP Basic Auth or using Bearer Token.
+For development purposes, you can set the environment variable SKIP_TLS_VERIFY to `true`, so you can use self-signed certificates, but this is not recommended in production. Default is `false`.
 
 Then use Terraform to deploy:
 
 ```bash
 ## use cloudwatch log group
-terraform apply -var "<ecr-repo>:<tag>" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'bearer_token=<bearer-token>' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=<value>" -var 'skip_tls_verify="false"'
+terraform apply -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'bearer_token=<bearer-token>' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=<value>" -var 'skip_tls_verify="false"'
+```
+
+```bash
+## use docker image uploaded to ECR
+terraform apply -var "lambda_promtail_image=<ecr-repo>:<tag>" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'bearer_token=<bearer-token>' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=<value>" -var 'skip_tls_verify="false"'
 ```
 
 ```bash
 ## use kinesis data stream
-terraform apply -var "<ecr-repo>:<tag>" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"'  -var 'drop_labels="name1,name2"' -var "tenant_id=<value>" -var 'skip_tls_verify="false"'
+terraform apply -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"'  -var 'drop_labels="name1,name2"' -var "tenant_id=<value>" -var 'skip_tls_verify="false"'
 ```
 
 or CloudFormation:
@@ -68,6 +67,8 @@ or CloudFormation:
 aws cloudformation create-stack --stack-name lambda-promtail-stack --template-body file://template.yaml --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM --region us-east-2 --parameters ParameterKey=WriteAddress,ParameterValue=https://your-loki-url/loki/api/v1/push ParameterKey=Username,ParameterValue=<basic-auth-username> ParameterKey=Password,ParameterValue=<basic-auth-pw> ParameterKey=BearerToken,ParameterValue=<bearer-token> ParameterKey=LambdaPromtailImage,ParameterValue=<ecr-repo>:<tag> ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2" ParameterKey=TenantID,ParameterValue=<value> ParameterKey=SkipTlsVerify,ParameterValue="false"
 ```
 
+**NOTE:** To use CloudFormation, you must build the docker image with `docker build . -f tools/lambda-promtail/Dockerfile` from the root of the Loki repository, upload it to an ECR, and pass it as the `LambdaPromtailImage` parameter to cloudformation.
+
 # Appendix
 
 ## Golang installation

tools/lambda-promtail/main.tf

@@ -148,15 +148,46 @@ resource "aws_cloudwatch_log_group" "this" {
   retention_in_days = 14
 }
 
+locals {
+  binary_path  = "${path.module}/lambda-promtail/bootstrap"
+  archive_path = "${path.module}/lambda-promtail/${var.name}.zip"
+}
+
+resource "null_resource" "function_binary" {
+  count = var.lambda_promtail_image == "" ? 1 : 0
+  triggers = {
+    always_run = timestamp()
+  }
+
+  provisioner "local-exec" {
+    command     = "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 GOFLAGS=-trimpath go build -mod=readonly -ldflags='-s -w' -o bootstrap"
+    working_dir = format("%s/%s", path.module, "lambda-promtail")
+  }
+}
+
+data "archive_file" "lambda" {
+  count      = var.lambda_promtail_image == "" ? 1 : 0
+  depends_on = [null_resource.function_binary[0]]
+
+  type        = "zip"
+  source_file = local.binary_path
+  output_path = local.archive_path
+}
+
 resource "aws_lambda_function" "this" {
-  image_uri     = var.lambda_promtail_image
   function_name = var.name
   role          = aws_iam_role.this.arn
   kms_key_arn   = var.kms_key_arn
 
+  image_uri        = var.lambda_promtail_image == "" ? null : var.lambda_promtail_image
+  filename         = var.lambda_promtail_image == "" ? local.archive_path : null
+  source_code_hash = var.lambda_promtail_image == "" ? data.archive_file.lambda[0].output_base64sha256 : null
+  runtime          = var.lambda_promtail_image == "" ? "provided.al2023" : null
+  handler          = var.lambda_promtail_image == "" ? local.binary_path : null
+
   timeout      = 60
   memory_size  = 128
-  package_type = "Image"
+  package_type = var.lambda_promtail_image == "" ? "Zip" : "Image"
 
   # From the Terraform AWS Lambda docs: If both subnet_ids and security_group_ids are empty then vpc_config is considered to be empty or unset.
   vpc_config {

tools/lambda-promtail/versions.tf

@@ -4,5 +4,11 @@ terraform {
     aws = {
       source = "hashicorp/aws"
     }
+    archive = {
+      source = "hashicorp/archive"
+    }
+    null = {
+      source = "hashicorp/null"
+    }
   }
 }