Merge branch 'release/4.9.0'

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+## 4.9.0 - 2019-11-14
+- Make use of on-demand instances in docker-machine #158 @skorfmann
+- Allow log retention configuration #157 @geota
+- Add option to encrypt logs via KMS #156 @npalm @hendrixroa 
+
 ## 4.8.0 - 2019-11-01
 - Upgraded the runners (docker-machine) to ubuntu 18.04. You can stay on 16.04 by setting: `runner_ami_filter = ["ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*"]`
 - Upgraded GitLab runner to 12.4.1
@@ -252,7 +257,8 @@ Module is available as Terraform 0.11 module, pin module to version 3.x. Please
 - Update default AMI's to The latest Amazon Linux AMI 2017.09.1 - released on 2018-01-17.
 - Minor updates in the example
 
-[Unreleased]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.6=8.0...HEAD
+[Unreleased]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.9.0...HEAD
+[4.9.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.8.0...4.9.0
 [4.8.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.7.0...4.8.0
 [4.7.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.6.0...4.7.0
 [4.6.0]: https://github.com/npalm/terraform-aws-gitlab-runner/compare/4.5.0...4.6.0

README.md

@@ -237,13 +237,15 @@ terraform destroy
 | cache\_bucket\_versioning | Boolean used to enable versioning on the cache bucket, false by default. | bool | `"false"` | no |
 | cache\_expiration\_days | Number of days before cache objects expires. | number | `"1"` | no |
 | cache\_shared | Enables cache sharing between runners, false by default. | bool | `"false"` | no |
+| cloudwatch\_logging\_retention\_in\_days | Retention for cloudwatch logs. Defaults to unlimited | number | `"0"` | no |
 | docker\_machine\_instance\_type | Instance type used for the instances hosting docker-machine. | string | `"m5a.large"` | no |
 | docker\_machine\_options | List of additional options for the docker machine config. Each element of this list must be a key=value pair. E.g. '["amazonec2-zone=a"]' | list(string) | `<list>` | no |
 | docker\_machine\_role\_json | Docker machine runner instance override policy, expected to be in JSON format. | string | `""` | no |
 | docker\_machine\_spot\_price\_bid | Spot price bid. | string | `"0.06"` | no |
 | docker\_machine\_version | Version of docker-machine. | string | `"0.16.2"` | no |
 | enable\_cloudwatch\_logging | Boolean used to enable or disable the CloudWatch logging. | bool | `"true"` | no |
 | enable\_gitlab\_runner\_ssh\_access | Enables SSH Access to the gitlab runner instance. | bool | `"false"` | no |
+| enable\_kms | Let the module manage a KMS key, logs will be encrypted via KMS. Be-aware of the costs of an custom key. | bool | `"false"` | no |
 | enable\_manage\_gitlab\_token | Boolean to enable the management of the GitLab token in SSM. If `true` the token will be stored in SSM, which means the SSM property is a terraform managed resource. If `false` the Gitlab token will be stored in the SSM by the user-data script during creation of the the instance. However the SSM parameter is not managed by terraform and will remain in SSM after a `terraform destroy`. | bool | `"true"` | no |
 | enable\_runner\_ssm\_access | Add IAM policies to the runner agent instance to connect via the Session Manager. | bool | `"false"` | no |
 | enable\_runner\_user\_data\_trace\_log | Enable bash xtrace for the user data script that creates the EC2 instance for the runner agent. Be aware this could log sensitive data such as you GitLab runner token. | bool | `"false"` | no |
@@ -254,6 +256,8 @@ terraform destroy
 | gitlab\_runner\_version | Version of the GitLab runner. | string | `"12.4.1"` | no |
 | instance\_role\_json | Default runner instance override policy, expected to be in JSON format. | string | `""` | no |
 | instance\_type | Instance type used for the GitLab runner. | string | `"t3.micro"` | no |
+| kms\_deletion\_window\_in\_days | Key rotation window, set to 0 for no rotation. Only used when `enable_kms` is set to `true`. | number | `"7"` | no |
+| kms\_key\_id | KMS key id to encrypted the CloudWatch logs. Ensure CloudWatch has access to the provided KMS key. | string | `""` | no |
 | overrides | This maps provides the possibility to override some defaults. The following attributes are supported: `name_sg` overwrite the `Name` tag for all security groups created by this module. `name_runner_agent_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` ovverrid the `Name` tag spot instances created by the runner agent. | map(string) | `<map>` | no |
 | runner\_ami\_filter | List of maps used to create the AMI filter for the Gitlab runner docker-machine AMI. | map(list(string)) | `<map>` | no |
 | runner\_ami\_owners | The list of owners used to select the AMI of Gitlab runner docker-machine instances. | list(string) | `<list>` | no |
@@ -283,6 +287,7 @@ terraform destroy
 | runners\_privileged | Runners will run in privileged mode, will be used in the runner config.toml | bool | `"true"` | no |
 | runners\_pull\_policy | pull_policy for the runners, will be used in the runner config.toml | string | `"always"` | no |
 | runners\_request\_concurrency | Limit number of concurrent requests for new jobs from GitLab (default 1) | number | `"1"` | no |
+| runners\_request\_spot\_instance | Whether or not to request spot instances via docker-machine | bool | `"true"` | no |
 | runners\_root\_size | Runner instance root size in GB. | number | `"16"` | no |
 | runners\_services\_volumes\_tmpfs | Mount temporary file systems to service containers. Must consist of pairs of strings e.g. "/var/lib/mysql" = "rw,noexec", see example | list | `<list>` | no |
 | runners\_shm\_size | shm_size for the runners, will be used in the runner config.toml | number | `"0"` | no |

_docs/TF_MODULE.md

@@ -13,13 +13,15 @@
 | cache\_bucket\_versioning | Boolean used to enable versioning on the cache bucket, false by default. | bool | `"false"` | no |
 | cache\_expiration\_days | Number of days before cache objects expires. | number | `"1"` | no |
 | cache\_shared | Enables cache sharing between runners, false by default. | bool | `"false"` | no |
+| cloudwatch\_logging\_retention\_in\_days | Retention for cloudwatch logs. Defaults to unlimited | number | `"0"` | no |
 | docker\_machine\_instance\_type | Instance type used for the instances hosting docker-machine. | string | `"m5a.large"` | no |
 | docker\_machine\_options | List of additional options for the docker machine config. Each element of this list must be a key=value pair. E.g. '["amazonec2-zone=a"]' | list(string) | `<list>` | no |
 | docker\_machine\_role\_json | Docker machine runner instance override policy, expected to be in JSON format. | string | `""` | no |
 | docker\_machine\_spot\_price\_bid | Spot price bid. | string | `"0.06"` | no |
 | docker\_machine\_version | Version of docker-machine. | string | `"0.16.2"` | no |
 | enable\_cloudwatch\_logging | Boolean used to enable or disable the CloudWatch logging. | bool | `"true"` | no |
 | enable\_gitlab\_runner\_ssh\_access | Enables SSH Access to the gitlab runner instance. | bool | `"false"` | no |
+| enable\_kms | Let the module manage a KMS key, logs will be encrypted via KMS. Be-aware of the costs of an custom key. | bool | `"false"` | no |
 | enable\_manage\_gitlab\_token | Boolean to enable the management of the GitLab token in SSM. If `true` the token will be stored in SSM, which means the SSM property is a terraform managed resource. If `false` the Gitlab token will be stored in the SSM by the user-data script during creation of the the instance. However the SSM parameter is not managed by terraform and will remain in SSM after a `terraform destroy`. | bool | `"true"` | no |
 | enable\_runner\_ssm\_access | Add IAM policies to the runner agent instance to connect via the Session Manager. | bool | `"false"` | no |
 | enable\_runner\_user\_data\_trace\_log | Enable bash xtrace for the user data script that creates the EC2 instance for the runner agent. Be aware this could log sensitive data such as you GitLab runner token. | bool | `"false"` | no |
@@ -30,6 +32,8 @@
 | gitlab\_runner\_version | Version of the GitLab runner. | string | `"12.4.1"` | no |
 | instance\_role\_json | Default runner instance override policy, expected to be in JSON format. | string | `""` | no |
 | instance\_type | Instance type used for the GitLab runner. | string | `"t3.micro"` | no |
+| kms\_deletion\_window\_in\_days | Key rotation window, set to 0 for no rotation. Only used when `enable_kms` is set to `true`. | number | `"7"` | no |
+| kms\_key\_id | KMS key id to encrypted the CloudWatch logs. Ensure CloudWatch has access to the provided KMS key. | string | `""` | no |
 | overrides | This maps provides the possibility to override some defaults. The following attributes are supported: `name_sg` overwrite the `Name` tag for all security groups created by this module. `name_runner_agent_instance` override the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` ovverrid the `Name` tag spot instances created by the runner agent. | map(string) | `<map>` | no |
 | runner\_ami\_filter | List of maps used to create the AMI filter for the Gitlab runner docker-machine AMI. | map(list(string)) | `<map>` | no |
 | runner\_ami\_owners | The list of owners used to select the AMI of Gitlab runner docker-machine instances. | list(string) | `<list>` | no |
@@ -59,6 +63,7 @@
 | runners\_privileged | Runners will run in privileged mode, will be used in the runner config.toml | bool | `"true"` | no |
 | runners\_pull\_policy | pull_policy for the runners, will be used in the runner config.toml | string | `"always"` | no |
 | runners\_request\_concurrency | Limit number of concurrent requests for new jobs from GitLab (default 1) | number | `"1"` | no |
+| runners\_request\_spot\_instance | Whether or not to request spot instances via docker-machine | bool | `"true"` | no |
 | runners\_root\_size | Runner instance root size in GB. | number | `"16"` | no |
 | runners\_services\_volumes\_tmpfs | Mount temporary file systems to service containers. Must consist of pairs of strings e.g. "/var/lib/mysql" = "rw,noexec", see example | list | `<list>` | no |
 | runners\_shm\_size | shm_size for the runners, will be used in the runner config.toml | number | `"0"` | no |

cache/main.tf

@@ -53,12 +53,10 @@ resource "aws_s3_bucket" "build_cache" {
 }
 
 data "template_file" "docker_machine_cache_policy" {
-  count = var.create_cache_bucket ? 1 : 0
-
   template = file("${path.module}/policies/cache.json")
 
   vars = {
-    s3_cache_arn = aws_s3_bucket.build_cache[0].arn
+    s3_cache_arn = var.create_cache_bucket == false || length(aws_s3_bucket.build_cache) == 0 ? "arn:aws:s3:::fake_bucket_doesnt_exist" : aws_s3_bucket.build_cache[0].arn
   }
 }
 
@@ -69,5 +67,5 @@ resource "aws_iam_policy" "docker_machine_cache" {
   path        = "/"
   description = "Policy for docker machine instance to access cache"
 
-  policy = data.template_file.docker_machine_cache_policy[0].rendered
+  policy = data.template_file.docker_machine_cache_policy.rendered
 }

kms.tf

@@ -0,0 +1,22 @@
+resource "aws_kms_key" "default" {
+  count = var.enable_kms ? 1 : 0
+
+  description             = "GitLab Runner module managed key - ${var.environment}"
+  deletion_window_in_days = var.kms_deletion_window_in_days > 0 ? var.kms_deletion_window_in_days : null
+  enable_key_rotation     = var.kms_deletion_window_in_days > 0 ? true : false
+  tags                    = local.tags
+  policy                  = data.template_file.kms_policy[0].rendered
+}
+
+data "template_file" "kms_policy" {
+  count = var.enable_kms ? 1 : 0
+
+  template = file("${path.module}/policies/kms-policy.json")
+
+  vars = {
+    aws_region = var.aws_region
+    account_id = data.aws_caller_identity.current.account_id
+  }
+}
+
+

logging.tf

@@ -1,3 +1,6 @@
+
+
+
 data "template_file" "instance_profile" {
   count    = var.enable_cloudwatch_logging ? 1 : 0
   template = file("${path.module}/policies/instance-logging-policy.json")
@@ -10,10 +13,16 @@ resource "aws_iam_role_policy" "instance" {
   policy = data.template_file.instance_profile[0].rendered
 }
 
-resource "aws_cloudwatch_log_group" "environment" {
-  count = var.enable_cloudwatch_logging ? 1 : 0
-  name  = var.environment
 
-  tags = local.tags
+locals {
+  provided_kms_key = var.kms_key_id != "" ? var.kms_key_id : ""
+  kms_key          = local.provided_kms_key == "" && var.enable_kms ? aws_kms_key.default[0].arn : local.provided_kms_key
 }
 
+resource "aws_cloudwatch_log_group" "environment" {
+  count             = var.enable_cloudwatch_logging ? 1 : 0
+  name              = var.environment
+  retention_in_days = var.cloudwatch_logging_retention_in_days
+  tags              = local.tags
+  kms_key_id        = local.kms_key
+}

main.tf

@@ -1,3 +1,5 @@
+data "aws_caller_identity" "current" {}
+
 resource "aws_key_pair" "key" {
   count      = var.ssh_key_pair == "" && var.ssh_public_key != "" ? 1 : 0
   key_name   = "${var.environment}-gitlab-runner"
@@ -216,6 +218,7 @@ data "template_file" "runners" {
     runners_iam_instance_profile_name = var.runners_iam_instance_profile_name
     runners_use_private_address_only  = var.runners_use_private_address
     runners_use_private_address       = ! var.runners_use_private_address
+    runners_request_spot_instance     = var.runners_request_spot_instance
     runners_environment_vars          = jsonencode(var.runners_environment_vars)
     runners_pre_build_script          = var.runners_pre_build_script
     runners_post_build_script         = var.runners_post_build_script

policies/kms-policy.json

@@ -0,0 +1,33 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "Enable IAM User Permissions",
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": [
+          "arn:aws:iam::${account_id}:root"
+        ]
+      },
+      "Action": "kms:*",
+      "Resource": "*"
+    },
+    {
+      "Sid": "allowLoggingToCloudWatch",
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "logs.${aws_region}.amazonaws.com"
+      },
+      "Action": [
+        "kms:Encrypt*",
+        "kms:Decrypt*",
+        "kms:ReEncrypt*",
+        "kms:GenerateDataKey*",
+        "kms:Describe*"
+      ],
+      "Resource": [
+        "*"
+      ]
+    }
+  ]
+}

template/runner-config.tpl

@@ -47,7 +47,7 @@ check_interval = 0
       "amazonec2-subnet-id=${runners_subnet_id}",
       "amazonec2-private-address-only=${runners_use_private_address_only}",
       "amazonec2-use-private-address=${runners_use_private_address}",
-      "amazonec2-request-spot-instance=true",
+      "amazonec2-request-spot-instance=${runners_request_spot_instance}",
       "amazonec2-spot-price=${runners_spot_price_bid}",
       "amazonec2-security-group=${runners_security_group_name}",
       "amazonec2-tags=${runners_tags}",

variables.tf

@@ -249,6 +249,12 @@ variable "runners_use_private_address" {
   default     = true
 }
 
+variable "runners_request_spot_instance" {
+  description = "Whether or not to request spot instances via docker-machine"
+  type        = bool
+  default     = true
+}
+
 variable "cache_bucket_prefix" {
   description = "Prefix for s3 cache bucket name."
   type        = string
@@ -303,6 +309,12 @@ variable "enable_cloudwatch_logging" {
   default     = true
 }
 
+variable "cloudwatch_logging_retention_in_days" {
+  description = "Retention for cloudwatch logs. Defaults to unlimited"
+  type        = number
+  default     = 0
+}
+
 variable "tags" {
   description = "Map of tags that will be added to created resources. By default resources will be tagged with name and environment."
   type        = map(string)
@@ -460,3 +472,21 @@ variable "runners_services_volumes_tmpfs" {
   type        = list
   default     = []
 }
+
+variable "kms_key_id" {
+  description = "KMS key id to encrypted the CloudWatch logs. Ensure CloudWatch has access to the provided KMS key."
+  type        = string
+  default     = ""
+}
+
+variable "enable_kms" {
+  description = "Let the module manage a KMS key, logs will be encrypted via KMS. Be-aware of the costs of an custom key."
+  type        = bool
+  default     = false
+}
+
+variable "kms_deletion_window_in_days" {
+  description = "Key rotation window, set to 0 for no rotation. Only used when `enable_kms` is set to `true`."
+  type        = number
+  default     = 7
+}