Merge pull request #48 from anyscale/brent/kms-support

feat: Add KMS support for EFS and S3 in Root module
anyscale · Mar 19, 2024 · 3a80b95 · 3a80b95
2 parents bedec02 + c6de585
commit 3a80b95
Show file tree

Hide file tree

Showing 39 changed files with 694 additions and 240 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 default_stages: [commit]
 repos:
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.83.3
+    rev: v1.88.2
     hooks:
       - id: terraform_fmt
       - id: terraform_validate
@@ -12,7 +12,7 @@ repos:
       - id: terraform_tflint
         args:
           - --args=--config=__GIT_WORKING_DIR__/.tflint.hcl
-      - id: terraform_tfsec
+      - id: terraform_trivy
       - id: terraform_checkov
         args:
           - --args=--quiet
@@ -24,7 +24,7 @@ repos:
       #   args:
       #     - --args=provider aws -v "~> 5.16.0"
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer

diff --git a/.tflint.hcl b/.tflint.hcl
@@ -1,6 +1,6 @@
 plugin "aws" {
   enabled = true
-  version = "0.27.0"
+  version = "0.30.0"
   source  = "github.com/terraform-linters/tflint-ruleset-aws"
 }
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,17 @@
+## 0.20.0 (Released)
+FEATURES:
+- Proper KMS support for EFS and S3 buckets
+
+BUG FIXES:
+- VPC Subnet name changes to remove deprecated double lookup call.
+
+BREAKING CHANGES:
+
+NOTES:
+- README updates
+- Add functional-verify to example outputs
+- Upgrade pre-commit from tfsec (deprecated) to trivy
+
 ## 0.19.4 (Released)
 FEATURES:
 

diff --git a/README.md b/README.md
diff --git a/examples/README.md b/examples/README.md
@@ -1,21 +1,56 @@
 # Examples for creating Anyscale related resources
 
-These examples are for learning purposes.
+These examples are for learning purposes and should be modified for your particular production use cases.
 
-### anyscale-v1-all-defaults
-This builds foundational Anyscale Cloud Resources with the AWS Cloud Provider. It creates the resources for a v1 stack including:
+### anyscale-v2-commonname
+This builds the foundational Anyscale Cloud Resources with a Common Name including:
   - IAM Roles
   - S3 Bucket
   - VPC with publicly routed subnets (no internal)
   - VPC Security Groups allowing public from Anyscale
   - EFS
 
-### anysale-v1-existing-vpc
-This builds foundational Anyscale Cloud Resources but expects an existing VPC. It creates the resources for a v1 stack including:
+### anyscale-v2-existing-s3
+This builds the resources required to run Anyscale, but requires an Existing S3 bucket:
+  - IAM Roles
+  - VPC with publicly routed subnets
+  - VPC Security Groups
+  - S3 Policy on an existing S3 Bucket
+
+### anysale-v2-existing-vpc
+This builds foundational Anyscale Cloud Resources but expects an existing VPC. It creates:
   - IAM Roles
   - S3 Bucket
   - VPC Security Groups allowing public from Anyscale
   - EFS
 
+### anyscale-v2-kitchensink
+This uses as many non-contradicting variables as possible to create the resources needed for an Anyscale Cloud. It creates:
+  - IAM Roles
+  - S3 Bucket
+  - VPC
+  - VPC Security Groups
+  - EFS
+  - MemoryDB for HA Head nodes
+
+### anyscale-v2-kms
+This example creates and uses a KMS key to encrypt both S3 and EFS. It creates:
+  - KMS Key
+  - KMS Key Policy
+  - IAM Roles
+  - S3 Bucket encrypted with KMS
+  - VPC
+  - VPC Security Groups
+  - EFS encrypted with KMS
+
+### anyscale-v2-privatesubnets
+This example creates a private networking solution for Anyscale. It requires a VPN or some other connectivity to manage end user traffic to the Anyscale Clusters. It creates:
+  - IAM Roles
+  - S3 Bucket
+  - VPC with a Public Internet Gateway, Private Subnets with NAT Gateways
+  - VPC Security Groups allowing only internal traffic inbound to the Anyscale Clusters
+  - EFS
+
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/examples/anyscale-v2-commonname/main.tf b/examples/anyscale-v2-commonname/main.tf
@@ -17,7 +17,6 @@ locals {
   )
 }
 
-#tfsec:ignore:aws-iam-no-policy-wildcards
 module "aws_anyscale_v2_common_name" {
   source = "../.." #this should be changed if executing this example outside of this repository
   tags   = local.full_tags

diff --git a/examples/anyscale-v2-commonname/outputs.tf b/examples/anyscale-v2-commonname/outputs.tf
@@ -64,6 +64,7 @@ output "anyscale_register_command" {
     --s3-bucket-id ${module.aws_anyscale_v2_common_name.anyscale_s3_bucket_id} \
     --anyscale-iam-role-id ${module.aws_anyscale_v2_common_name.anyscale_iam_role_arn} \
     --instance-iam-role-id ${module.aws_anyscale_v2_common_name.anyscale_iam_role_cluster_node_arn} \
-    --efs-id ${module.aws_anyscale_v2_common_name.anyscale_efs_id}
+    --efs-id ${module.aws_anyscale_v2_common_name.anyscale_efs_id} \
+    --functional-verify workspace
   EOT
 }
diff --git a/examples/anyscale-v2-existing-s3/outputs.tf b/examples/anyscale-v2-existing-s3/outputs.tf
@@ -64,6 +64,7 @@ output "anyscale_register_command" {
     --s3-bucket-id ${split(":", var.existing_s3_bucket_arn)[5]} \
     --anyscale-iam-role-id ${module.aws_anyscale_v2_existing_s3.anyscale_iam_role_arn} \
     --instance-iam-role-id ${module.aws_anyscale_v2_existing_s3.anyscale_iam_role_cluster_node_arn} \
-    --efs-id ${module.aws_anyscale_v2_existing_s3.anyscale_efs_id}
+    --efs-id ${module.aws_anyscale_v2_existing_s3.anyscale_efs_id} \
+    --functional-verify workspace
   EOT
 }
diff --git a/examples/anyscale-v2-existing-vpc/outputs.tf b/examples/anyscale-v2-existing-vpc/outputs.tf
@@ -42,6 +42,7 @@ output "anyscale_register_command" {
     --s3-bucket-id ${module.aws_anyscale_v2_existing_vpc.anyscale_s3_bucket_id} \
     --anyscale-iam-role-id ${module.aws_anyscale_v2_existing_vpc.anyscale_iam_role_arn} \
     --instance-iam-role-id ${module.aws_anyscale_v2_existing_vpc.anyscale_iam_role_cluster_node_arn} \
-    --efs-id ${module.aws_anyscale_v2_existing_vpc.anyscale_efs_id}
+    --efs-id ${module.aws_anyscale_v2_existing_vpc.anyscale_efs_id} \
+    --functional-verify workspace
   EOT
 }
diff --git a/examples/anyscale-v2-kitchensink/main.tf b/examples/anyscale-v2-kitchensink/main.tf
@@ -10,7 +10,8 @@
 # ---------------------------------------------------------------------------------------------------------------------
 locals {
   full_tags = merge(tomap({
-    anyscale-cloud-id = var.anyscale_cloud_id
+    anyscale-cloud-id = var.anyscale_cloud_id,
+    anyscale-org-id   = var.anyscale_org_id
     }),
     var.tags
   )

diff --git a/examples/anyscale-v2-kitchensink/outputs.tf b/examples/anyscale-v2-kitchensink/outputs.tf
@@ -19,6 +19,7 @@ output "anyscale_register_command" {
     --instance-iam-role-id ${module.aws_anyscale_v2_kitchen_sink.anyscale_iam_role_cluster_node_arn} \
     --efs-id ${module.aws_anyscale_v2_kitchen_sink.anyscale_efs_id} \
     --memorydb-cluster-id ${module.aws_anyscale_v2_kitchen_sink.anyscale_memorydb_cluster_id} \
-    --private-network
+    --private-network \
+    --functional-verify workspace
   EOT
 }
diff --git a/examples/anyscale-v2-kms/README.md b/examples/anyscale-v2-kms/README.md
@@ -0,0 +1,82 @@
+# Anyscale Networking Stack v2 - KMS Encryption for S3 and EFS
+
+This **example** will build the resources necessary to run Anyscale in an AWS account. This example will build a
+[Direct Networking](https://docs.anyscale.com/cloud-deployment/aws/manage-clouds#anyscale-clouds-on-aws) solution.
+
+EFS and S3 will use a common KMS Key, however each of these could be unique.
+
+## To execute
+A general understanding of Terraform and AWS are useful for executing this Terraform. For a high level overview of both,
+please see the [getting started guide](https://github.com/anyscale/terraform-aws-anyscale-cloudfoundation-modules/blob/main/getting-started.md).
+
+## Using with Anyscale CLI
+
+The outputs from this Terraform can be used to build an anyscale cloud with the anyscale CLI. To use:
+1. Make sure you have the latest Anyscale CLI installed `pip install anyscale --upgrade`
+2. The terraform output, `anyscale_register_command` will provide an example Anyscale CLI command that can be used to register an Anyscale Cloud. You will need to change `<CUSTOMER_DEFINED_NAME>` to a cloud name that you would like to use.
+
+example:
+
+```
+anyscale cloud register --provider aws \
+  --name <CUSTOMER_DEFINED_NAME> \
+  --region <VPC_REGION> \
+  --vpc-id <VPC ID FROM Outputs> \
+  --subnet-ids <SUBNET_ID1>,<SUBNET_ID2>,<SUBNET_ID3>,<SUBNET_ID4> \
+  --efs-id <FILE_SYSTEM_ID> \
+  --anyscale-iam-role-id <ANYSCALE_IAM_ROLE_ARN> \
+  --instance-iam-role-id <INSTANCE_IAM_ROLE_ARN> \
+  --security-group-ids <SECURITY_GROUP_ID> \
+  --s3-bucket-id <S3_BUCKET_NAME>
+
+anyscale cloud verify --name <CUSTOMER_DEFINED_NAME>
+anyscale cloud delete --name <CUSTOMER_DEFINED_NAME>
+```
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.41.0 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_aws_anyscale_v2_kms"></a> [aws\_anyscale\_v2\_kms](#module\_aws\_anyscale\_v2\_kms) | ../.. | n/a |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_kms_key.anyscale_v2_kms](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_iam_policy_document.anyscale_v2_kms_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | The AWS region in which all resources will be created.<br>ex:<pre>aws_region = "us-east-2"</pre> | `string` | n/a | yes |
+| <a name="input_customer_ingress_cidr_ranges"></a> [customer\_ingress\_cidr\_ranges](#input\_customer\_ingress\_cidr\_ranges) | The IPv4 CIDR block that is allowed to access the clusters.<br>This provides the ability to lock down the v1 stack to just the public IPs of a corporate network.<br>This is added to the security group and allows port 443 (https) and 22 (ssh) access.<br><br>While not recommended, you can set this to `0.0.0.0/0` to allow access from anywhere.<br>ex:<pre>customer_ingress_cidr_ranges = "52.1.1.23/32,10.1.0.0/16"</pre> | `string` | n/a | yes |
+| <a name="input_key_admin_role_name"></a> [key\_admin\_role\_name](#input\_key\_admin\_role\_name) | (Required)<br>The name of the IAM Role that should have permissions to manage the KMS key created.<br><br>ex:<pre>key_admin_role_name = "anyscale-key-admin-role"</pre> | `string` | n/a | yes |
+| <a name="input_anyscale_cloud_id"></a> [anyscale\_cloud\_id](#input\_anyscale\_cloud\_id) | (Optional) Anyscale Cloud ID.<br>This is used to lock down the cross account access role by Cloud ID. Because the Cloud ID is unique to each<br>customer, this ensures that only the customer can access their own resources. The Cloud ID is not known until the<br>Cloud is created, so this is an optional variable.<br>ex:<pre>anyscale_cloud_id = "cld_abcdefghijklmnop1234567890"</pre> | `string` | `null` | no |
+| <a name="input_anyscale_deploy_env"></a> [anyscale\_deploy\_env](#input\_anyscale\_deploy\_env) | (Optional) Anyscale deployment environment. Used in resource names and tags.<br>ex:<pre>anyscale_deploy_env = "production"</pre> | `string` | `"production"` | no |
+| <a name="input_anyscale_org_id"></a> [anyscale\_org\_id](#input\_anyscale\_org\_id) | (Optional) Anyscale Organization ID.<br><br>This is used to lock down the cross account access role by Organization ID. Because the Organization ID is unique to each<br>customer, this ensures that only the customer can access their own resources.<br><br>ex:<pre>anyscale_org_id = "org_abcdefghijklmn1234567890"</pre> | `string` | `null` | no |
+| <a name="input_common_prefix"></a> [common\_prefix](#input\_common\_prefix) | (Optional)<br>Default for this EXAMPLE is `anyscale-pfx-test-` | `string` | `"anyscale-pfx-test-"` | no |
+| <a name="input_tags"></a> [tags](#input\_tags) | (Optional) A map of tags.<br>These tags will be added to all cloud resources that accept tags.<br>ex:<pre>tags = {<br>  "environment" = "test",<br>  "team" = "anyscale"<br>}</pre> | `map(string)` | <pre>{<br>  "environment": "test",<br>  "test": true<br>}</pre> | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_anyscale_register_command"></a> [anyscale\_register\_command](#output\_anyscale\_register\_command) | Anyscale register command.<br>This output can be used with the Anyscale CLI to register a new Anyscale Cloud.<br>You will need to replace `<CUSTOMER_DEFINED_NAME>` with a name of your choosing before running the Anyscale CLI command. |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/examples/anyscale-v2-kms/data.tf b/examples/anyscale-v2-kms/data.tf
@@ -0,0 +1 @@
+data "aws_caller_identity" "current" {}
diff --git a/examples/anyscale-v2-kms/main.tf b/examples/anyscale-v2-kms/main.tf
@@ -0,0 +1,117 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# Create core Anyscale v2 Stack resources with minimal parameters but a common name
+#   Should be executed in us-east-2
+#   Creates a v2 stack including
+#     - IAM Roles
+#     - S3 Bucket
+#     - VPC with publicly routed subnets (no private subnets)
+#     - VPC Security Groups
+#     - EFS
+# ---------------------------------------------------------------------------------------------------------------------
+locals {
+  full_tags = merge(tomap({
+    anyscale-cloud-id           = var.anyscale_cloud_id,
+    anyscale-deploy-environment = var.anyscale_deploy_env
+    }),
+    var.tags
+  )
+}
+
+# Create a KMS Key that will be used to encrypt EFS and the S3 bucket
+#trivy:ignore:avd-aws-0065:This is a test KMS key and does not require rotation
+resource "aws_kms_key" "anyscale_v2_kms" {
+  #checkov:skip=CKV_AWS_7:Rotation not required for this example
+  description             = "Anyscale Terraform Example v2 KMS Key"
+  deletion_window_in_days = 7
+  policy                  = data.aws_iam_policy_document.anyscale_v2_kms_policy.json
+}
+
+# Create a policy that will be attached to the KMS Key that allows the Anyscale IAM Roles to use the KMS Key
+data "aws_iam_policy_document" "anyscale_v2_kms_policy" {
+  #checkov:skip=CKV_AWS_111:Constraints are handled outside of this example policy
+  #checkov:skip=CKV_AWS_356:Allowed wildcards
+  #checkov:skip=CKV_AWS_109:Constraints are handled outside of this example policy
+  # Default policy - account wide access to all key operations
+  statement {
+    sid       = "Default"
+    actions   = ["kms:*"]
+    resources = ["*"]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+  }
+
+  # Allow the Key Administrators to manage the KMS Key
+  statement {
+    sid = "AllowKeyAdministrators"
+    actions = [
+      "kms:Create*",
+      "kms:Describe*",
+      "kms:Enable*",
+      "kms:List*",
+      "kms:Put*",
+      "kms:Update*",
+      "kms:Revoke*",
+      "kms:Disable*",
+      "kms:Get*",
+      "kms:Delete*",
+      "kms:TagResource",
+      "kms:UntagResource",
+      "kms:ScheduleKeyDeletion",
+      "kms:CancelKeyDeletion"
+    ]
+    resources = ["*"]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.key_admin_role_name}"]
+    }
+  }
+
+  # Allow the Anyscale IAM Roles to use the KMS Key
+  statement {
+    sid       = "AllowAnyscaleRoles"
+    actions   = ["kms:Decrypt", "kms:Encrypt", "kms:ReEncrypt*", "kms:GenerateDataKey*", "kms:DescribeKey"]
+    resources = ["*"]
+    principals {
+      type = "AWS"
+      identifiers = [
+        module.aws_anyscale_v2_kms.anyscale_iam_role_arn,
+        module.aws_anyscale_v2_kms.anyscale_iam_role_cluster_node_arn
+      ]
+    }
+  }
+}
+
+
+module "aws_anyscale_v2_kms" {
+  source = "../.." #this should be changed if executing this example outside of this repository
+  tags   = local.full_tags
+
+  anyscale_deploy_env = var.anyscale_deploy_env
+  anyscale_cloud_id   = var.anyscale_cloud_id
+  anyscale_org_id     = var.anyscale_org_id
+
+  create_cluster_node_cloudwatch_policy = true
+
+  # VPC Related
+  anyscale_vpc_cidr_block     = "172.24.0.0/16"
+  anyscale_vpc_public_subnets = ["172.24.21.0/24", "172.24.22.0/24", "172.24.23.0/24"]
+
+  common_prefix   = var.common_prefix
+  use_common_name = true
+
+  # Security Group Related
+  security_group_ingress_allow_access_from_cidr_range = var.customer_ingress_cidr_ranges
+
+  # S3 Bucket Related
+  anyscale_s3_server_side_encryption = {
+    kms_master_key_id = aws_kms_key.anyscale_v2_kms.key_id
+    sse_algorithm     = "aws:kms"
+  }
+
+  # EFS Related
+  efs_kms_key_id = aws_kms_key.anyscale_v2_kms.arn
+}
diff --git a/examples/anyscale-v2-kms/outputs.tf b/examples/anyscale-v2-kms/outputs.tf
@@ -0,0 +1,23 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# Anyscale v2 Stack resources
+# ---------------------------------------------------------------------------------------------------------------------
+output "anyscale_register_command" {
+  description = <<-EOF
+    Anyscale register command.
+    This output can be used with the Anyscale CLI to register a new Anyscale Cloud.
+    You will need to replace `<CUSTOMER_DEFINED_NAME>` with a name of your choosing before running the Anyscale CLI command.
+  EOF
+  value       = <<-EOT
+    anyscale cloud register --provider aws \
+    --name <CUSTOMER_DEFINED_NAME> \
+    --region ${var.aws_region} \
+    --vpc-id ${module.aws_anyscale_v2_kms.anyscale_vpc_id} \
+    --subnet-ids ${join(",", module.aws_anyscale_v2_kms.anyscale_vpc_public_subnet_ids)} \
+    --security-group-ids ${module.aws_anyscale_v2_kms.anyscale_security_group_id} \
+    --s3-bucket-id ${module.aws_anyscale_v2_kms.anyscale_s3_bucket_id} \
+    --anyscale-iam-role-id ${module.aws_anyscale_v2_kms.anyscale_iam_role_arn} \
+    --instance-iam-role-id ${module.aws_anyscale_v2_kms.anyscale_iam_role_cluster_node_arn} \
+    --efs-id ${module.aws_anyscale_v2_kms.anyscale_efs_id} \
+    --functional-verify workspace
+  EOT
+}