From 1d7be3fac17ae6ee70f9763dd6e9f2f19e82b7a7 Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Thu, 2 Sep 2021 14:30:11 -0700 Subject: [PATCH 01/14] first stab at getting certmanager up --- terraform/provision/appmesh.tf | 145 +++++++++++++++++++++++++++++++ terraform/provision/ingress.tf | 3 +- terraform/provision/providers.tf | 4 +- terraform/provision/variables.tf | 27 +++--- terraform/provision/vpc.tf | 2 +- 5 files changed, 165 insertions(+), 16 deletions(-) create mode 100644 terraform/provision/appmesh.tf diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf new file mode 100644 index 00000000..7b40654d --- /dev/null +++ b/terraform/provision/appmesh.tf @@ -0,0 +1,145 @@ +# --------------------------------------------------------- +# Provision cert-manager using Helm and a self-signed cert +# --------------------------------------------------------- +resource "kubernetes_namespace" "cert-manager" { + metadata { + name = "cert-manager" + } + + depends_on = [ + aws_eks_fargate_profile.default_namespaces + ] +} + +resource "helm_release" "cert-manager" { + name = "cert-manager" + chart = "cert-manager" + repository = "https://charts.jetstack.io/" + version = "v1.5.3" + namespace = "cert-manager" + cleanup_on_fail = "true" + atomic = "true" + timeout = 600 + + set { + name = "installCRDs" + value = "true" + } + + depends_on = [ + kubernetes_namespace.cert-manager + ] +} + +resource "tls_private_key" "cert-manager" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "tls_self_signed_cert" "cert-manager" { + key_algorithm = "RSA" + private_key_pem = tls_private_key.cert-manager.private_key_pem + + subject { + common_name = "appmesh" + organization = "GSA" + } + + validity_period_hours = 87600 + + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + ] +} + +resource "kubernetes_secret" "cert-manager" { + metadata { + name = "ca-key-pair" + namespace = "default" + } + type = "kubernetes.io/tls" + + data = { + "tls.crt" = tls_self_signed_cert.cert-manager.cert_pem + "tls.key" = tls_private_key.cert-manager.private_key_pem + } +} + +# Until we can use terraform 0.14+, and thus be able to use kubernetes_manifest, +# we need to do this with kubectl. :-( +data "template_file" "ca-issuer" { + template = <<-EOF +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: ca-issuer + namespace: default +spec: + ca: + secretName: ca-key-pair +EOF +} + +resource "null_resource" "ca-issuer" { + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = base64encode(module.eks.kubeconfig) + } + command = <<-EOF + kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) apply -f <(echo '${data.template_file.ca-issuer.rendered}') + EOF + } + depends_on = [ + helm_release.cert-manager, + kubernetes_secret.cert-manager + ] +} + + +# # --------------------------------------------------------- +# # Provision the App Mesh Controller using Helm +# # --------------------------------------------------------- +# resource "kubernetes_namespace" "appmesh-system" { +# metadata { +# name = "appmesh-system" +# } + +# depends_on = [ +# aws_eks_fargate_profile.default_namespaces +# ] +# } + +# resource "helm_release" "appmesh-controller" { +# name = "appmesh-controller" +# chart = "eks/appmesh-controller" +# repository = "https://aws.github.io/eks-charts" +# version = "1.4.1" + +# namespace = "appmesh-system" +# cleanup_on_fail = "true" +# atomic = "true" +# timeout = 600 + +# depends_on = [ +# kubernetes_namespace.appmesh-system +# ] +# } + +# resource "null_resource" "label" { +# provisioner "local-exec" { +# interpreter = ["/bin/bash", "-c"] +# environment = { +# KUBECONFIG = base64encode(module.eks.kubeconfig) +# } +# command = <<-EOF +# kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default mesh=default ; +# kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default appmesh.k8s.aws/sidecarInjectorWebhook=enabled +# EOF +# } +# depends_on = [ +# aws_eks_fargate_profile.default_namespaces +# ] +# } diff --git a/terraform/provision/ingress.tf b/terraform/provision/ingress.tf index 47e40f40..4258c851 100644 --- a/terraform/provision/ingress.tf +++ b/terraform/provision/ingress.tf @@ -1,7 +1,6 @@ locals { base_domain = var.zone - domain = "${local.subdomain}.${local.base_domain}" - subdomain = var.subdomain + domain = "${var.subdomain}.${local.base_domain}" } # We need an OIDC provider for the ALB ingress controller to work diff --git a/terraform/provision/providers.tf b/terraform/provision/providers.tf index b1ed7414..948bb8e3 100644 --- a/terraform/provision/providers.tf +++ b/terraform/provision/providers.tf @@ -14,7 +14,7 @@ provider "kubernetes" { load_config_file = false exec { api_version = "client.authentication.k8s.io/v1alpha1" - args = ["token", "--cluster-id", data.aws_eks_cluster.main.id] + args = ["token", "--cluster-id", local.cluster_name] command = "aws-iam-authenticator" } version = "~> 1.13.3" @@ -28,7 +28,7 @@ provider "helm" { config_path = "./kubeconfig_${module.eks.cluster_id}" exec { api_version = "client.authentication.k8s.io/v1alpha1" - args = ["token", "--cluster-id", data.aws_eks_cluster.main.id] + args = ["token", "--cluster-id", local.cluster_name] command = "aws-iam-authenticator" } } diff --git a/terraform/provision/variables.tf b/terraform/provision/variables.tf index cce743a9..670f572b 100644 --- a/terraform/provision/variables.tf +++ b/terraform/provision/variables.tf @@ -1,23 +1,28 @@ +variable "zone" { + description = "existing dns zone that this is under, like test.gov" + type = string +} + variable "subdomain" { - type = string - default = "" + description = "subdomain that is under the zone, so for foo.test.gov, this would be 'foo'" + type = string + default = "" } variable "instance_name" { - type = string - default = "" + description = "name of the eks cluster" + type = string + default = "" } variable "labels" { - type = map(any) - default = {} -} - -variable "zone" { - type = string + description = "tags that are applied to most AWS resources" + type = map(any) + default = {} } variable "region" { - type = string + description = "AWS region, like us-west-2" + type = string } diff --git a/terraform/provision/vpc.tf b/terraform/provision/vpc.tf index 3fde281d..844c32dd 100644 --- a/terraform/provision/vpc.tf +++ b/terraform/provision/vpc.tf @@ -17,7 +17,7 @@ module "vpc" { # See https://aws.amazon.com/premiumsupport/knowledge-center/eks-vpc-subnet-discovery/ global_tags = merge(var.labels, { "kubernetes.io/cluster/${local.cluster_name}" = "shared", - "domain" = local.domain + "domain" = local.domain }) public_subnet_tags = { "kubernetes.io/role/elb" = 1 From 1cf4e934d304dee0525d55ba800ea2befe94d496 Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Wed, 8 Sep 2021 16:58:21 -0700 Subject: [PATCH 02/14] make cert-manager happen in fargate --- terraform/provision/appmesh.tf | 5 +++++ terraform/provision/eks.tf | 3 +++ 2 files changed, 8 insertions(+) diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index 7b40654d..1efb6a33 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -25,6 +25,11 @@ resource "helm_release" "cert-manager" { name = "installCRDs" value = "true" } + set { + # https://github.com/jetstack/cert-manager/issues/3237 + name = "webhook.securePort" + value = "10260" + } depends_on = [ kubernetes_namespace.cert-manager diff --git a/terraform/provision/eks.tf b/terraform/provision/eks.tf index d6b19eb1..7d8091ec 100644 --- a/terraform/provision/eks.tf +++ b/terraform/provision/eks.tf @@ -60,6 +60,9 @@ resource "aws_eks_fargate_profile" "default_namespaces" { selector { namespace = "kube-system" } + selector { + namespace = "cert-manager" + } # Per AWS docs, you have to patch the coredns deployment to remove the # constraint that it wants to run on ec2, then restart it so it will come up on Fargate. From 4c14bcbb5b48ffc07c5e6ed8d55d8b19b12dd818 Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Thu, 9 Sep 2021 09:33:55 -0700 Subject: [PATCH 03/14] add app mesh controller, make sure dependencies are proper --- terraform/provision/appmesh.tf | 90 +++++++++++++++++----------------- terraform/provision/crds.tf | 2 +- terraform/provision/eks.tf | 3 ++ 3 files changed, 49 insertions(+), 46 deletions(-) diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index 1efb6a33..01b2536f 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -76,7 +76,7 @@ resource "kubernetes_secret" "cert-manager" { # we need to do this with kubectl. :-( data "template_file" "ca-issuer" { template = <<-EOF -apiVersion: cert-manager.io/v1alpha2 +apiVersion: cert-manager.io/v1 kind: Issuer metadata: name: ca-issuer @@ -104,47 +104,47 @@ resource "null_resource" "ca-issuer" { } -# # --------------------------------------------------------- -# # Provision the App Mesh Controller using Helm -# # --------------------------------------------------------- -# resource "kubernetes_namespace" "appmesh-system" { -# metadata { -# name = "appmesh-system" -# } - -# depends_on = [ -# aws_eks_fargate_profile.default_namespaces -# ] -# } - -# resource "helm_release" "appmesh-controller" { -# name = "appmesh-controller" -# chart = "eks/appmesh-controller" -# repository = "https://aws.github.io/eks-charts" -# version = "1.4.1" - -# namespace = "appmesh-system" -# cleanup_on_fail = "true" -# atomic = "true" -# timeout = 600 - -# depends_on = [ -# kubernetes_namespace.appmesh-system -# ] -# } - -# resource "null_resource" "label" { -# provisioner "local-exec" { -# interpreter = ["/bin/bash", "-c"] -# environment = { -# KUBECONFIG = base64encode(module.eks.kubeconfig) -# } -# command = <<-EOF -# kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default mesh=default ; -# kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default appmesh.k8s.aws/sidecarInjectorWebhook=enabled -# EOF -# } -# depends_on = [ -# aws_eks_fargate_profile.default_namespaces -# ] -# } +# --------------------------------------------------------- +# Provision the App Mesh Controller using Helm +# --------------------------------------------------------- +resource "kubernetes_namespace" "appmesh-system" { + metadata { + name = "appmesh-system" + } + + depends_on = [ + null_resource.ca-issuer + ] +} + +resource "helm_release" "appmesh-controller" { + name = "appmesh-controller" + chart = "appmesh-controller" + repository = "https://aws.github.io/eks-charts" + version = "1.4.1" + + namespace = "appmesh-system" + cleanup_on_fail = "true" + atomic = "true" + timeout = 600 + + depends_on = [ + kubernetes_namespace.appmesh-system + ] +} + +resource "null_resource" "appmesh-label" { + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = base64encode(module.eks.kubeconfig) + } + command = <<-EOF + kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default mesh=default ; + kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default appmesh.k8s.aws/sidecarInjectorWebhook=enabled + EOF + } + depends_on = [ + helm_release.appmesh-controller + ] +} diff --git a/terraform/provision/crds.tf b/terraform/provision/crds.tf index eaeec9f1..b2ad2f88 100644 --- a/terraform/provision/crds.tf +++ b/terraform/provision/crds.tf @@ -11,7 +11,7 @@ resource "helm_release" "zookeeper-operator" { atomic = "true" depends_on = [ module.vpc, - aws_eks_fargate_profile.default_namespaces, + null_resource.appmesh-label, ] } diff --git a/terraform/provision/eks.tf b/terraform/provision/eks.tf index 7d8091ec..9949bd47 100644 --- a/terraform/provision/eks.tf +++ b/terraform/provision/eks.tf @@ -63,6 +63,9 @@ resource "aws_eks_fargate_profile" "default_namespaces" { selector { namespace = "cert-manager" } + selector { + namespace = "appmesh-system" + } # Per AWS docs, you have to patch the coredns deployment to remove the # constraint that it wants to run on ec2, then restart it so it will come up on Fargate. From f95ebceb07469fee112fc02ba7532c39fba4d8b1 Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Thu, 9 Sep 2021 09:53:47 -0700 Subject: [PATCH 04/14] may still need to set IRSA up, but this gets it running at least --- terraform/provision/appmesh.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index 01b2536f..a6309678 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -128,6 +128,15 @@ resource "helm_release" "appmesh-controller" { atomic = "true" timeout = 600 + set { + name = "region" + value = data.aws_region.current.name + } + set { + name = "accountId" + value = data.aws_caller_identity.current.account_id + } + depends_on = [ kubernetes_namespace.appmesh-system ] From ecb8b17e3e0b7e14f3caa3944046e57afcf1467b Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Thu, 9 Sep 2021 11:26:19 -0700 Subject: [PATCH 05/14] add the default appmesh --- terraform/provision/appmesh.tf | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index a6309678..97382fea 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -157,3 +157,33 @@ resource "null_resource" "appmesh-label" { helm_release.appmesh-controller ] } + +# Until we can use terraform 0.14+, and thus be able to use kubernetes_manifest, +# we need to do this with kubectl. :-( +data "template_file" "appmesh-default" { + template = <<-EOF +apiVersion: appmesh.k8s.aws/v1beta2 +kind: Mesh +metadata: + name: default +spec: + namespaceSelector: + matchLabels: + mesh: default +EOF +} + +resource "null_resource" "appmesh-default" { + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = base64encode(module.eks.kubeconfig) + } + command = <<-EOF + kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) apply -f <(echo '${data.template_file.appmesh-default.rendered}') + EOF + } + depends_on = [ + null_resource.appmesh-label + ] +} From 75345e38486f8f1a5c2cc322e535320691f02c01 Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Thu, 9 Sep 2021 14:55:28 -0700 Subject: [PATCH 06/14] needed to add IRSA after all --- terraform/provision/appmesh.tf | 141 +++++++++++++++++++++++++++++++-- 1 file changed, 133 insertions(+), 8 deletions(-) diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index 97382fea..818133c9 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -117,6 +117,132 @@ resource "kubernetes_namespace" "appmesh-system" { ] } +# This role is assigned with IRSA to the appmesh controller +resource "aws_iam_role" "appmesh-controller" { + name = "appmesh-controller-${local.cluster_name}" + tags = var.labels + assume_role_policy = < Date: Thu, 9 Sep 2021 15:08:09 -0700 Subject: [PATCH 07/14] docs are wrong somehow about the label --- terraform/provision/appmesh.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index 818133c9..2ef4a688 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -274,7 +274,7 @@ resource "null_resource" "appmesh-label" { KUBECONFIG = base64encode(module.eks.kubeconfig) } command = <<-EOF - kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default mesh=default ; + kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default meshes.appmesh.k8s.aws=default ; kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default appmesh.k8s.aws/sidecarInjectorWebhook=enabled EOF } From 06e73c7d14c37bc4e27e684a09d6a877aec9db75 Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Thu, 9 Sep 2021 15:27:36 -0700 Subject: [PATCH 08/14] added a comment --- terraform/provision/appmesh.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index 2ef4a688..d613693a 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -298,6 +298,7 @@ spec: EOF } +# This is actually what causes the mesh-controller to start up an app mesh. resource "null_resource" "appmesh-default" { provisioner "local-exec" { interpreter = ["/bin/bash", "-c"] From 1e21576dc787a0c4057c70782ef7127c9093664b Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Mon, 20 Sep 2021 10:32:45 -0700 Subject: [PATCH 09/14] added mesh to 2048, fixed bugs with IRSA and certs --- terraform/provision/2048_fixture.yml | 94 +++++++++++++++++++++++++++- terraform/provision/appmesh.tf | 85 +++++++++++++++++++++++-- 2 files changed, 172 insertions(+), 7 deletions(-) diff --git a/terraform/provision/2048_fixture.yml b/terraform/provision/2048_fixture.yml index ec9cb84c..3e3847e3 100644 --- a/terraform/provision/2048_fixture.yml +++ b/terraform/provision/2048_fixture.yml @@ -12,7 +12,10 @@ spec: metadata: labels: app.kubernetes.io/name: app-2048 + annotations: + appmesh.k8s.aws/secretMounts: ca-key-pair:/etc/keys/2048/ spec: + serviceAccountName: appmesh-pod containers: - image: alexwhen/docker-2048 imagePullPolicy: Always @@ -33,6 +36,95 @@ spec: selector: app.kubernetes.io/name: app-2048 --- +apiVersion: appmesh.k8s.aws/v1beta2 +kind: VirtualNode +metadata: + namespace: default + name: deployment-2048 +spec: + awsName: deployment-2048-virtual-node + podSelector: + matchLabels: + app.kubernetes.io/name: app-2048 + listeners: + - portMapping: + port: 80 + protocol: http + serviceDiscovery: + dns: + hostname: service-2048.default.svc.cluster.local + backendDefaults: + clientPolicy: + tls: + enforce: true + validation: + trust: + file: + certificateChain: /etc/keys/2048/ca.crt +--- +apiVersion: appmesh.k8s.aws/v1beta2 +kind: VirtualService +metadata: + namespace: default + name: service-2048 +spec: + awsName: service-2048-virtual-service + provider: + virtualNode: + virtualNodeRef: + name: deployment-2048 +--- +apiVersion: appmesh.k8s.aws/v1beta2 +kind: VirtualGateway +metadata: + name: gw-2048 + namespace: default +spec: + backendDefaults: + clientPolicy: + tls: + enforce: true + validation: + trust: + file: + certificateChain: /etc/keys/2048/ca.crt + namespaceSelector: + matchLabels: + mesh: default + # podSelector: + # matchLabels: + # app.kubernetes.io/name: app-2048 + gatewayRouteSelector: + matchLabels: + gateway: gw-2048 + listeners: + - portMapping: + port: 80 + protocol: http + tls: + certificate: + file: + certificateChain: /etc/keys/2048/tls.crt + privateKey: /etc/keys/2048/tls.key + mode: STRICT +--- +apiVersion: appmesh.k8s.aws/v1beta2 +kind: GatewayRoute +metadata: + name: gateway-route + namespace: default + labels: + gateway: gw-2048 +spec: + httpRoute: + match: + prefix: "/" + action: + target: + virtualService: + virtualServiceRef: + name: service-2048 +--- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -50,4 +142,4 @@ spec: service: name: service-2048 port: - number: 80 \ No newline at end of file + number: 80 diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index d613693a..363f543e 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -12,10 +12,10 @@ resource "kubernetes_namespace" "cert-manager" { } resource "helm_release" "cert-manager" { - name = "cert-manager" - chart = "cert-manager" - repository = "https://charts.jetstack.io/" - version = "v1.5.3" + name = "cert-manager" + chart = "cert-manager" + repository = "https://charts.jetstack.io/" + version = "v1.5.3" namespace = "cert-manager" cleanup_on_fail = "true" atomic = "true" @@ -36,6 +36,9 @@ resource "helm_release" "cert-manager" { ] } +# XXX should probably use aws_acmpca_certificate_authority here? +# Sounds like there should be a way to wire ACM in. +# Maybe https://github.com/cert-manager/aws-privateca-issuer ? resource "tls_private_key" "cert-manager" { algorithm = "RSA" rsa_bits = 4096 @@ -51,11 +54,14 @@ resource "tls_self_signed_cert" "cert-manager" { } validity_period_hours = 87600 + is_ca_certificate = true allowed_uses = [ "key_encipherment", "digital_signature", "server_auth", + "client_auth", + "cert_signing" ] } @@ -119,8 +125,8 @@ resource "kubernetes_namespace" "appmesh-system" { # This role is assigned with IRSA to the appmesh controller resource "aws_iam_role" "appmesh-controller" { - name = "appmesh-controller-${local.cluster_name}" - tags = var.labels + name = "appmesh-controller-${local.cluster_name}" + tags = var.labels assume_role_policy = < Date: Wed, 29 Sep 2021 17:23:12 -0700 Subject: [PATCH 10/14] this seems like it should work, but it does not. Checkpoint --- terraform/provision/2048_fixture.yml | 99 ++++++++++++++++++++++++++-- terraform/provision/appmesh.tf | 3 +- 2 files changed, 94 insertions(+), 8 deletions(-) diff --git a/terraform/provision/2048_fixture.yml b/terraform/provision/2048_fixture.yml index 3e3847e3..d151c8bf 100644 --- a/terraform/provision/2048_fixture.yml +++ b/terraform/provision/2048_fixture.yml @@ -1,4 +1,16 @@ --- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: deployment-2048-cert + namespace: default +spec: + dnsNames: + - "deployment-2048.default.svc.cluster.local" + secretName: deployment-2048-tls + issuerRef: + name: ca-issuer +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -7,7 +19,7 @@ spec: selector: matchLabels: app.kubernetes.io/name: app-2048 - replicas: 2 + replicas: 1 template: metadata: labels: @@ -22,6 +34,14 @@ spec: name: app-2048 ports: - containerPort: 80 + volumeMounts: + - mountPath: "/etc/keys/2048" + name: deployment-2048-tls + readOnly: true + volumes: + - name: deployment-2048-tls + secret: + secretName: deployment-2048-tls --- apiVersion: v1 kind: Service @@ -50,6 +70,12 @@ spec: - portMapping: port: 80 protocol: http + tls: + certificate: + file: + certificateChain: /etc/keys/2048/tls.crt + privateKey: /etc/keys/2048/tls.key + mode: STRICT serviceDiscovery: dns: hostname: service-2048.default.svc.cluster.local @@ -73,6 +99,64 @@ spec: virtualNode: virtualNodeRef: name: deployment-2048 + +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: gw-2048-cert + namespace: default +spec: + dnsNames: + - "gw-2048.default.svc.cluster.local" + secretName: gw-2048-tls + issuerRef: + name: ca-issuer +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gw-2048 + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: gw-2048 + template: + metadata: + labels: + app: gw-2048 + annotations: + appmesh.k8s.aws/secretMounts: ca-key-pair:/etc/keys/2048/ + spec: + serviceAccountName: appmesh-pod + containers: + - name: envoy + image: 840364872350.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-envoy:v1.19.1.0-prod + ports: + - containerPort: 8443 + volumeMounts: + - mountPath: "/etc/keys/2048" + name: gw-2048-tls + readOnly: true + volumes: + - name: gw-2048-tls + secret: + secretName: gw-2048-tls +--- +apiVersion: v1 +kind: Service +metadata: + name: gw-2048 +spec: + ports: + - port: 8443 + targetPort: 8443 + protocol: TCP + type: ClusterIP + selector: + app: gw-2048 --- apiVersion: appmesh.k8s.aws/v1beta2 kind: VirtualGateway @@ -91,15 +175,15 @@ spec: namespaceSelector: matchLabels: mesh: default - # podSelector: - # matchLabels: - # app.kubernetes.io/name: app-2048 + podSelector: + matchLabels: + app: gw-2048 gatewayRouteSelector: matchLabels: gateway: gw-2048 listeners: - portMapping: - port: 80 + port: 8443 protocol: http tls: certificate: @@ -129,6 +213,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: ingress-2048 + namespace: default annotations: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/rewrite-target: / @@ -140,6 +225,6 @@ spec: pathType: Prefix backend: service: - name: service-2048 + name: gw-2048 port: - number: 80 + number: 8443 diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index 363f543e..95d39a4f 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -253,7 +253,7 @@ resource "helm_release" "appmesh-controller" { name = "appmesh-controller" chart = "appmesh-controller" repository = "https://aws.github.io/eks-charts" - version = "1.4.1" + version = "1.4.2" namespace = "appmesh-system" cleanup_on_fail = "true" @@ -281,6 +281,7 @@ resource "null_resource" "appmesh-label" { } command = <<-EOF kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default mesh=default ; + kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default gateway=gw-2048 ; kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default meshes.appmesh.k8s.aws=default ; kubectl --kubeconfig <(echo $KUBECONFIG | base64 -d) label namespace default appmesh.k8s.aws/sidecarInjectorWebhook=enabled EOF From f0d6aff0f9aed8deab912a475b6b941df436738a Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Tue, 5 Oct 2021 15:58:31 -0700 Subject: [PATCH 11/14] try using nlb instead of alb --- terraform/provision/2048_fixture.yml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/terraform/provision/2048_fixture.yml b/terraform/provision/2048_fixture.yml index d151c8bf..374352ee 100644 --- a/terraform/provision/2048_fixture.yml +++ b/terraform/provision/2048_fixture.yml @@ -149,12 +149,15 @@ apiVersion: v1 kind: Service metadata: name: gw-2048 + namespace: default + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" spec: ports: - port: 8443 targetPort: 8443 protocol: TCP - type: ClusterIP + type: LoadBalancer selector: app: gw-2048 --- @@ -208,23 +211,3 @@ spec: virtualService: virtualServiceRef: name: service-2048 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-2048 - namespace: default - annotations: - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/rewrite-target: / -spec: - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: gw-2048 - port: - number: 8443 From d2c6672891524114faf3c1c572f091cde7222ccd Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Thu, 14 Oct 2021 15:00:39 -0700 Subject: [PATCH 12/14] Now have it working from gw-2048 to the app --- terraform/provision/2048_fixture.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/terraform/provision/2048_fixture.yml b/terraform/provision/2048_fixture.yml index 374352ee..416b2964 100644 --- a/terraform/provision/2048_fixture.yml +++ b/terraform/provision/2048_fixture.yml @@ -152,11 +152,12 @@ metadata: namespace: default annotations: service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "ssl" spec: ports: - - port: 8443 + - port: 443 targetPort: 8443 - protocol: TCP + name: https type: LoadBalancer selector: app: gw-2048 @@ -171,10 +172,6 @@ spec: clientPolicy: tls: enforce: true - validation: - trust: - file: - certificateChain: /etc/keys/2048/ca.crt namespaceSelector: matchLabels: mesh: default From d8f36c6d09bdfc2ed4c062632041a06f026b08be Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Thu, 14 Oct 2021 15:43:29 -0700 Subject: [PATCH 13/14] scale 2048 back up like it was before --- terraform/provision/2048_fixture.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/provision/2048_fixture.yml b/terraform/provision/2048_fixture.yml index 416b2964..dbf513e7 100644 --- a/terraform/provision/2048_fixture.yml +++ b/terraform/provision/2048_fixture.yml @@ -19,7 +19,7 @@ spec: selector: matchLabels: app.kubernetes.io/name: app-2048 - replicas: 1 + replicas: 2 template: metadata: labels: From 779e18d513e3ad7058d53939b40a5e97fb8c22aa Mon Sep 17 00:00:00 2001 From: timothy-spencer Date: Thu, 14 Oct 2021 15:45:57 -0700 Subject: [PATCH 14/14] terraform fmt --- terraform/provision/appmesh.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/provision/appmesh.tf b/terraform/provision/appmesh.tf index 95d39a4f..935d821d 100644 --- a/terraform/provision/appmesh.tf +++ b/terraform/provision/appmesh.tf @@ -376,8 +376,8 @@ resource "aws_iam_role_policy_attachment" "envoyAccess5" { resource "kubernetes_service_account" "appmesh-pod" { metadata { - name = "appmesh-pod" - namespace = "default" + name = "appmesh-pod" + namespace = "default" annotations = { "eks.amazonaws.com/role-arn" = aws_iam_role.appmesh-pod.arn }