From 68798260f4550d646d9478a6510a69e6b4005acb Mon Sep 17 00:00:00 2001 From: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> Date: Mon, 12 Feb 2024 15:48:02 -0600 Subject: [PATCH] Net 7238 - consul k8s modify gateway resources job to create apigw gatewayclass and gatewayclassconfig (#3564) * configmap update * udpate chart to respect api-gateway-config * fix typo * added unit tests, added some stuff missed in initial pass * added thorough unit tests for gateway-resources-configmap.yaml * remove unneeded extra line * additional debugging * test * test * remove extra escapes * final test * test again * one more test * this should work * fix spacing issue --- .../gateway-resources-configmap.yaml | 69 ++++- .../unit/gateway-resources-configmap.bats | 237 +++++++++++++++++- 2 files changed, 294 insertions(+), 12 deletions(-) diff --git a/charts/consul/templates/gateway-resources-configmap.yaml b/charts/consul/templates/gateway-resources-configmap.yaml index 842ba6690d..d00f9b3e86 100644 --- a/charts/consul/templates/gateway-resources-configmap.yaml +++ b/charts/consul/templates/gateway-resources-configmap.yaml @@ -21,9 +21,10 @@ data: resources.json: | {{ toJson .Values.connectInject.apiGateway.managedGatewayClass.resources }} {{- end }} - {{- if and (mustHas "resource-apis" .Values.global.experiments) .Values.meshGateway.enabled }} + {{- if and (mustHas "resource-apis" .Values.global.experiments) (or .Values.meshGateway.enabled .Values.connectInject.apiGateway.managedGatewayClass) }} config.yaml: | gatewayClassConfigs: + {{- if .Values.meshGateway.enabled }} - apiVersion: mesh.consul.hashicorp.com/v2beta1 metadata: name: consul-mesh-gateway @@ -87,7 +88,8 @@ data: min: {{ .Values.meshGateway.replicas }} max: {{ .Values.meshGateway.replicas }} {{- if .Values.meshGateway.tolerations }} - tolerations: {{ fromYamlArray .Values.meshGateway.tolerations | toJson }} + tolerations: + {{ fromYamlArray .Values.meshGateway.tolerations | toJson }} {{- end }} service: {{- if .Values.meshGateway.service.annotations }} @@ -100,6 +102,67 @@ data: annotations: set: {{ toJson .Values.meshGateway.serviceAccount.annotations }} {{- end }} + {{- end }} + {{- if .Values.connectInject.apiGateway.managedGatewayClass }} + - apiVersion: mesh.consul.hashicorp.com/v2beta1 + metadata: + name: consul-api-gateway + kind: GatewayClassConfig + spec: + labels: + set: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + component: api-gateway + {{- if .Values.connectInject.apiGateway.managedGatewayClass.copyAnnotations }} + {{- if .Values.connectInject.apiGateway.managedGatewayClass.copyAnnotations.service }} + annotations: + service: + {{ fromYamlArray .Values.connectInject.apiGateway.managedGatewayClass.copyAnnotations.service.annotations | toYaml }} + {{- end}} + {{- end}} + deployment: + {{- if .Values.connectInject.apiGateway.managedGatewayClass.nodeSelector }} + nodeSelector: + {{ fromYamlArray .Values.connectInject.apiGateway.managedGatewayClass.nodeSelector | toYaml }} + {{- end }} + initContainer: + {{- if .Values.connectInject.apiGateway.managedGatewayClass.mapPrivilegedContainerPorts }} + portModifier: {{ .Values.connectInject.apiGateway.managedGatewayClass.mapPrivilegedContainerPorts }} + {{- end }} + consul: + logging: + level: {{ .Values.global.logLevel }} + container: + {{- if .Values.connectInject.apiGateway.managedGatewayClass.mapPrivilegedContainerPorts }} + portModifier: {{ .Values.connectInject.apiGateway.managedGatewayClass.mapPrivilegedContainerPorts }} + {{- end }} + consul: + logging: + level: {{ .Values.global.logLevel }} + replicas: + default: {{ .Values.connectInject.apiGateway.managedGatewayClass.deployment.defaultInstances }} + min: {{ .Values.connectInject.apiGateway.managedGatewayClass.deployment.minInstances }} + max: {{ .Values.connectInject.apiGateway.managedGatewayClass.deployment.maxInstances }} + {{- if .Values.connectInject.apiGateway.managedGatewayClass.tolerations }} + tolerations: + {{ fromYamlArray .Values.connectInject.apiGateway.managedGatewayClass.tolerations | toYaml }} + {{- end }} + {{- if .Values.connectInject.apiGateway.managedGatewayClass.service }} + service: + annotations: + set: {{ toYaml .Values.connectInject.apiGateway.managedGatewayClass.service.annotations }} + {{- end }} + type: {{ .Values.connectInject.apiGateway.managedGatewayClass.serviceType }} + {{- if .Values.connectInject.apiGateway.managedGatewayClass.serviceAccount }} + serviceAccount: + annotations: + set: {{ toYaml .Values.connectInject.apiGateway.managedGatewayClass.serviceAccount.annotations }} + {{- end }} + {{- end }} + {{- if .Values.meshGateway.enabled }} meshGateways: - apiVersion: mesh.consul.hashicorp.com/v2beta1 kind: MeshGateway @@ -107,7 +170,6 @@ data: name: mesh-gateway namespace: {{ .Release.Namespace }} annotations: - # TODO are these annotations even necessary? "consul.hashicorp.com/gateway-wan-address-source": {{ .Values.meshGateway.wanAddress.source | quote }} "consul.hashicorp.com/gateway-wan-address-static": {{ .Values.meshGateway.wanAddress.static | quote }} {{- if eq .Values.meshGateway.wanAddress.source "Service" }} @@ -128,5 +190,6 @@ data: workloads: prefixes: - "mesh-gateway" + {{- end }} {{- end }} {{- end }} diff --git a/charts/consul/test/unit/gateway-resources-configmap.bats b/charts/consul/test/unit/gateway-resources-configmap.bats index e827644792..ea3decc5c7 100644 --- a/charts/consul/test/unit/gateway-resources-configmap.bats +++ b/charts/consul/test/unit/gateway-resources-configmap.bats @@ -94,6 +94,28 @@ target=templates/gateway-resources-configmap.yaml } +#-------------------------------------------------------------------- +# API Gateway logLevel configuration + +@test "gateway-resources/ConfigMap: API Gateway logLevel default configuration" { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'meshGateway.enabled=false' \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.container.consul.logging.level') + [ "${actual}" = 'info' ] + + local actual=$(echo "$config" | yq -r '.initContainer.consul.logging.level') + [ "${actual}" = 'info' ] +} + + + @test "gateway-resources/ConfigMap: Mesh Gateway logLevel custom global configuration" { cd `chart_dir` local config=$(helm template \ @@ -199,16 +221,17 @@ target=templates/gateway-resources-configmap.yaml . | tee /dev/stderr | yq -r '.data["config.yaml"]' | yq -r '.meshGateways[0].metadata.annotations' | tee /dev/stderr) - local actual=$(echo "$annotations" | jq -r '.["consul.hashicorp.com/gateway-wan-address-source"]') + local actual=$(echo "$annotations" | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]') [ "${actual}" = 'Service' ] - local actual=$(echo "$annotations" | jq -r '.["consul.hashicorp.com/gateway-wan-port"]') + local actual=$(echo "$annotations" | yq -r '.["consul.hashicorp.com/gateway-wan-port"]') [ "${actual}" = '443' ] - local actual=$(echo "$annotations" | jq -r '.["consul.hashicorp.com/gateway-wan-address-static"]') + local actual=$(echo "$annotations" | yq -r '.["consul.hashicorp.com/gateway-wan-address-static"]') [ "${actual}" = '' ] } + @test "gateway-resources/ConfigMap: Mesh Gateway WAN Address NodePort annotations" { cd `chart_dir` local annotations=$(helm template \ @@ -223,13 +246,13 @@ target=templates/gateway-resources-configmap.yaml . | tee /dev/stderr | yq -r '.data["config.yaml"]' | yq -r '.meshGateways[0].metadata.annotations' | tee /dev/stderr) - local actual=$(echo "$annotations" | jq -r '.["consul.hashicorp.com/gateway-wan-address-source"]') + local actual=$(echo "$annotations" | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]') [ "${actual}" = 'Service' ] - local actual=$(echo "$annotations" | jq -r '.["consul.hashicorp.com/gateway-wan-port"]') + local actual=$(echo "$annotations" | yq -r '.["consul.hashicorp.com/gateway-wan-port"]') [ "${actual}" = '30000' ] - local actual=$(echo "$annotations" | jq -r '.["consul.hashicorp.com/gateway-wan-address-static"]') + local actual=$(echo "$annotations" | yq -r '.["consul.hashicorp.com/gateway-wan-address-static"]') [ "${actual}" = '' ] } @@ -246,13 +269,209 @@ target=templates/gateway-resources-configmap.yaml . | tee /dev/stderr | yq -r '.data["config.yaml"]' | yq -r '.meshGateways[0].metadata.annotations' | tee /dev/stderr) - local actual=$(echo "$annotations" | jq -r '.["consul.hashicorp.com/gateway-wan-address-source"]') + local actual=$(echo "$annotations" | yq -r '.["consul.hashicorp.com/gateway-wan-address-source"]') [ "${actual}" = 'Static' ] - local actual=$(echo "$annotations" | jq -r '.["consul.hashicorp.com/gateway-wan-port"]') + local actual=$(echo "$annotations" | yq -r '.["consul.hashicorp.com/gateway-wan-port"]') [ "${actual}" = '443' ] - local actual=$(echo "$annotations" | jq -r '.["consul.hashicorp.com/gateway-wan-address-static"]') + local actual=$(echo "$annotations" | yq -r '.["consul.hashicorp.com/gateway-wan-address-static"]') [ "${actual}" = '127.0.0.1' ] } +#-------------------------------------------------------------------- +# API Gateway Tests mapPrivilageContainerPorts + +@test "gateway-resources/ConfigMap: API Gateway mapPrivilageContainerPorts empty by default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.container.portModifier') + + [ "${actual}" = 'null' ] + + local actual=$(echo "$config" | yq -r '.initContainer.portModifier') + + [ "${actual}" = 'null' ] +} + + +@test "gateway-resources/ConfigMap: API Gateway mapPrivilageContainerPorts overrides default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + --set 'connectInject.apiGateway.managedGatewayClass.mapPrivilegedContainerPorts=80' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.container.portModifier') + + [ "${actual}" = '80' ] + + local actual=$(echo "$config" | yq -r '.initContainer.portModifier') + + [ "${actual}" = '80' ] +} + +#-------------------------------------------------------------------- +# API Gateway Tests deployment replicas + +@test "gateway-resources/ConfigMap: API Gateway deploymentConfig overrides default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + --set 'connectInject.apiGateway.managedGatewayClass.deployment.defaultInstances=2' \ + --set 'connectInject.apiGateway.managedGatewayClass.deployment.maxInstances=3' \ + --set 'connectInject.apiGateway.managedGatewayClass.deployment.minInstances=1' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.replicas.default') + [ "${actual}" = '2' ] + + local actual=$(echo "$config" | yq -r '.replicas.min') + [ "${actual}" = '1' ] + + local actual=$(echo "$config" | yq -r '.replicas.max') + [ "${actual}" = '3' ] +} + +@test "gateway-resources/ConfigMap: API Gateway deploymentConfig default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.replicas.default') + [ "${actual}" = '1' ] + + local actual=$(echo "$config" | yq -r '.replicas.min') + [ "${actual}" = '1' ] + + local actual=$(echo "$config" | yq -r '.replicas.max') + [ "${actual}" = '1' ] +} + +#-------------------------------------------------------------------- +# API Gateway Tests nodeSelector + +@test "gateway-resources/ConfigMap: API Gateway nodeSelector overrides default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + --set 'connectInject.apiGateway.managedGatewayClass.nodeSelector=- key: value' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.nodeSelector[0].key') + echo ${actual} + + [ "${actual}" = 'value' ] +} + +@test "gateway-resources/ConfigMap: API Gateway nodeSelector default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.nodeSelector') + [ "${actual}" = 'null' ] +} + +#-------------------------------------------------------------------- +# API Gateway Tests tolerations + +@test "gateway-resources/ConfigMap: API Gateway tolerations overrides default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + --set 'connectInject.apiGateway.managedGatewayClass.tolerations=- key: value' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.tolerations[0].key') + echo "${actual}" + + [ "${actual}" = 'value' ] +} + + + +@test "gateway-resources/ConfigMap: API Gateway tolerations default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.tolerations') + [ "${actual}" = 'null' ] +} + + +#-------------------------------------------------------------------- +# API Gateway Tests copyAnnotations + +@test "gateway-resources/ConfigMap: API Gateway copyAnnotations overrides default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + --set 'connectInject.apiGateway.managedGatewayClass.copyAnnotations.service.annotations=- annotation.name' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.annotations' | tee /dev/stderr) + + local actual=$(echo "$config" | yq -r '.service[0]') + echo "${actual}" + [ "${actual}" = 'annotation.name' ] +} + +@test "gateway-resources/ConfigMap: API Gateway copyAnnotations default { + cd `chart_dir` + local config=$(helm template \ + -s $target \ + --set 'global.experiments[0]=resource-apis' \ + --set 'ui.enabled=false' \ + --set 'global.logLevel=error' \ + . | tee /dev/stderr | + yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.annotations' | tee /dev/stderr) + + local actual=$(echo "$config" | jq -r '.service') + [ "${actual}" = 'null' ] +} + + +#-------------------------------------------------------------------- +# TODO openShiftSSCName \ No newline at end of file