backport of commit 7d2782e

.changelog/4256.txt

@@ -1,3 +1,3 @@
 ```release-note:improvement
 config-entry: add validate_clusters to mesh config entry
-```
+```

.changelog/4313.txt

@@ -0,0 +1,4 @@
+```release-note:security
+Upgrade Go to use 1.22.7. This addresses CVE 
+[CVE-2024-34155](https://nvd.nist.gov/vuln/detail/CVE-2024-34155)
+```

.github/scripts/check_skip_ci.sh

@@ -24,10 +24,11 @@ function contains() {
 #
 # ... `git merge-base origin/$SKIP_CHECK_BRANCH HEAD` would return commit `D`
 # `...HEAD` specifies from the common ancestor to the latest commit on the current branch (HEAD)..
-files_to_check=$(git diff --name-only "$(git merge-base origin/$SKIP_CHECK_BRANCH HEAD~)"...HEAD)
+skip_check_branch=${SKIP_CHECK_BRANCH:?SKIP_CHECK_BRANCH is required}
+files_to_check=$(git diff --name-only "$(git merge-base origin/$skip_check_branch HEAD~)"...HEAD)
 
 # Define the directories to check
-skipped_directories=("assets" ".changelog/", "version")
+skipped_directories=("assets" ".changelog")
 
 files_to_skip=("LICENSE" ".copywrite.hcl" ".gitignore")
 
@@ -43,7 +44,7 @@ for file_to_check in "${files_to_check_array[@]}"; do
   # - Markdown files
   for dir in "${skipped_directories[@]}"; do
     if [[ "$file_to_check" == */check_skip_ci.sh ]] ||
-      [[ "$file_to_check" == "$dir"* ]] ||
+      [[ "$file_to_check" == "$dir/"* ]] ||
       [[ "$file_to_check" == *.md ]] ||
       contains "${files_to_skip[*]}" "$file_to_check"; then
       file_is_skipped=true

.github/workflows/pr.yml

@@ -44,6 +44,7 @@ jobs:
           - check-name: Unit test control plane
           - check-name: Unit test cli
           - check-name: Unit test acceptance
+          - check-name: Unit test helm gen
     steps:
     - name: Update final status
       uses: docker://ghcr.io/curtbushko/commit-status-action:e1d661c757934ab35c74210b4b70c44099ec747a

.github/workflows/security-scan.yml

@@ -46,7 +46,8 @@ jobs:
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
         with:
           repository: hashicorp/security-scanner
-          token: ${{ secrets.HASHIBOT_PRODSEC_GITHUB_TOKEN }}
+          #TODO: replace w/ HASHIBOT_PRODSEC_GITHUB_TOKEN once provisioned
+          token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
           path: security-scanner
           ref: main
 
@@ -65,4 +66,4 @@ jobs:
       - name: Upload SARIF file
         uses: github/codeql-action/upload-sarif@c4fb451437765abf5018c6fbf22cce1a7da1e5cc # codeql-bundle-v2.17.1
         with:
-          sarif_file: results.sarif
+          sarif_file: results.sarif

.github/workflows/weekly-acceptance-1-5-x.yml → .github/workflows/weekly-acceptance-1-4-0-rc1.yml

@@ -1,7 +1,7 @@
 # Dispatch to the consul-k8s-workflows with a weekly cron
 #
 # A separate file is needed for each release because the cron schedules are different for each release.
-name: weekly-acceptance-1-5-x
+name: weekly-acceptance-1-4-0-rc1
 on:
   schedule:
     # * is a special character in YAML so you have to quote this string
@@ -10,7 +10,7 @@ on:
 
 # these should be the only settings that you will ever need to change
 env:
-  BRANCH: "release/1.5.x"
+  BRANCH: "release/1.4.0-rc1"
   CONTEXT: "weekly"
 
 jobs:

.gitignore

@@ -11,4 +11,3 @@ pkg/
 .vscode
 .bob/
 control-plane/cni/cni
-acceptance/tests/consul-dns/coredns-custom.yaml

.go-version

@@ -1 +1 @@
-1.22.5
+1.22.7

CHANGELOG.md

@@ -1,24 +1,35 @@
-## 1.5.1 (July 16, 2024)
+## 1.5.3 (August 30, 2024)
 
 SECURITY:
 
-* Upgrade go version to 1.22.5 to address [CVE-2024-24791](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-24791) [[GH-4154](https://github.com/hashicorp/consul-k8s/issues/4154)]
-* Upgrade go-retryablehttp to v0.7.7 to address [GHSA-v6v8-xj6m-xwqh](https://github.com/advisories/GHSA-v6v8-xj6m-xwqh) [[GH-4169](https://github.com/hashicorp/consul-k8s/issues/4169)]
+* Bump Go to 1.22.5 to address [CVE-2024-24791](https://nvd.nist.gov/vuln/detail/CVE-2024-24791) [[GH-4228](https://github.com/hashicorp/consul-k8s/issues/4228)]
+* Upgrade Docker cli to use v.27.1. This addresses CVE
+[CVE-2024-41110](https://nvd.nist.gov/vuln/detail/CVE-2024-41110) [[GH-4228](https://github.com/hashicorp/consul-k8s/issues/4228)]
 
 IMPROVEMENTS:
 
-* api-gateways: Change security settings to make root file system read only and to not allow privilage escalation. [[GH-3959](https://github.com/hashicorp/consul-k8s/issues/3959)]
-* control-plane: Remove anyuid Security Context Constraints (SCC) requirement in OpenShift. [[GH-4152](https://github.com/hashicorp/consul-k8s/issues/4152)]
-* partition-init: Role no longer includes unnecessary access to Secrets resource. [[GH-4053](https://github.com/hashicorp/consul-k8s/issues/4053)]
+* docker: update go-discover binary [[GH-4287](https://github.com/hashicorp/consul-k8s/issues/4287)]
+* docker: update ubi base image to `ubi9-minimal:9.4`. [[GH-4287](https://github.com/hashicorp/consul-k8s/issues/4287)]
+* helm: Adds `webhookCertManager.resources` field which can be configured to override the `resource` settings for the `webhook-cert-manager` deployment. [[GH-4184](https://github.com/hashicorp/consul-k8s/issues/4184)]
+* helm: Adds `connectInject.apiGateway.managedGatewayClass.resourceJob.resources` field which can be configured to override the `resource` settings for the `gateway-resources-job` job. [[GH-4184](https://github.com/hashicorp/consul-k8s/issues/4184)]
+* config-entry: add validate_clusters to mesh config entry [[GH-4256](https://github.com/hashicorp/consul-k8s/issues/4256)]
+* helm: Kubernetes v1.30 is now supported. Minimum tested version of Kubernetes is now v1.27. [[GH-4244](https://github.com/hashicorp/consul-k8s/issues/4244)]
 
 BUG FIXES:
 
-* api-gateway: fix issue where API Gateway specific acl roles/policy were not being cleaned up on deletion of an api-gateway [[GH-4060](https://github.com/hashicorp/consul-k8s/issues/4060)]
-* connect-inject: add NET_BIND_SERVICE capability when injecting consul-dataplane sidecar [[GH-4152](https://github.com/hashicorp/consul-k8s/issues/4152)]
-* endpoints-controller: graceful shutdown logic should not run on a new pod with the same name. Fixes a case where statefulset rollouts could get stuck in graceful shutdown when the new pods come up. [[GH-4059](https://github.com/hashicorp/consul-k8s/issues/4059)]
-* terminating-gateway: Fix generated acl policy for external services to include the namespace and partition block if they are enabled. [[GH-4153](https://github.com/hashicorp/consul-k8s/issues/4153)]
+* Fixes install of Consul on GKE Autopilot where the option 'manageNonStandardCRDs' was not being used for the TCPRoute CRD. [[GH-4213](https://github.com/hashicorp/consul-k8s/issues/4213)]
+* api-gateway: fix nil pointer deref bug when the section name in a gateway policy is not specified [[GH-4247](https://github.com/hashicorp/consul-k8s/issues/4247)]
+* helm: adds imagePullSecret to the gateway-resources job and the gateway-cleanup job, would fail before if the image was in a private registry [[GH-4210](https://github.com/hashicorp/consul-k8s/issues/4210)]
+* openshift: order SecurityContextConstraint volumes alphabetically to match OpenShift behavior.
+This ensures that diff detection tools like ArgoCD consider the source and reconciled resources to be identical. [[GH-4227](https://github.com/hashicorp/consul-k8s/issues/4227)]
+* sync-catalog: fix infinite retry loop when the catalog fails to connect to consul-server during the sync process [[GH-4266](https://github.com/hashicorp/consul-k8s/issues/4266)]
+* terminating-gateways: Fix bug where namespace field was not correctly set on ACL policies if using the `Registration` CRD with the service's namespace unset. [[GH-4224](https://github.com/hashicorp/consul-k8s/issues/4224)]
 
-## 1.4.4 (July 15, 2024)
+## 1.5.2 (August 29, 2024)
+
+Release redacted, use `1.5.3`
+
+## 1.5.1 (July 16, 2024)
 
 SECURITY:
 
@@ -27,52 +38,16 @@ SECURITY:
 
 IMPROVEMENTS:
 
-* upgrade go version to v1.22.4. [[GH-4085](https://github.com/hashicorp/consul-k8s/issues/4085)]
 * api-gateways: Change security settings to make root file system read only and to not allow privilage escalation. [[GH-3959](https://github.com/hashicorp/consul-k8s/issues/3959)]
-* cni: package `consul-cni` as .deb and .rpm files [[GH-4040](https://github.com/hashicorp/consul-k8s/issues/4040)]
 * control-plane: Remove anyuid Security Context Constraints (SCC) requirement in OpenShift. [[GH-4152](https://github.com/hashicorp/consul-k8s/issues/4152)]
 * partition-init: Role no longer includes unnecessary access to Secrets resource. [[GH-4053](https://github.com/hashicorp/consul-k8s/issues/4053)]
 
 BUG FIXES:
 
 * api-gateway: fix issue where API Gateway specific acl roles/policy were not being cleaned up on deletion of an api-gateway [[GH-4060](https://github.com/hashicorp/consul-k8s/issues/4060)]
-* cni: fix incorrect release version due to unstable submodule pinning [[GH-4091](https://github.com/hashicorp/consul-k8s/issues/4091)]
 * connect-inject: add NET_BIND_SERVICE capability when injecting consul-dataplane sidecar [[GH-4152](https://github.com/hashicorp/consul-k8s/issues/4152)]
 * endpoints-controller: graceful shutdown logic should not run on a new pod with the same name. Fixes a case where statefulset rollouts could get stuck in graceful shutdown when the new pods come up. [[GH-4059](https://github.com/hashicorp/consul-k8s/issues/4059)]
-
-## 1.3.7 (July 16, 2024)
-
-SECURITY:
-
-* Upgrade go version to 1.22.5 to address [CVE-2024-24791](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-24791) [[GH-4154](https://github.com/hashicorp/consul-k8s/issues/4154)]
-* Upgrade go-retryablehttp to v0.7.7 to address [GHSA-v6v8-xj6m-xwqh](https://github.com/advisories/GHSA-v6v8-xj6m-xwqh) [[GH-4169](https://github.com/hashicorp/consul-k8s/issues/4169)]
-
-IMPROVEMENTS:
-
-* upgrade go version to v1.22.4. [[GH-4085](https://github.com/hashicorp/consul-k8s/issues/4085)]
-* partition-init: Role no longer includes unnecessary access to Secrets resource. [[GH-4053](https://github.com/hashicorp/consul-k8s/issues/4053)]
-
-BUG FIXES:
-
-* api-gateway: fix issue where API Gateway specific acl roles/policy were not being cleaned up on deletion of an api-gateway [[GH-4060](https://github.com/hashicorp/consul-k8s/issues/4060)]
-* cni: fix incorrect release version due to unstable submodule pinning [[GH-4091](https://github.com/hashicorp/consul-k8s/issues/4091)]
-* endpoints-controller: graceful shutdown logic should not run on a new pod with the same name. Fixes a case where statefulset rollouts could get stuck in graceful shutdown when the new pods come up. [[GH-4059](https://github.com/hashicorp/consul-k8s/issues/4059)]
-
-## 1.1.14 (July 16, 2024)
-
-SECURITY:
-
-* Upgrade go version to 1.22.5 to address [CVE-2024-24791](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-24791) [[GH-4154](https://github.com/hashicorp/consul-k8s/issues/4154)]
-* Upgrade go-retryablehttp to v0.7.7 to address [GHSA-v6v8-xj6m-xwqh](https://github.com/advisories/GHSA-v6v8-xj6m-xwqh) [[GH-4169](https://github.com/hashicorp/consul-k8s/issues/4169)]
-
-IMPROVEMENTS:
-
-* upgrade go version to v1.22.4. [[GH-4085](https://github.com/hashicorp/consul-k8s/issues/4085)]
-* partition-init: Role no longer includes unnecessary access to Secrets resource. [[GH-4053](https://github.com/hashicorp/consul-k8s/issues/4053)]
-
-BUG FIXES:
-
-* cni: fix incorrect release version due to unstable submodule pinning [[GH-4091](https://github.com/hashicorp/consul-k8s/issues/4091)]
+* terminating-gateway: Fix generated acl policy for external services to include the namespace and partition block if they are enabled. [[GH-4153](https://github.com/hashicorp/consul-k8s/issues/4153)]
 
 ## 1.5.0 (June 13, 2024)
 

CONTRIBUTING.md

@@ -961,23 +961,17 @@ The tests are organized like this :
 ```shell
 demo $ tree -L 1 -d acceptance/tests
 acceptance/tests
-├── api-gateway
 ├── basic
 ├── cli
-├── cloud
 ├── config-entries
 ├── connect
 ├── consul-dns
-├── datadog
 ├── example
 ├── fixtures
 ├── ingress-gateway
 ├── metrics
 ├── partitions
 ├── peering
-├── sameness
-├── segments
-├── server
 ├── snapshot-agent
 ├── sync
 ├── terminating-gateway
@@ -1015,9 +1009,7 @@ $ kind create cluster --name=dc1 && kind create cluster --name=dc2
   `-consul-k8s-image=<your-custom-image>` && `-consul-image=<your-custom-image>`
 * You can set custom helm flags by modifying the test file directly in the respective directory.
 
-Finally, you have two options on how you can run your test:
-1. Take the following steps, this will run the test through to completion but not teardown any resources created by the test so you can inspect the state of the cluster
-at that point. You will be responsible for cleaning up the resources or deleting the cluster entirely when you're done.
+Finally, run the test like shown above:
 ```shell
 $ cd acceptance/tests
 $ go test -run Vault_WANFederationViaGateways ./vault/... -p 1 -timeout 2h -failfast -use-kind -no-cleanup-on-failure -kubecontext=kind-dc1 -secondary-kubecontext=kind-dc2 -enable-multi-cluster -debug-directory=/tmp/debug
@@ -1026,36 +1018,6 @@ You can interact with the running kubernetes clusters now using `kubectl [COMMAN
 
 * `kind delete clusters --all` is helpful for cleanup!
 
-2. The other option is to use the helper method in the framework: `helpers.WaitForInput(t)` at the spot in your acceptance test where you would like to pause execution to inspect the cluster. This will pause the test execution until you execute a request to `localhost:38501` which tells the test to continue running, you can override the port value used by setting the `CONSUL_K8S_TEST_PAUSE_PORT` environment variable to a port of your choosing. When running the tests with the `-v` flag you will see a log output of the endpoint that the test is waiting on.
-
-First you'll want to add the helper method to your test file:
-
-```go
-import "github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
-
-func TestSomeTest(t *testing.T) {
-  // stuff to setup
-
-  // test execution will pause here until the endpoint is hit
-  helpers.WaitForInput(t)
-
-  // rest of test
-}
-```
-
-Then run the tests (note the removal of the `-no-cleanup-on-failure` flag):
-```shell
-$ cd acceptance/tests
-$ go test -run Vault_WANFederationViaGateways ./vault/... -p 1 -timeout 2h -failfast -use-kind -kubecontext=kind-dc1 -secondary-kubecontext=kind-dc2 -enable-multi-cluster -debug-directory=/tmp/debug
-```
-
-You can interact with the running kubernetes clusters now using `kubectl [COMMAND] --context=<kind-dc1/kind-dc2>`
-
-When you're done interacting you can tell the test to continue by issuing a curl command to the endpoint (if you are using a non-default port for this test then replace the `38501` port value with the value you have set):
-```shell
-curl localhost:38501
-```
-
 ### Example Debugging session using the acceptance test framework to bootstrap and debug a Vault backed federated Consul installation:
 This test utilizes the `consul-k8s` acceptance test framework, with a custom consul-k8s branch which:
 * Modifies the acceptance test to use custom consul+consul-k8s images and sleeps at the end of the test to allow analysis.

Makefile

@@ -9,8 +9,6 @@ KUBECTL_VERSION= $(shell ./control-plane/build-support/scripts/read-yaml-config.
 
 GO_MODULES := $(shell find . -name go.mod -exec dirname {} \; | sort)
 
-GOTESTSUM_PATH?=$(shell command -v gotestsum)
-
 ##@ Helm Targets
 
 .PHONY: gen-helm-docs
@@ -99,32 +97,11 @@ control-plane-fips-dev-docker: ## Build consul-k8s-control-plane FIPS dev Docker
 
 .PHONY: control-plane-test
 control-plane-test: ## Run go test for the control plane.
-ifeq ("$(GOTESTSUM_PATH)","")
-	cd control-plane && go test ./...
-else
-	cd control-plane && \
-	gotestsum \
-		--format=short-verbose \
-		--debug \
-		--rerun-fails=3 \
-		--packages="./..."
-endif
-
+	cd control-plane; go test ./...
 
 .PHONY: control-plane-ent-test
 control-plane-ent-test: ## Run go test with Consul enterprise tests. The consul binary in your PATH must be Consul Enterprise.
-ifeq ("$(GOTESTSUM_PATH)","")
-	cd control-plane && go test ./... -tags=enterprise
-else
-	cd control-plane && \
-	gotestsum \
-		--format=short-verbose \
-		--debug \
-		--rerun-fails=3 \
-		--packages="./..." \
-		-- \
-		--tags enterprise
-endif
+	cd control-plane; go test ./... -tags=enterprise
 
 .PHONY: control-plane-cov
 control-plane-cov: ## Run go test with code coverage.
@@ -427,7 +404,7 @@ ifndef CONSUL_K8S_RELEASE_DATE
 	$(error CONSUL_K8S_RELEASE_DATE is required, use format <Month> <Day>, <Year> (ex. October 4, 2022))
 endif
 ifndef CONSUL_K8S_NEXT_RELEASE_VERSION
-	$(error CONSUL_K8S_NEXT_RELEASE_VERSION is required)
+	$(error CONSUL_K8S_RELEASE_VERSION is required)
 endif
 ifndef CONSUL_K8S_CONSUL_VERSION
 	$(error CONSUL_K8S_CONSUL_VERSION is required)

acceptance/framework/consul/helm_cluster.go

@@ -13,6 +13,8 @@ import (
 	"github.com/gruntwork-io/terratest/modules/helm"
 	terratestLogger "github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 	corev1 "k8s.io/api/core/v1"
 	policyv1beta "k8s.io/api/policy/v1beta1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -26,6 +28,7 @@ import (
 
 	"github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1"
 	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/proto-public/pbresource"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 
 	"github.com/hashicorp/consul-k8s/acceptance/framework/config"
@@ -484,6 +487,26 @@ func (h *HelmCluster) CreatePortForwardTunnel(t *testing.T, remotePort int, rele
 	return portforward.CreateTunnelToResourcePort(t, serverPod, remotePort, h.helmOptions.KubectlOptions, h.logger)
 }
 
+// ResourceClient returns a resource service grpc client for the given helm release.
+func (h *HelmCluster) ResourceClient(t *testing.T, secure bool, release ...string) (client pbresource.ResourceServiceClient) {
+	if secure {
+		panic("TODO: add support for secure resource client")
+	}
+	releaseName := h.releaseName
+	if len(release) > 0 {
+		releaseName = release[0]
+	}
+
+	// TODO: get grpc port from somewhere
+	localTunnelAddr := h.CreatePortForwardTunnel(t, 8502, releaseName)
+
+	// Create a grpc connection to the server pod.
+	grpcConn, err := grpc.Dial(localTunnelAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.NoError(t, err)
+	resourceClient := pbresource.NewResourceServiceClient(grpcConn)
+	return resourceClient
+}
+
 func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool, release ...string) (client *api.Client, configAddress string) {
 	t.Helper()