Merge remote-tracking branch 'origin/main' into support-more-VitessSh…

…ardTabletPool

.buildkite/pipeline.yml

@@ -7,10 +7,10 @@ steps:
   - name: "Upgrade Test"
     command:
     - apk add g++ make bash gcompat curl mysql mysql-client libc6-compat
-    - wget https://golang.org/dl/go1.20.2.linux-amd64.tar.gz
-    - tar -C /usr/local -xzf go1.20.2.linux-amd64.tar.gz
+    - wget https://golang.org/dl/go1.21.0.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.21.0.linux-amd64.tar.gz
     - export PATH=$PATH:/usr/local/go/bin
-    - rm go1.20.2.linux-amd64.tar.gz
+    - rm go1.21.0.linux-amd64.tar.gz
     - ln -s /lib/libc.so.6 /usr/lib/libresolv.so.2
     - make upgrade-test
     concurrency: 1
@@ -26,10 +26,10 @@ steps:
   - name: "Backup Restore Test"
     command:
     - apk add g++ make bash gcompat curl mysql mysql-client libc6-compat
-    - wget https://golang.org/dl/go1.20.2.linux-amd64.tar.gz
-    - tar -C /usr/local -xzf go1.20.2.linux-amd64.tar.gz
+    - wget https://golang.org/dl/go1.21.0.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.21.0.linux-amd64.tar.gz
     - export PATH=$PATH:/usr/local/go/bin
-    - rm go1.20.2.linux-amd64.tar.gz
+    - rm go1.21.0.linux-amd64.tar.gz
     - ln -s /lib/libc.so.6 /usr/lib/libresolv.so.2
     - make backup-restore-test
     concurrency: 1
@@ -45,10 +45,10 @@ steps:
   - name: "VTOrc and VTAdmin Test"
     command:
     - apk add g++ make bash gcompat curl mysql mysql-client libc6-compat chromium
-    - wget https://golang.org/dl/go1.20.2.linux-amd64.tar.gz
-    - tar -C /usr/local -xzf go1.20.2.linux-amd64.tar.gz
+    - wget https://golang.org/dl/go1.21.0.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.21.0.linux-amd64.tar.gz
     - export PATH=$PATH:/usr/local/go/bin
-    - rm go1.20.2.linux-amd64.tar.gz
+    - rm go1.21.0.linux-amd64.tar.gz
     - ln -s /lib/libc.so.6 /usr/lib/libresolv.so.2
     - make vtorc-vtadmin-test
     concurrency: 1

.github/workflows/integration-test.yaml

@@ -17,7 +17,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.2
+        go-version: 1.21.1
       id: go
 
     - name: Check out code into the Go module directory

.github/workflows/make-generate-and-diff.yaml

@@ -17,7 +17,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.2
+        go-version: 1.21.1
       id: go
 
     - name: Check out code into the Go module directory

.github/workflows/unit-test.yaml

@@ -17,14 +17,14 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.20.2
+        go-version: 1.21.1
       id: go
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v1
 
     - name: Build
-      run: go install -v ./...
+      run: CGO_ENABLED=0 go install -v ./...
 
     - name: Unit Test
       run: make unit-test

.github/workflows/upgrade-vitess-dependency.yaml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.20.2
+          go-version: 1.21.1
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2

Makefile

@@ -11,7 +11,7 @@ IMAGE_NAME:=$(IMAGE_REGISTRY)/$(IMAGE)
 export GO111MODULE=on
 
 build:
-	go build -o build/_output/bin/vitess-operator ./cmd/manager
+	CGO_ENABLED=0 go build -o build/_output/bin/vitess-operator ./cmd/manager
 
 # Release build is slow but self-contained (doesn't depend on anything in your
 # local machine). We use this for automated builds that we publish.

README.md

@@ -27,6 +27,7 @@ compatible with certain Vitess and Kubernetes versions, as shown in this table:
 | `v2.8.*`                | `v15.0.*`                   | `v1.22.*`, `v1.23.*`, or `v1.24.*`            |
 | `v2.9.*`                | `v16.0.*`                   | `v1.22.*`, `v1.23.*`, or `v1.24.*`            |
 | `v2.10.*`               | `v17.0.*`                   | `v1.22.*`, `v1.23.*`, `v1.24.*`, or `v1.25.*` |
+| `v2.11.*`               | `v18.0.*`                   | `v1.22.*`, `v1.23.*`, `v1.24.*`, or `v1.25.*` |
 | `latest`                | `latest`                    | `v1.22.*`, `v1.23.*`, `v1.24.*`, or `v1.25.*` |
 
 If for some reason you must attempt to use versions outside the recommend

build/Dockerfile.release

@@ -11,12 +11,12 @@
 # without messing up file permissions, since the Docker container doesn't run as
 # your actual user.
 
-FROM golang:1.20.2 AS build
+FROM golang:1.21.1 AS build
 
 ENV GO111MODULE=on
 WORKDIR /go/src/planetscale.dev/vitess-operator
 COPY . /go/src/planetscale.dev/vitess-operator
-RUN go install /go/src/planetscale.dev/vitess-operator/cmd/manager
+RUN CGO_ENABLED=0 go install /go/src/planetscale.dev/vitess-operator/cmd/manager
 
 # The rest is meant to mimic the output from operator-sdk's Dockerfile.
 # We just copy the binary we built inside Docker instead of from outside.

docs/release-process.md

@@ -2,6 +2,8 @@
 
 This doc describes the process to cut a new release of Vitess Operator.
 
+-------------------
+
 ## Prepare for Release
 
 For each major release of the Operator, there is a release branch, for instance: `release-2.7`.
@@ -19,6 +21,8 @@ build/Dockerfile.release
 go.mod
 ```
 
+-------------------
+
 ### Update Compatibility Table
 
 Add a new entry for the planned minor version to the [compatibility table](https://github.com/planetscale/vitess-operator/blob/main/README.md#compatibility)
@@ -38,9 +42,7 @@ time that we update to build against a newer Vitess release.
 The Kubernetes library version is determined by the version of [Operator SDK](https://github.com/operator-framework/operator-sdk)
 that's in use.
 
-### Update the test output
-
-The `upgrade.sh`, `backup_restore_test.sh` and `vtorc_vtadmin_test.sh` files must be updated with the proper release increment. Change the `verifyVtGateVersion` function calls to use the proper version (next version and new old version).
+-------------------
 
 ## Cut Release
 
@@ -95,23 +97,41 @@ OLD_VITESS_VERSION="13.0.0" NEW_VITESS_VERSION="14.0.3" NEW_OPERATOR_VERSION="2.
 Here we want to release the version `2.7.4`. It will be tested against Vitess `v14.0.3`. The upgrade downgrade tests will begin with Vitess `v13.0.0`.
 
 [Docker Hub](https://hub.docker.com/repository/docker/planetscale/vitess-operator)
-hould automatically detect the new tag and begin building a new image.
+should automatically detect the new tag and begin building a new image.
 
 Follow the instructions prompted by the `do_release.sh` script. You will need to push the tag and push the temporary branch to finally create a Pull Request. The Pull Request should be merged onto the release branch.
 
 > **Note**
 > Make sure to Normal Merge the pull request i.e. merge the pull request with merge commit and not a squash merge. This is required because we create the tag
 from the pull request, so in order to have the tag on the release branche's history, it has to be a normal merge.
 
-##### On `main`
+##### Update the test output
 
-Once you have done the release on the release branch, there are several steps to follow on `main`.
+The `upgrade_test.sh`, `backup_restore_test.sh` and `vtorc_vtadmin_test.sh` files must be updated with the proper release increment. Change the `verifyVtGateVersion` function calls to use the proper version (current version being released and latest previous version (only used in `upgrade_test.sh`)).
 
-- The `vitess/lite` image tag must be changed in [101_initial_cluster.yaml](..%2Ftest%2Fendtoend%2Foperator%2F101_initial_cluster.yaml). The latest Vitess release tag must be used.
-- We must copy the [operator-latest.yaml](..%2Ftest%2Fendtoend%2Foperator%2Foperator-latest.yaml) file we created during the release onto `main`'s [operator.yaml](..%2Ftest%2Fendtoend%2Foperator%2Foperator.yaml) file.
-- Bump the `planetscale/vitess-operator` image tag to the latest version we just released in [operator.yaml](..%2Ftest%2Fendtoend%2Foperator%2Foperator.yaml).
+##### CI Failures
+
+> **Note**
+> It is likely that the buildkite tests will fail on the release PR initially because of the unavailability of the latest vitess and vitess-operator docker images. This however doesn't block the release. The tests should be restarted after the said images are built and available.
+
+-------------------
 
 ### UI Release
 
 Create a [new release](https://github.com/planetscale/vitess-operator/releases/new)
 in GitHub UI and make sure to add the release-notes from the docs (if any).
+
+-------------------
+
+### On `main`
+
+> **Note**
+> This step is only required for the latest major release and patch releases of the latest major release.
+
+Once you have done the release on the release branch, there are several steps to follow on `main`.
+
+- The `vitess/lite` image tag must be changed in [101_initial_cluster.yaml](..%2Ftest%2Fendtoend%2Foperator%2F101_initial_cluster.yaml). The latest Vitess release tag must be used.
+- We must copy the [operator-latest.yaml](..%2Ftest%2Fendtoend%2Foperator%2Foperator-latest.yaml) file we created during the release onto `main`'s [operator.yaml](..%2Ftest%2Fendtoend%2Foperator%2Foperator.yaml) file. Once copied, remove the change that adds `imagePullPolicy: Never` and update the `image: vitess-operator-pr:latest` to use the docker image of latest vitess-operator patch like `image: planetscale/vitess-operator:v2.10.0`.
+- The `upgrade_test.sh`, `backup_restore_test.sh` and `vtorc_vtadmin_test.sh` files must be updated with the proper release increment. Change the `verifyVtGateVersion` function calls to use the proper version (new snapshot Vitess version and current version being released (only used in `upgrade_test.sh`)).
+
+

go.mod

@@ -1,61 +1,62 @@
 module planetscale.dev/vitess-operator
 
-go 1.20
+go 1.21.1
 
 require (
 	github.com/ahmetb/gen-crd-api-reference-docs v0.1.5-0.20190629210212-52e137b8d003
 	github.com/google/uuid v1.3.0
 	github.com/planetscale/operator-sdk-libs v0.0.0-20220216002626-1af183733234
-	github.com/prometheus/client_golang v1.14.0
+	github.com/prometheus/client_golang v1.15.1
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.2
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.26.1
 	k8s.io/apimachinery v0.26.1
 	k8s.io/client-go v0.26.1
 	k8s.io/klog v1.0.0
 	k8s.io/kubectl v0.21.9
-	k8s.io/kubernetes v1.26.1
+	k8s.io/kubernetes v1.26.5
 	k8s.io/utils v0.0.0-20230115233650-391b47cb4029
 	sigs.k8s.io/controller-runtime v0.14.3
 	sigs.k8s.io/controller-tools v0.11.3
 	sigs.k8s.io/kustomize v2.0.3+incompatible
-	vitess.io/vitess v0.10.3-0.20230601181117-ea38d23c5ce1
+	vitess.io/vitess v0.10.3-0.20230923204637-5b65a25eb655
 )
 
 require (
-	cloud.google.com/go v0.109.0 // indirect
-	cloud.google.com/go/compute v1.18.0 // indirect
+	cloud.google.com/go v0.110.0 // indirect
+	cloud.google.com/go/compute v1.19.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.10.0 // indirect
+	cloud.google.com/go/iam v0.13.0 // indirect
 	cloud.google.com/go/storage v1.29.0 // indirect
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
 	github.com/Azure/azure-storage-blob-go v0.15.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/obfuscate v0.42.0 // indirect
-	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.42.0 // indirect
+	github.com/DataDog/appsec-internal-go v1.0.0 // indirect
+	github.com/DataDog/datadog-agent/pkg/obfuscate v0.43.1 // indirect
+	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.45.0-rc.1 // indirect
 	github.com/DataDog/datadog-go/v5 v5.2.0 // indirect
+	github.com/DataDog/go-libddwaf v1.1.0 // indirect
 	github.com/DataDog/go-tuf v0.3.0--fix-localmeta-fork // indirect
 	github.com/DataDog/sketches-go v1.4.1 // indirect
 	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/aquarapid/vaultlib v0.5.1 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
-	github.com/aws/aws-sdk-go v1.44.192 // indirect
+	github.com/aws/aws-sdk-go v1.44.258 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/fatih/color v1.14.1 // indirect
+	github.com/fatih/color v1.15.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
@@ -68,20 +69,21 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/s2a-go v0.1.3 // indirect
 	github.com/google/safehtml v0.1.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
-	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+	github.com/googleapis/gax-go/v2 v2.8.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
-	github.com/hashicorp/consul/api v1.18.0 // indirect
+	github.com/hashicorp/consul/api v1.20.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-hclog v1.4.0 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
@@ -97,8 +99,8 @@ require (
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-ieproxy v0.0.9 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-ieproxy v0.0.10 // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 // indirect
@@ -111,20 +113,21 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/outcaste-io/ristretto v0.2.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.7 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pires/go-proxyproto v0.6.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.43.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
-	github.com/rivo/uniseg v0.4.3 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.5.0 // indirect
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/cobra v1.6.1 // indirect
@@ -136,34 +139,33 @@ require (
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
 	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
 	github.com/z-division/go-zookeeper v1.0.0 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.7 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
-	go.etcd.io/etcd/client/v3 v3.5.7 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.8 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.8 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.8 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
-	go4.org/intern v0.0.0-20220617035311-6925f38cc365 // indirect
-	go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect
-	golang.org/x/crypto v0.5.0 // indirect
-	golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 // indirect
-	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/term v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	go4.org/intern v0.0.0-20230205224052-192e9f60865c // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20230426161633-7e06285ff160 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
+	golang.org/x/oauth2 v0.7.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/term v0.11.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.5.0 // indirect
+	golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
-	google.golang.org/api v0.109.0 // indirect
+	google.golang.org/api v0.121.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230131230820-1c016267d619 // indirect
-	google.golang.org/grpc v1.52.3 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
-	gopkg.in/DataDog/dd-trace-go.v1 v1.47.0 // indirect
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/grpc v1.55.0-dev // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/DataDog/dd-trace-go.v1 v1.50.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect