diff --git a/docs/antctl.md b/docs/antctl.md index 8b4dd67c80b..f19bfd59905 100644 --- a/docs/antctl.md +++ b/docs/antctl.md @@ -1,6 +1,6 @@ # Antctl -Antctl is the command-line tool for Antrea. At the moment, antctl supports +antctl is the command-line tool for Antrea. At the moment, antctl supports running in three different modes: * "controller mode": when run out-of-cluster or from within the Antrea @@ -36,6 +36,7 @@ running in three different modes: - [Multi-cluster commands](#multi-cluster-commands) - [Multicast commands](#multicast-commands) - [Showing memberlist state](#showing-memberlist-state) + - [Upgrade existing objects of CRDs](#upgrade-existing-objects-of-crds) ## Installation @@ -497,7 +498,7 @@ $ antctl traceflow -D pod1 -f tcp,tcp_dst=80 --live-traffic --dropped-only -t 10 ### Antctl Proxy -Antctl can run as a reverse proxy for the Antrea API (Controller or arbitrary +antctl can run as a reverse proxy for the Antrea API (Controller or arbitrary Agent). Usage is very similar to `kubectl proxy` and the implementation is essentially the same. @@ -669,7 +670,8 @@ testmulticast-vw7gx5b9 test3-sender-1 0 10 ### Showing memberlist state -`antctl` agent command `get memberlist` (or `get ml`) prints the state of memberlist cluster of Antrea Agent. +`antctl` agent command `get memberlist` (or `get ml`) prints the state of memberlist +cluster of Antrea Agent. ```bash $ antctl get memberlist @@ -679,3 +681,78 @@ worker1 172.18.0.4 Alive worker2 172.18.0.3 Alive worker3 172.18.0.2 Dead ``` + +### Upgrade existing objects of CRDs + +antctl supports upgrading existing objects of Antrea CRDs to the storage version. +The related sub-commands should be run out-of-cluster. Please ensure that the +kubeconfig file used by antctl has the necessary permissions. The required permissions +are listed in the following sample ClusterRole. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: antctl +rules: + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions/status + verbs: + - update + - apiGroups: + - crd.antrea.io + resources: + - "*" + verbs: + - get + - list + - update +``` + +This command performs a dry-run to upgrade all existing objects of Antrea CRDs to +the storage version: + +```bash +antctl upgrade api-storage --dry-run +``` + +This command upgrades all existing objects of Antrea CRDs to the storage version: + +```bash +antctl upgrade api-storage +``` + +This command upgrades existing AntreaAgentInfo objects to the storage version: + +```bash +antctl upgrade api-storage --crds=antreaagentinfos.crd.antrea.io +``` + +This command upgrades existing Egress and Group objects to the storage version: + +```bash +antctl upgrade api-storage --crds=egresses.crd.antrea.io,groups.crd.antrea.io +``` + +If you encounter any errors related to permissions while running the commands, double-check +the permissions of the kubeconfig used by antctl. Ensure that the ClusterRole has the +required permissions. The following sample errors are caused by insufficient permissions: + +```bash +Error: failed to get CRD list: customresourcedefinitions.apiextensions.k8s.io is forbidden: User "user" cannot list resource "customresourcedefinitions" in API group "apiextensions.k8s.io" at the cluster scope + +Error: externalippools.crd.antrea.io is forbidden: User "user" cannot list resource "externalippools" in API group "crd.antrea.io" at the cluster scope + +Error: error upgrading object prod-external-ip-pool of CRD "externalippools.crd.antrea.io": externalippools.crd.antrea.io "prod-external-ip-pool" is forbidden: User "user" cannot update resource "externalippools" in API group "crd.antrea.io" at the cluster scope + +Error: error updating CRD "externalippools.crd.antrea.io" status.storedVersion: customresourcedefinitions.apiextensions.k8s.io "externalippools.crd.antrea.io" is forbidden: User "user" cannot update resource "customresourcedefinitions/status" in API group "apiextensions.k8s.io" at the cluster scope +``` diff --git a/pkg/antctl/antctl.go b/pkg/antctl/antctl.go index a1f9e53a971..2988c9b67f2 100644 --- a/pkg/antctl/antctl.go +++ b/pkg/antctl/antctl.go @@ -32,6 +32,7 @@ import ( "antrea.io/antrea/pkg/antctl/raw/set" "antrea.io/antrea/pkg/antctl/raw/supportbundle" "antrea.io/antrea/pkg/antctl/raw/traceflow" + "antrea.io/antrea/pkg/antctl/raw/upgrade/apistorage" "antrea.io/antrea/pkg/antctl/transform/addressgroup" "antrea.io/antrea/pkg/antctl/transform/appliedtogroup" "antrea.io/antrea/pkg/antctl/transform/controllerinfo" @@ -679,6 +680,12 @@ $ antctl get podmulticaststats pod -n namespace`, supportController: false, supportFlowAggregator: true, }, + { + cobraCommand: apistorage.NewCommand(), + supportAgent: false, + supportController: false, + commandGroup: upgrade, + }, }, codec: scheme.Codecs, } diff --git a/pkg/antctl/command_definition.go b/pkg/antctl/command_definition.go index 369564f36d5..1f6cc36665a 100644 --- a/pkg/antctl/command_definition.go +++ b/pkg/antctl/command_definition.go @@ -72,6 +72,7 @@ const ( get query mc + upgrade ) var groupCommands = map[commandGroup]*cobra.Command{ @@ -90,6 +91,11 @@ var groupCommands = map[commandGroup]*cobra.Command{ Short: "Sub-commands of multi-cluster feature", Long: "Sub-commands of multi-cluster feature", }, + upgrade: { + Use: "upgrade", + Short: "Sub-commands for upgrade operations", + Long: "Sub-commands for upgrade operations", + }, } type endpointResponder interface { diff --git a/pkg/antctl/command_list.go b/pkg/antctl/command_list.go index cd5b3e21815..e60f381e435 100644 --- a/pkg/antctl/command_list.go +++ b/pkg/antctl/command_list.go @@ -63,7 +63,8 @@ func (cl *commandList) applyToRootCommand(root *cobra.Command, client AntctlClie if (runtime.Mode == runtime.ModeAgent && cmd.supportAgent) || (runtime.Mode == runtime.ModeController && cmd.supportController) || (runtime.Mode == runtime.ModeFlowAggregator && cmd.supportFlowAggregator) || - (!runtime.InPod && cmd.commandGroup == mc) { + (!runtime.InPod && cmd.commandGroup == mc) || + (!runtime.InPod && cmd.commandGroup == upgrade) { if groupCommand, ok := groupCommands[cmd.commandGroup]; ok { groupCommand.AddCommand(cmd.cobraCommand) } else { diff --git a/pkg/antctl/raw/upgrade/apistorage/command.go b/pkg/antctl/raw/upgrade/apistorage/command.go new file mode 100644 index 00000000000..bd1b4ad3658 --- /dev/null +++ b/pkg/antctl/raw/upgrade/apistorage/command.go @@ -0,0 +1,273 @@ +// Copyright 2023 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apistorage + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/spf13/cobra" + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + "antrea.io/antrea/pkg/antctl/raw" + "antrea.io/antrea/pkg/util/k8s" +) + +var example = strings.Trim(` + Perform a dry-run to upgrade all existing objects of Antrea CRDs to the storage API version + $ antctl upgrade api-storage --dry-run + + Upgrade all existing objects of Antrea CRDs to the storage version + $ antctl upgrade api-storage + + Upgrade existing AntreaAgentInfo objects to the storage version + $ antctl upgrade api-storage --crds=antreaagentinfos.crd.antrea.io + + Upgrade existing Egress and Group objects to the storage version + $ antctl upgrade api-storage --crds=egresses.crd.antrea.io,groups.crd.antrea.io +`, "\n") + +type options struct { + k8sClient client.Client + crdNames []string + dryRun bool +} + +var opts *options + +func NewCommand() *cobra.Command { + command := &cobra.Command{ + Use: "api-storage", + Short: "Upgrade existing objects of Antrea CRDs to the storage version", + Example: example, + RunE: runE, + } + o := &options{} + command.Flags().StringSliceVar(&o.crdNames, "crds", nil, "Specify some Antrea CRDs to upgrade") + command.Flags().BoolVar(&o.dryRun, "dry-run", false, "Only print objects that would be upgraded") + opts = o + + return command +} + +func (o *options) complete(cmd *cobra.Command) error { + if o.k8sClient != nil { + return nil + } + + scheme := runtime.NewScheme() + apiextinstall.Install(scheme) + kubeconfig, err := raw.ResolveKubeconfig(cmd) + if err != nil { + return err + } + + o.k8sClient, err = client.New(kubeconfig, client.Options{Scheme: scheme}) + if err != nil { + return err + } + + if o.dryRun { + o.k8sClient = client.NewDryRunClient(o.k8sClient) + } + + return nil +} + +func runE(cmd *cobra.Command, _ []string) error { + if err := opts.complete(cmd); err != nil { + return err + } + + writer := cmd.ErrOrStderr() + crdNamesToUpgrade, err := getCRDNamesToUpgrade(writer, opts.k8sClient, sets.New[string](opts.crdNames...)) + if err != nil { + return err + } + + crdObjectsToUpgrade, err := getCRDsToUpgrade(writer, opts.k8sClient, crdNamesToUpgrade) + if err != nil { + return err + } + for _, crd := range crdObjectsToUpgrade { + if err = upgradeCRDObjects(writer, opts.k8sClient, crd); err != nil { + return err + } + if err = updateCRDStoredVersions(opts.k8sClient, crd); err != nil { + return err + } + } + + return nil +} + +// getAntreaCRDNames gets names of all Antrea CRDs. +func getAntreaCRDNames(k8sClient client.Client) (sets.Set[string], error) { + crdNames := sets.New[string]() + crdList := &apiextv1.CustomResourceDefinitionList{} + if err := k8sClient.List(context.TODO(), crdList); err != nil { + return nil, fmt.Errorf("failed to get CRD list: %v", err) + } + for _, crd := range crdList.Items { + if strings.HasSuffix(crd.Name, "crd.antrea.io") { + crdNames.Insert(crd.Name) + } + } + + return crdNames, nil +} + +// getCRDNamesToUpgrade gets names of Antrea CRDs to upgrade. +func getCRDNamesToUpgrade(writer io.Writer, k8sClient client.Client, crdNamesToUpgrade sets.Set[string]) (sets.Set[string], error) { + antreaCRDNames, err := getAntreaCRDNames(k8sClient) + if err != nil { + return nil, err + } + // If the user-provided name list of CRDs to upgrade is empty, upgrade all Antrea CRDs. + if crdNamesToUpgrade.Len() == 0 { + crdNamesToUpgrade = antreaCRDNames + } else { + // If the user-provided name list of CRDs to upgrade is not empty, and it contains CRDs without suffix + // "crd.antrea.io", skip these CRDs. + for name := range crdNamesToUpgrade.Difference(antreaCRDNames) { + fmt.Fprintf(writer, "Skip CRD %q which is not created by Antrea.\n", name) + } + // Only upgrade the CRDs with suffix "crd.antrea.io". + crdNamesToUpgrade = crdNamesToUpgrade.Intersection(antreaCRDNames) + } + return crdNamesToUpgrade, nil +} + +// getCRDsToUpgrade gets a list of Antrea CRDs to upgrade. +func getCRDsToUpgrade(writer io.Writer, k8sClient client.Client, crdNamesToUpgrade sets.Set[string]) ([]*apiextv1.CustomResourceDefinition, error) { + var crdsToUpgrade []*apiextv1.CustomResourceDefinition + for crdName := range crdNamesToUpgrade { + crd := &apiextv1.CustomResourceDefinition{} + if err := k8sClient.Get(context.TODO(), client.ObjectKey{Name: crdName}, crd); err != nil { + return nil, fmt.Errorf("error getting CRD %q: %w", crdName, err) + } + // Skip the CRD that has only one version. + if len(crd.Spec.Versions) == 1 { + fmt.Fprintf(writer, "Skip upgrading CRD %q since it only has one version.\n", crdName) + continue + } + // Skip the CRD that all stored objects are in the storage version. + if len(crd.Status.StoredVersions) == 1 && crd.Status.StoredVersions[0] == getCRDStorageVersion(crd) { + fmt.Fprintf(writer, "Skip upgrading CRD %q since all stored objects are in the storage version.\n", crdName) + continue + } + crdsToUpgrade = append(crdsToUpgrade, crd) + } + return crdsToUpgrade, nil +} + +// upgradeCRDObjects upgrades the existing objects of a CRD. +func upgradeCRDObjects(writer io.Writer, k8sClient client.Client, crd *apiextv1.CustomResourceDefinition) error { + objList := &unstructured.UnstructuredList{} + objList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: getCRDStorageVersion(crd), + Kind: crd.Spec.Names.ListKind, + }) + + if err := k8sClient.List(context.TODO(), objList); err != nil { + return err + } + + itemCount := len(objList.Items) + if itemCount == 0 { + return nil + } + + fmt.Fprintf(writer, "Upgrading %d objects of CRD %q.\n", itemCount, crd.Name) + for _, item := range objList.Items { + if err := upgradeCRDObject(writer, k8sClient, crd, item); err != nil { + itemNamespacedName := k8s.NamespacedName(item.GetNamespace(), item.GetName()) + if apierrors.IsNotFound(err) { + fmt.Fprintf(writer, "Skip upgrading object %s of CRD %q which is not found.\n", itemNamespacedName, crd.Name) + } else { + return fmt.Errorf("error upgrading object %s of CRD %q: %w", itemNamespacedName, crd.Name, err) + } + } + } + fmt.Fprintf(writer, "Successfully upgraded %d objects of CRD %q.\n", itemCount, crd.Name) + + return nil +} + +func upgradeCRDObject(writer io.Writer, k8sClient client.Client, crd *apiextv1.CustomResourceDefinition, obj unstructured.Unstructured) error { + objToUpdate := &obj + var updateErr, getErr error + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + updateErr = k8sClient.Update(context.TODO(), objToUpdate) + // If there is a conflict error, update pointer "objToUpdate" to retry. + if updateErr != nil && apierrors.IsConflict(updateErr) { + fmt.Fprintf(writer, "Got conflict error when upgrading object %s of CRD %q, retry upgrading.\n", k8s.NamespacedName(obj.GetNamespace(), obj.GetName()), crd.Name) + objToUpdate = &unstructured.Unstructured{} + objToUpdate.SetGroupVersionKind(schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: getCRDStorageVersion(crd), + Kind: crd.Spec.Names.Kind, + }) + if getErr = k8sClient.Get(context.TODO(), client.ObjectKeyFromObject(&obj), objToUpdate); getErr != nil { + return getErr + } + } + return updateErr + }) +} + +// updateCRDStoredVersions updates status.storedVersion of a CRD. +func updateCRDStoredVersions(k8sClient client.Client, crd *apiextv1.CustomResourceDefinition) error { + copiedCRD := *crd + copiedCRD.Status.StoredVersions = []string{getCRDStorageVersion(crd)} + if err := k8sClient.Status().Update(context.TODO(), &copiedCRD); err != nil { + if apierrors.IsConflict(err) { + return newUnexpectedChangeError(crd) + } + return fmt.Errorf("error updating CRD %q status.storedVersion: %w", crd.Name, err) + } + return nil +} + +func getCRDStorageVersion(crd *apiextv1.CustomResourceDefinition) string { + storageVersion := "" + for _, v := range crd.Spec.Versions { + if v.Storage { + storageVersion = v.Name + break + } + } + return storageVersion +} + +func newUnexpectedChangeError(crd *apiextv1.CustomResourceDefinition) error { + errorFmt := "The CRD %q unexpectedly changed during the upgrade. This means that either an object was persisted in a\n" + + "non-storage version, or the storage version was changed by someone else during the upgrade process.\n" + + "Please ensure that no changes to the CRDs are made during the upgrade process and re-run the command\n" + + "until you no longer see this message." + return fmt.Errorf(errorFmt, crd.Name) +} diff --git a/pkg/antctl/raw/upgrade/apistorage/command_test.go b/pkg/antctl/raw/upgrade/apistorage/command_test.go new file mode 100644 index 00000000000..73d9567e10c --- /dev/null +++ b/pkg/antctl/raw/upgrade/apistorage/command_test.go @@ -0,0 +1,178 @@ +// Copyright 2023 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apistorage + +import ( + "bytes" + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + antreaCRDAlfa = createMockCRD("Alfa", "AlfaList", "alfas", "alfa", "crd.antrea.io", map[string]bool{"v1alpha1": false, "v1beta1": true}, "999") + updatedAntreaCRDAlfa = createMockCRD("Alfa", "AlfaList", "alfas", "alfa", "crd.antrea.io", map[string]bool{"v1alpha1": false, "v1beta1": false, "v1": true}, "1000") + antreaCRDBravo = createMockCRD("Bravo", "BravoList", "bravos", "bravo", "crd.antrea.io", map[string]bool{"v1beta1": true}, "999") + nonAntreaCRDCharlie = createMockCRD("Charlie", "CharlieList", "charlies", "charlie", "crd.misc.io", map[string]bool{"v1beta1": true}, "999") + alfa1 = createMockCRDObject("crd.antrea.io/v1beta1", "Alfa", "alfa1", "999") + alfa2 = createMockCRDObject("crd.antrea.io/v1beta1", "Alfa", "alfa2", "999") + updatedAlfa2 = createMockCRDObject("crd.antrea.io/v1beta1", "Alfa", "alfa2", "1000") + bravo1 = createMockCRDObject("crd.antrea.io/v1beta1", "Bravo", "bravo1", "999") +) + +func createMockCRD(kind, listKind, plural, singular, group string, versions map[string]bool, resourceVersion string) *apiextv1.CustomResourceDefinition { + crd := &apiextv1.CustomResourceDefinition{} + var crdVersions []apiextv1.CustomResourceDefinitionVersion + var crdStoredVersions []string + for version, served := range versions { + crdVersions = append(crdVersions, apiextv1.CustomResourceDefinitionVersion{ + Name: version, + Served: served, + Storage: served, + }) + // Add every version to StoredVersions to mock that the CRD has objects stored in multiple versions. + crdStoredVersions = append(crdStoredVersions, version) + } + + crd.SetName(fmt.Sprintf("%s.%s", plural, group)) + crd.Spec = apiextv1.CustomResourceDefinitionSpec{ + Group: group, + Names: apiextv1.CustomResourceDefinitionNames{ + Kind: kind, + ListKind: listKind, + Plural: plural, + Singular: singular, + }, + Versions: crdVersions, + } + crd.Status.StoredVersions = crdStoredVersions + crd.ResourceVersion = resourceVersion + return crd +} + +func createMockCRDObject(apiVersion, kind, name, resourceVersion string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(apiVersion) + obj.SetNamespace("default") + obj.SetName(name) + obj.SetKind(kind) + obj.SetResourceVersion(resourceVersion) + return obj +} + +func TestCommands(t *testing.T) { + tests := []struct { + name string + crdsToUpgrade []string + originalCRDs []client.Object + expectedOutput string + }{ + { + name: "Upgrade all CRDs", + originalCRDs: []client.Object{antreaCRDAlfa, antreaCRDBravo, nonAntreaCRDCharlie, alfa1, alfa2, bravo1}, + expectedOutput: `Skip upgrading CRD "bravos.crd.antrea.io" since it only has one version. +Upgrading 2 objects of CRD "alfas.crd.antrea.io". +Successfully upgraded 2 objects of CRD "alfas.crd.antrea.io". +`, + }, + { + name: "Upgrade a CRD", + crdsToUpgrade: []string{"alfas.crd.antrea.io"}, + originalCRDs: []client.Object{antreaCRDAlfa, antreaCRDBravo, nonAntreaCRDCharlie, alfa1, alfa2, bravo1}, + expectedOutput: `Upgrading 2 objects of CRD "alfas.crd.antrea.io". +Successfully upgraded 2 objects of CRD "alfas.crd.antrea.io". +`, + }, + { + name: "Upgrade some CRDs", + crdsToUpgrade: []string{"alfas.crd.antrea.io", "bravos.crd.antrea.io"}, + originalCRDs: []client.Object{antreaCRDAlfa, antreaCRDBravo, nonAntreaCRDCharlie, alfa1, alfa2, bravo1}, + expectedOutput: `Skip upgrading CRD "bravos.crd.antrea.io" since it only has one version. +Upgrading 2 objects of CRD "alfas.crd.antrea.io". +Successfully upgraded 2 objects of CRD "alfas.crd.antrea.io". +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := new(bytes.Buffer) + cmd := NewCommand() + cmd.SetErr(buf) + + scheme := runtime.NewScheme() + apiextinstall.Install(scheme) + opts.k8sClient = fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(tt.originalCRDs...). + Build() + + args := []string{"--dry-run"} + if len(tt.crdsToUpgrade) != 0 { + args = append(args, fmt.Sprintf("--crds=%s", strings.Join(tt.crdsToUpgrade, ","))) + cmd.SetArgs(args) + } + + assert.NoError(t, cmd.Execute()) + assert.Equal(t, tt.expectedOutput, buf.String()) + }) + } +} + +func TestUpgradeCRDObject(t *testing.T) { + buf := new(bytes.Buffer) + cmd := NewCommand() + cmd.SetErr(buf) + + scheme := runtime.NewScheme() + apiextinstall.Install(scheme) + opts.k8sClient = fake.NewClientBuilder(). + WithObjects(updatedAlfa2). + WithScheme(scheme). + Build() + + assert.EqualError(t, upgradeCRDObject(cmd.ErrOrStderr(), opts.k8sClient, antreaCRDAlfa, *alfa1), `alfas.crd.antrea.io "alfa1" not found`) + assert.NoError(t, upgradeCRDObject(cmd.ErrOrStderr(), opts.k8sClient, antreaCRDAlfa, *alfa2)) + assert.Contains(t, buf.String(), `Got conflict error when upgrading object default/alfa2 of CRD "alfas.crd.antrea.io", retry upgrading.`) +} + +func TestUpdateCRDStoredVersions(t *testing.T) { + opts = &options{} + scheme := runtime.NewScheme() + apiextinstall.Install(scheme) + opts.k8sClient = fake.NewClientBuilder(). + WithObjects(updatedAntreaCRDAlfa, antreaCRDBravo). + WithScheme(scheme). + Build() + + assert.EqualError(t, updateCRDStoredVersions(opts.k8sClient, antreaCRDAlfa), `The CRD "alfas.crd.antrea.io" unexpectedly changed during the upgrade. This means that either an object was persisted in a +non-storage version, or the storage version was changed by someone else during the upgrade process. +Please ensure that no changes to the CRDs are made during the upgrade process and re-run the command +until you no longer see this message.`) + assert.NoError(t, updateCRDStoredVersions(opts.k8sClient, antreaCRDBravo)) + + upgradedCRDEcho := &apiextv1.CustomResourceDefinition{} + assert.NoError(t, opts.k8sClient.Get(context.TODO(), client.ObjectKey{Name: antreaCRDBravo.GetName()}, upgradedCRDEcho)) + assert.Equal(t, 1, len(upgradedCRDEcho.Status.StoredVersions)) + assert.Equal(t, "v1beta1", upgradedCRDEcho.Status.StoredVersions[0]) +}