tests: dump the kubeconfig for the testenv server

This is going to be useful to inspect the state of the cluster after a
test fails.

Set the env variable SKIP_CLEANUP to "true" or "1" to skip the cleanup
of the environment to make use of the kubeconfig.

The kubeconfig is dumped in the testbin directory.

After debugging you will have to kill the testEnv. The two binaries that
are used by the testEnv are etcd and kube-apiserver. Check for the
running process with a command like

ps aux | grep -e kube-apiserver -e etcd

Signed-off-by: Raghavendra Talur <[email protected]>
(cherry picked from commit 07a61b4)

internal/controller/suite_test.go

@@ -5,6 +5,7 @@ package controllers_test
 
 import (
 	"context"
+	"log"
 	"os"
 	"path/filepath"
 	"testing"
@@ -89,6 +90,8 @@ var (
 	objectStorers [2]ramencontrollers.ObjectStorer
 
 	ramenNamespace = "ns-envtest"
+
+	skipCleanup bool
 )
 
 func TestAPIs(t *testing.T) {
@@ -146,6 +149,11 @@ var _ = BeforeSuite(func() {
 		ramenNamespace = rNs
 	}
 
+	skipCleanupEnv, set := os.LookupEnv("SKIP_CLEANUP")
+	if set && (skipCleanupEnv == "true" || skipCleanupEnv == "1") {
+		skipCleanup = true
+	}
+
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
 		CRDDirectoryPaths: []string{
@@ -164,10 +172,25 @@ var _ = BeforeSuite(func() {
 		defer GinkgoRecover()
 		cfg, err = testEnv.Start()
 		DeferCleanup(func() error {
+			if skipCleanup {
+				By("skipping cleanup of the test environment")
+
+				return nil
+			}
 			By("tearing down the test environment")
 
 			return testEnv.Stop()
 		})
+		kubeConfigContent, err := ramencontrollers.ConvertRestConfigToKubeConfig(cfg)
+		if err != nil {
+			log.Fatalf("Failed to convert rest.Config to kubeconfig: %v", err)
+		}
+
+		filePath := "../../testbin/kubeconfig.yaml"
+		if err := ramencontrollers.WriteKubeConfigToFile(kubeConfigContent, filePath); err != nil {
+			log.Fatalf("Failed to write kubeconfig file: %v", err)
+		}
+
 		close(done)
 	}()
 	Eventually(done).WithTimeout(time.Minute).Should(BeClosed())
@@ -241,7 +264,15 @@ var _ = BeforeSuite(func() {
 	ramenConfig.DrClusterOperator.S3SecretDistributionEnabled = true
 	ramenConfig.MultiNamespace.FeatureEnabled = true
 	configMapCreate(ramenConfig)
-	DeferCleanup(configMapDelete)
+	DeferCleanup(func() error {
+		if skipCleanup {
+			By("skipping cleanup of the test environment")
+
+			return nil
+		}
+
+		return configMapDelete()
+	})
 
 	s3Secrets[0] = corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{Namespace: ramenNamespace, Name: "s3secret0"},
@@ -382,7 +413,18 @@ var _ = BeforeSuite(func() {
 	Expect(err).ToNot(HaveOccurred())
 
 	ctx, cancel = context.WithCancel(context.TODO())
-	DeferCleanup(cancel)
+	DeferCleanup(func() error {
+		if skipCleanup {
+			By("skipping cleanup of the test environment")
+
+			return nil
+		}
+
+		cancel()
+
+		return nil
+	})
+
 	go func() {
 		err = k8sManager.Start(ctx)
 		Expect(err).ToNot(HaveOccurred())

internal/controller/testutils.go

@@ -0,0 +1,44 @@
+package controllers
+
+import (
+	"os"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/clientcmd/api"
+)
+
+// WriteKubeConfigToFile writes the kubeconfig content to a specified file path.
+func WriteKubeConfigToFile(kubeConfigContent, filePath string) error {
+	return os.WriteFile(filePath, []byte(kubeConfigContent), 0o600)
+}
+
+// ConvertRestConfigToKubeConfig converts a rest.Config to a kubeconfig string.
+func ConvertRestConfigToKubeConfig(restConfig *rest.Config) (string, error) {
+	kubeConfig := api.NewConfig()
+
+	cluster := api.NewCluster()
+	cluster.Server = restConfig.Host
+	cluster.CertificateAuthorityData = restConfig.CAData
+
+	user := api.NewAuthInfo()
+	user.ClientCertificateData = restConfig.CertData
+	user.ClientKeyData = restConfig.KeyData
+	user.Token = restConfig.BearerToken
+
+	context := api.NewContext()
+	context.Cluster = "cluster"
+	context.AuthInfo = "user"
+
+	kubeConfig.Clusters["cluster"] = cluster
+	kubeConfig.AuthInfos["user"] = user
+	kubeConfig.Contexts["test"] = context
+	kubeConfig.CurrentContext = "test"
+
+	kubeConfigContent, err := clientcmd.Write(*kubeConfig)
+	if err != nil {
+		return "", err
+	}
+
+	return string(kubeConfigContent), nil
+}

internal/controller/volsync/volsync_suite_test.go

@@ -5,6 +5,7 @@ package volsync_test
 
 import (
 	"context"
+	"log"
 	"os"
 	"path/filepath"
 	"testing"
@@ -19,6 +20,7 @@ import (
 	storagev1 "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	cfgpolicyv1 "open-cluster-management.io/config-policy-controller/api/v1"
 	policyv1 "open-cluster-management.io/governance-policy-propagator/api/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -29,6 +31,7 @@ import (
 	metrics "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	volsyncv1alpha1 "github.com/backube/volsync/api/v1alpha1"
+	controllers "github.com/ramendr/ramen/internal/controller"
 	"github.com/ramendr/ramen/internal/controller/util"
 )
 
@@ -56,6 +59,9 @@ var (
 	volumeSnapshotClassB *snapv1.VolumeSnapshotClass
 
 	totalVolumeSnapshotClassCount = 0
+
+	skipCleanup bool
+	cfg         *rest.Config
 )
 
 func TestVolsync(t *testing.T) {
@@ -64,6 +70,8 @@ func TestVolsync(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
+	var err error
+
 	logger = zap.New(zap.UseFlagOptions(&zap.Options{
 		Development: true,
 		DestWriter:  GinkgoWriter,
@@ -85,6 +93,11 @@ var _ = BeforeSuite(func() {
 		Expect(os.Setenv("KUBEBUILDER_ASSETS", string(content))).To(Succeed())
 	}
 
+	skipCleanupEnv, set := os.LookupEnv("SKIP_CLEANUP")
+	if set && (skipCleanupEnv == "true" || skipCleanupEnv == "1") {
+		skipCleanup = true
+	}
+
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
 		CRDDirectoryPaths: []string{
@@ -93,7 +106,7 @@ var _ = BeforeSuite(func() {
 		},
 	}
 
-	cfg, err := testEnv.Start()
+	cfg, err = testEnv.Start()
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
 
@@ -217,6 +230,19 @@ var _ = BeforeSuite(func() {
 })
 
 var _ = AfterSuite(func() {
+	if skipCleanup {
+		kubeConfigContent, err := controllers.ConvertRestConfigToKubeConfig(cfg)
+		if err != nil {
+			log.Fatalf("Failed to convert rest.Config to kubeconfig: %v", err)
+		}
+
+		filePath := "../../../testbin/kubeconfig.yaml"
+		if err := controllers.WriteKubeConfigToFile(kubeConfigContent, filePath); err != nil {
+			log.Fatalf("Failed to write kubeconfig file: %v", err)
+		}
+
+		return
+	}
 	cancel()
 	By("tearing down the test environment")
 	err := testEnv.Stop()