Merge pull request #41 from ucloud/v0.2.3

V0.2.3

deploy/cluster/cluster_role.yaml

@@ -19,6 +19,7 @@ rules:
       - ""
     resources:
       - configmaps
+      - pods/exec
       - secrets
       - services
       - events

deploy/namespace/role.yaml

@@ -19,6 +19,7 @@ rules:
       - ""
     resources:
       - configmaps
+      - pods/exec
       - secrets
       - services
       - events

go.sum

@@ -203,6 +203,7 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docker/libnetwork v0.0.0-20180830151422-a9cd636e3789/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
 github.com/docker/libtrust v0.0.0-20150526203908-9cbd2a1374f4/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s=
 github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -211,7 +212,9 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f h1:8GDPb0tCY8LQ+OJ3dbHb5sA6YZWXFORQYZx5sdsTlMs=
 github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f h1:AUj1VoZUfhPhOPHULCQQDnGhRelpFWHMLhQVWDsS0v4=
 github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.3+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=

hack/e2e.sh

@@ -24,6 +24,8 @@ if [[ -z $TEST_TIMEOUT ]]; then
 fi
 
 echo "run e2e tests..."
-echo "cd ${GOPATH}/src/${REPO_PATH} && ginkgo -v --skip=${GINKGO_SKIP} --timeout=${TEST_TIMEOUT} test/e2e/... -- --rename-command-path=${GOPATH}/src/${REPO_PATH}/test/e2e --rename-command-file=rename.conf"
-cd ${GOPATH}/src/${REPO_PATH} && ginkgo -v --skip=${GINKGO_SKIP} --timeout=${TEST_TIMEOUT} test/e2e/... -- --rename-command-path=${GOPATH}/src/${REPO_PATH}/test/e2e --rename-command-file=rename.conf
+e2ecmd="cd ${GOPATH}/src/${REPO_PATH} && ginkgo -v --mod=vendor --failFast --skip=${GINKGO_SKIP} --timeout=${TEST_TIMEOUT} test/e2e/... -- --rename-command-path=${GOPATH}/src/${REPO_PATH}/test/e2e --rename-command-file=rename.conf"
+echo "${e2ecmd}"
+eval "${e2ecmd}"
+
 

pkg/apis/redis/v1alpha1/constants.go

@@ -45,6 +45,8 @@ const (
 	ClusterStatusRebalancing ClusterStatus = "Rebalancing"
 	// ClusterStatusRollingUpdate ClusterStatus RollingUpdate
 	ClusterStatusRollingUpdate ClusterStatus = "RollingUpdate"
+	// ClusterStatusResetPassword ClusterStatus ResetPassword
+	ClusterStatusResetPassword ClusterStatus = "ResetPassword"
 )
 
 // NodesPlacementInfo Redis Nodes placement mode information

pkg/apis/redis/v1alpha1/distributedrediscluster_types.go

@@ -15,22 +15,25 @@ type DistributedRedisClusterSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file
 	// Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html
-	Image           string                       `json:"image,omitempty"`
-	Command         []string                     `json:"command,omitempty"`
-	MasterSize      int32                        `json:"masterSize,omitempty"`
-	ClusterReplicas int32                        `json:"clusterReplicas,omitempty"`
-	ServiceName     string                       `json:"serviceName,omitempty"`
-	Config          map[string]string            `json:"config,omitempty"`
-	Affinity        *corev1.Affinity             `json:"affinity,omitempty"`
-	NodeSelector    map[string]string            `json:"nodeSelector,omitempty"`
-	ToleRations     []corev1.Toleration          `json:"toleRations,omitempty"`
-	SecurityContext *corev1.PodSecurityContext   `json:"securityContext,omitempty"`
-	Annotations     map[string]string            `json:"annotations,omitempty"`
-	Storage         *RedisStorage                `json:"storage,omitempty"`
-	Resources       *corev1.ResourceRequirements `json:"resources,omitempty"`
-	PasswordSecret  *corev1.LocalObjectReference `json:"passwordSecret,omitempty"`
-	Monitor         *AgentSpec                   `json:"monitor,omitempty"`
-	Init            *InitSpec                    `json:"init,omitempty"`
+	Image            string                        `json:"image,omitempty"`
+	ImagePullPolicy  corev1.PullPolicy             `json:"imagePullPolicy,omitempty"`
+	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
+	Command          []string                      `json:"command,omitempty"`
+	Env              []corev1.EnvVar               `json:"env,omitempty"`
+	MasterSize       int32                         `json:"masterSize,omitempty"`
+	ClusterReplicas  int32                         `json:"clusterReplicas,omitempty"`
+	ServiceName      string                        `json:"serviceName,omitempty"`
+	Config           map[string]string             `json:"config,omitempty"`
+	Affinity         *corev1.Affinity              `json:"affinity,omitempty"`
+	NodeSelector     map[string]string             `json:"nodeSelector,omitempty"`
+	ToleRations      []corev1.Toleration           `json:"toleRations,omitempty"`
+	SecurityContext  *corev1.PodSecurityContext    `json:"securityContext,omitempty"`
+	Annotations      map[string]string             `json:"annotations,omitempty"`
+	Storage          *RedisStorage                 `json:"storage,omitempty"`
+	Resources        *corev1.ResourceRequirements  `json:"resources,omitempty"`
+	PasswordSecret   *corev1.LocalObjectReference  `json:"passwordSecret,omitempty"`
+	Monitor          *AgentSpec                    `json:"monitor,omitempty"`
+	Init             *InitSpec                     `json:"init,omitempty"`
 }
 
 type AgentSpec struct {

pkg/apis/redis/v1alpha1/redisclusterbackup_types.go

@@ -16,11 +16,12 @@ const (
 // RedisClusterBackupSpec defines the desired state of RedisClusterBackup
 // +k8s:openapi-gen=true
 type RedisClusterBackupSpec struct {
-	Image            string        `json:"image,omitempty"`
-	RedisClusterName string        `json:"redisClusterName"`
-	Storage          *RedisStorage `json:"storage,omitempty"`
-	store.Backend    `json:",inline"`
-	PodSpec          *PodSpec `json:"podSpec,omitempty"`
+	Image                 string        `json:"image,omitempty"`
+	RedisClusterName      string        `json:"redisClusterName"`
+	Storage               *RedisStorage `json:"storage,omitempty"`
+	store.Backend         `json:",inline"`
+	PodSpec               *PodSpec `json:"podSpec,omitempty"`
+	ActiveDeadlineSeconds *int64   `json:"activeDeadlineSeconds,omitempty"`
 }
 
 type PodSpec struct {

pkg/controller/distributedrediscluster/distributedrediscluster_controller.go

@@ -8,7 +8,11 @@ import (
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
+	runtimeschema "k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -22,8 +26,10 @@ import (
 	"github.com/ucloud/redis-cluster-operator/pkg/config"
 	"github.com/ucloud/redis-cluster-operator/pkg/controller/heal"
 	clustermanger "github.com/ucloud/redis-cluster-operator/pkg/controller/manager"
+	"github.com/ucloud/redis-cluster-operator/pkg/exec"
 	"github.com/ucloud/redis-cluster-operator/pkg/k8sutil"
 	"github.com/ucloud/redis-cluster-operator/pkg/redisutil"
+	"github.com/ucloud/redis-cluster-operator/pkg/resources/statefulsets"
 	"github.com/ucloud/redis-cluster-operator/pkg/utils"
 )
 
@@ -55,11 +61,22 @@ func FlagSet() *pflag.FlagSet {
 // Add creates a new DistributedRedisCluster Controller and adds it to the Manager. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
 func Add(mgr manager.Manager) error {
-	return add(mgr, newReconciler(mgr))
+	gvk := runtimeschema.GroupVersionKind{
+		Group:   "",
+		Version: "v1",
+		Kind:    "Pod",
+	}
+	restClient, err := apiutil.RESTClientForGVK(gvk, mgr.GetConfig(), serializer.NewCodecFactory(scheme.Scheme))
+	if err != nil {
+		return err
+	}
+	execer := exec.NewRemoteExec(restClient, mgr.GetConfig(), log)
+
+	return add(mgr, newReconciler(mgr, execer))
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, execer exec.IExec) reconcile.Reconciler {
 	reconiler := &ReconcileDistributedRedisCluster{client: mgr.GetClient(), scheme: mgr.GetScheme()}
 	reconiler.statefulSetController = k8sutil.NewStatefulSetController(reconiler.client)
 	reconiler.serviceController = k8sutil.NewServiceController(reconiler.client)
@@ -68,6 +85,7 @@ func newReconciler(mgr manager.Manager) reconcile.Reconciler {
 	reconiler.crController = k8sutil.NewCRControl(reconiler.client)
 	reconiler.ensurer = clustermanger.NewEnsureResource(reconiler.client, log)
 	reconiler.checker = clustermanger.NewCheck(reconiler.client)
+	reconiler.execer = execer
 	return reconiler
 }
 
@@ -136,6 +154,7 @@ type ReconcileDistributedRedisCluster struct {
 	scheme                *runtime.Scheme
 	ensurer               clustermanger.IEnsureResource
 	checker               clustermanger.ICheck
+	execer                exec.IExec
 	statefulSetController k8sutil.IStatefulSetControl
 	serviceController     k8sutil.IServiceControl
 	pdbController         k8sutil.IPodDisruptionBudgetControl
@@ -208,7 +227,7 @@ func (r *ReconcileDistributedRedisCluster) Reconcile(request reconcile.Request)
 		return reconcile.Result{RequeueAfter: requeueAfter}, nil
 	}
 
-	password, err := getClusterPassword(r.client, instance)
+	password, err := statefulsets.GetClusterPassword(r.client, instance)
 	if err != nil {
 		return reconcile.Result{}, Kubernetes.Wrap(err, "getClusterPassword")
 	}

pkg/controller/distributedrediscluster/helper.go

@@ -1,15 +1,12 @@
 package distributedrediscluster
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"time"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	redisv1alpha1 "github.com/ucloud/redis-cluster-operator/pkg/apis/redis/v1alpha1"
 	"github.com/ucloud/redis-cluster-operator/pkg/config"
@@ -24,30 +21,13 @@ var (
 	}
 )
 
-const passwordKey = "password"
-
 func getLabels(cluster *redisv1alpha1.DistributedRedisCluster) map[string]string {
 	dynLabels := map[string]string{
 		redisv1alpha1.LabelClusterName: cluster.Name,
 	}
 	return utils.MergeLabels(defaultLabels, dynLabels, cluster.Labels)
 }
 
-func getClusterPassword(client client.Client, cluster *redisv1alpha1.DistributedRedisCluster) (string, error) {
-	if cluster.Spec.PasswordSecret == nil {
-		return "", nil
-	}
-	secret := &corev1.Secret{}
-	err := client.Get(context.TODO(), types.NamespacedName{
-		Name:      cluster.Spec.PasswordSecret.Name,
-		Namespace: cluster.Namespace,
-	}, secret)
-	if err != nil {
-		return "", err
-	}
-	return string(secret.Data[passwordKey]), nil
-}
-
 // newRedisAdmin builds and returns new redis.Admin from the list of pods
 func newRedisAdmin(pods []*corev1.Pod, password string, cfg *config.Redis, reqLogger logr.Logger) (redisutil.IAdmin, error) {
 	nodesAddrs := []string{}

pkg/controller/distributedrediscluster/status.go

@@ -38,6 +38,11 @@ func SetClusterUpdating(status *redisv1alpha1.DistributedRedisClusterStatus, rea
 	status.Reason = reason
 }
 
+func SetClusterResetPassword(status *redisv1alpha1.DistributedRedisClusterStatus, reason string) {
+	status.Status = redisv1alpha1.ClusterStatusResetPassword
+	status.Reason = reason
+}
+
 func buildClusterStatus(clusterInfos *redisutil.ClusterInfos, pods []*corev1.Pod,
 	cluster *redisv1alpha1.DistributedRedisCluster, reqLogger logr.Logger) *redisv1alpha1.DistributedRedisClusterStatus {
 	oldStatus := cluster.Status

pkg/controller/distributedrediscluster/sync_handler.go

@@ -8,6 +8,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 
 	redisv1alpha1 "github.com/ucloud/redis-cluster-operator/pkg/apis/redis/v1alpha1"
+	"github.com/ucloud/redis-cluster-operator/pkg/config"
 	"github.com/ucloud/redis-cluster-operator/pkg/controller/clustering"
 	"github.com/ucloud/redis-cluster-operator/pkg/controller/manager"
 	"github.com/ucloud/redis-cluster-operator/pkg/k8sutil"
@@ -45,6 +46,11 @@ func (r *ReconcileDistributedRedisCluster) ensureCluster(ctx *syncContext) error
 	if err := r.ensurer.EnsureRedisConfigMap(cluster, labels); err != nil {
 		return Kubernetes.Wrap(err, "EnsureRedisConfigMap")
 	}
+
+	if err := r.resetClusterPassword(ctx); err != nil {
+		return Cluster.Wrap(err, "ResetPassword")
+	}
+
 	if updated, err := r.ensurer.EnsureRedisStatefulsets(cluster, labels); err != nil {
 		ctx.reqLogger.Error(err, "EnsureRedisStatefulSets")
 		return Kubernetes.Wrap(err, "EnsureRedisStatefulSets")
@@ -306,3 +312,57 @@ func (r *ReconcileDistributedRedisCluster) scalingDown(ctx *syncContext, current
 	}
 	return nil
 }
+
+func (r *ReconcileDistributedRedisCluster) resetClusterPassword(ctx *syncContext) error {
+	if err := r.checker.CheckRedisNodeNum(ctx.cluster); err == nil {
+		namespace := ctx.cluster.Namespace
+		name := ctx.cluster.Name
+		sts, err := r.statefulSetController.GetStatefulSet(namespace, statefulsets.ClusterStatefulSetName(name, 0))
+		if err != nil {
+			return err
+		}
+
+		if !statefulsets.IsPasswordChanged(ctx.cluster, sts) {
+			return nil
+		}
+
+		SetClusterResetPassword(&ctx.cluster.Status, "updating cluster's password")
+		r.crController.UpdateCRStatus(ctx.cluster)
+
+		matchLabels := getLabels(ctx.cluster)
+		redisClusterPods, err := r.statefulSetController.GetStatefulSetPodsByLabels(namespace, matchLabels)
+		if err != nil {
+			return err
+		}
+
+		oldPassword, err := statefulsets.GetOldRedisClusterPassword(r.client, sts)
+		if err != nil {
+			return err
+		}
+
+		newPassword, err := statefulsets.GetClusterPassword(r.client, ctx.cluster)
+		if err != nil {
+			return err
+		}
+
+		podSet := clusterPods(redisClusterPods.Items)
+		admin, err := newRedisAdmin(podSet, oldPassword, config.RedisConf(), ctx.reqLogger)
+		if err != nil {
+			return err
+		}
+		defer admin.Close()
+
+		// Update the password recorded in the file /etc/redis_password, redis pod preStop hook
+		// need /etc/redis_password do CLUSTER FAILOVER
+		cmd := fmt.Sprintf("echo %s > /etc/redis_password", newPassword)
+		if err := r.execer.ExecCommandInPodSet(podSet, "/bin/sh", "-c", cmd); err != nil {
+			return err
+		}
+
+		// Reset all redis pod's password.
+		if err := admin.ResetPassword(newPassword); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/controller/manager/checker.go

@@ -49,6 +49,9 @@ func (c *realCheck) checkRedisNodeNum(expectNodeNum int32, ss *appsv1.StatefulSe
 	if expectNodeNum != ss.Status.ReadyReplicas {
 		return fmt.Errorf("redis pods are not all ready")
 	}
+	if expectNodeNum != ss.Status.CurrentReplicas {
+		return fmt.Errorf("redis pods need to be updated")
+	}
 
 	return nil
 }

pkg/controller/manager/ensurer.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/go-logr/logr"
 	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -101,15 +100,9 @@ func shouldUpdateRedis(cluster *redisv1alpha1.DistributedRedisCluster, sts *apps
 	if cluster.Spec.Image != sts.Spec.Template.Spec.Containers[0].Image {
 		return true
 	}
-	if cluster.Spec.PasswordSecret != nil {
-		envSet := sts.Spec.Template.Spec.Containers[0].Env
-		secretName := getSecretKeyRefByKey(redisv1alpha1.PasswordENV, envSet)
-		if secretName == "" {
-			return true
-		}
-		if secretName != cluster.Spec.PasswordSecret.Name {
-			return true
-		}
+
+	if statefulsets.IsPasswordChanged(cluster, sts) {
+		return true
 	}
 
 	expectResource := cluster.Spec.Resources
@@ -129,17 +122,6 @@ func shouldUpdateRedis(cluster *redisv1alpha1.DistributedRedisCluster, sts *apps
 	return false
 }
 
-func getSecretKeyRefByKey(key string, envSet []corev1.EnvVar) string {
-	for _, value := range envSet {
-		if key == value.Name {
-			if value.ValueFrom != nil && value.ValueFrom.SecretKeyRef != nil {
-				return value.ValueFrom.SecretKeyRef.Name
-			}
-		}
-	}
-	return ""
-}
-
 func (r *realEnsureResource) ensureRedisPDB(cluster *redisv1alpha1.DistributedRedisCluster, name string, labels map[string]string) error {
 	_, err := r.pdbClient.GetPodDisruptionBudget(cluster.Namespace, name)
 	if err != nil && errors.IsNotFound(err) {

pkg/controller/redisclusterbackup/sync_handler.go

@@ -243,6 +243,7 @@ func (r *ReconcileRedisClusterBackup) getBackupJob(reqLogger logr.Logger, backup
 			},
 		},
 		Spec: batchv1.JobSpec{
+			ActiveDeadlineSeconds: backup.Spec.ActiveDeadlineSeconds,
 			Template: corev1.PodTemplateSpec{
 				Spec: corev1.PodSpec{
 					Containers: containers,