Support upgrade of minor MySQL version

docs/release-notes/2_9_0_summary.md

@@ -9,4 +9,9 @@ This change of not using `enforceSemiSync` should be done before upgrading to `2
 ### VTOrc becomes mandatory
 
 VTOrc is now a **required** component of Vitess starting from v16. So, the vitess-operator will always run
-a VTOrc instance for a keyspace, even if its configuration is unspecified.
+a VTOrc instance for a keyspace, even if its configuration is unspecified.
+
+### MySQL Version Upgrade
+
+The Vitess-Operator will now support minor MySQL version upgrade. For instance from MySQL `8.0.23` to `8.0.31`.
+The global variable `innodb_fast_shutdown` will be set to `0` before shutting down the `mysqld` container.

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/ahmetb/gen-crd-api-reference-docs v0.1.5-0.20190629210212-52e137b8d003
 	github.com/planetscale/operator-sdk-libs v0.0.0-20220216002626-1af183733234
 	github.com/prometheus/client_golang v1.14.0
-	github.com/sirupsen/logrus v1.8.1
+	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.1
 	gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -675,8 +675,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/slok/noglog v0.2.0 h1:1czu4l2EoJ8L92UwdSXXa1Y+c5TIjFAFm2P+mjej95E=
 github.com/slok/noglog v0.2.0/go.mod h1:TfKxwpEZPT+UA83bQ6RME146k0MM4e8mwHLf6bhcGDI=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -1047,6 +1048,7 @@ golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

pkg/controller/vitessshard/reconcile_rollout.go

@@ -5,6 +5,8 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	apilabels "k8s.io/apimachinery/pkg/labels"
+	"planetscale.dev/vitess-operator/pkg/operator/drain"
+	"planetscale.dev/vitess-operator/pkg/operator/toposerver"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"vitess.io/vitess/go/vt/topo/topoproto"
@@ -51,8 +53,14 @@ func (r *ReconcileVitessShard) reconcileRollout(ctx context.Context, vts *planet
 		}
 	}
 
+	primaryAlias, err := getPrimaryTabletAlias(ctx, vts)
+	if err != nil {
+		r.recorder.Eventf(vts, corev1.EventTypeWarning, "RolloutBlocked", "Could not get TabletAlias for the Primary.")
+		return resultBuilder.Error(err)
+	}
+
 	// Retrieve tablet pod to be released during this reconcile.
-	tabletKey, pod := getNextScheduledTablet(tabletKeys, tabletPods)
+	tabletKey, pod := getNextScheduledTablet(tabletKeys, tabletPods, primaryAlias)
 	if tabletKey == "" {
 		// If we have no more scheduled tablets, uncascade the shard.
 		if err := r.uncascadeShard(ctx, vts); err != nil {
@@ -127,15 +135,51 @@ func (r *ReconcileVitessShard) uncascadeShard(ctx context.Context, vts *planetsc
 	return r.client.Update(ctx, vts)
 }
 
-func getNextScheduledTablet(tabletKeys []string, tabletPods map[string]*corev1.Pod) (string, *corev1.Pod) {
+func getNextScheduledTablet(tabletKeys []string, tabletPods map[string]*corev1.Pod, primaryAlias string) (string, *corev1.Pod) {
+	scheduledTablets := map[string]bool{}
+
 	for _, tabletKey := range tabletKeys {
 		pod := tabletPods[tabletKey]
-		if !rollout.Scheduled(pod) {
-			continue
+		if rollout.Scheduled(pod) {
+			scheduledTablets[tabletKey] = true
+
+			// If a Pod is scheduled for rollout and it's already drained
+			// then it's the next tablet to release since the drain controller
+			// will not drain any more tablets in the shard.
+			// A tablet may have been drained by something other than a rollout.
+			if drain.Finished(pod) {
+				return tabletKey, pod
+			}
 		}
+	}
+
+	// Release any scheduled tablet
+	for tabletKey := range scheduledTablets {
+		if tabletKey != primaryAlias {
+			return tabletKey, tabletPods[tabletKey]
+		}
+	}
 
-		return tabletKey, pod
+	// If there are no remaining scheduled tablets, then release the Primary if its scheduled
+	if _, scheduled := scheduledTablets[primaryAlias]; scheduled {
+		return primaryAlias, tabletPods[primaryAlias]
 	}
 
 	return "", nil
 }
+
+func getPrimaryTabletAlias(ctx context.Context, vts *planetscalev2.VitessShard) (string, error) {
+	ts, err := toposerver.Open(ctx, vts.Spec.GlobalLockserver)
+	if err != nil {
+		return "", err
+	}
+	defer ts.Close()
+
+	keyspaceName := vts.Labels[planetscalev2.KeyspaceLabel]
+	shard, err := ts.GetShard(ctx, keyspaceName, vts.Spec.Name)
+	if err != nil {
+		return "", err
+	}
+
+	return topoproto.TabletAliasString(shard.PrimaryAlias), nil
+}

pkg/operator/vttablet/pod.go

@@ -27,7 +27,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
 	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/topo/topoproto"
 
@@ -191,6 +190,7 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 			// TODO(enisoc): Add liveness probes that make sense for mysqld.
 			Env:          env,
 			VolumeMounts: mysqldMounts,
+			Lifecycle:    getMySQLdLifecycle(),
 		}
 
 		update.ResourceRequirements(&mysqldContainer.Resources, &spec.Mysqld.Resources)
@@ -370,6 +370,16 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) {
 	}
 }
 
+func getMySQLdLifecycle() *corev1.Lifecycle {
+	return &corev1.Lifecycle{
+		PreStop: &corev1.LifecycleHandler{
+			Exec: &corev1.ExecAction{
+				Command: []string{"mysql -S /vt/socket/mysql.sock -u root -e \"set global innodb_fast_shutdown=0\" || true"},
+			},
+		},
+	}
+}
+
 // AliasFromPod returns a TabletAlias corresponding to a vttablet Pod.
 func AliasFromPod(pod *corev1.Pod) topodatapb.TabletAlias {
 	uid, _ := strconv.ParseUint(pod.Labels[planetscalev2.TabletUidLabel], 10, 32)