try to Graceful Drain TiCDC for improper constraint image tag

pingcap · Jul 14, 2023 · 6c4d0e3 · 6c4d0e3
1 parent df734da
commit 6c4d0e3
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 8 deletions.
diff --git a/pkg/controller/ticdc_control.go b/pkg/controller/ticdc_control.go
@@ -115,8 +115,9 @@ func (c *defaultTiCDCControl) DrainCapture(tc *v1alpha1.TidbCluster, ordinal int
 		// Let caller retry drain capture.
 		return 0, true, nil
 	}
-	if len(captures) == 1 {
+	if len(captures) == 1 || len(captures) == 0 {
 		// No way to drain a single node TiCDC cluster, ignore.
+		// Can't get capture info, ignore.
 		return 0, false, nil
 	}
 
@@ -186,8 +187,9 @@ func (c *defaultTiCDCControl) ResignOwner(tc *v1alpha1.TidbCluster, ordinal int3
 		// Let caller retry resign owner.
 		return false, nil
 	}
-	if len(captures) == 1 {
+	if len(captures) == 1 || len(captures) == 0 {
 		// No way to resign owner in a single node TiCDC cluster, ignore.
+		// Can't get capture info, ignore.
 		return true, nil
 	}
 
@@ -237,6 +239,10 @@ func (c *defaultTiCDCControl) IsHealthy(tc *v1alpha1.TidbCluster, ordinal int32)
 		// Let caller retry.
 		return false, nil
 	}
+	if len(captures) == 0 {
+		// No way to get capture info, ignore.
+		return true, nil
+	}
 
 	_, owner := getOrdinalAndOwnerCaptureInfo(tc, ordinal, captures)
 	if owner == nil {

diff --git a/pkg/manager/member/ticdc_scaler.go b/pkg/manager/member/ticdc_scaler.go
@@ -235,15 +235,17 @@ func isTiCDCPodSupportGracefulUpgrade(
 	currentVersion := tc.TiCDCVersion()
 	ge, err := cmpver.Compare(podVersion, cmpver.GreaterOrEqual, currentVersion)
 	if err != nil {
-		klog.Errorf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
+		klog.Warningf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
 			action, podVersion, currentVersion, err)
-		return false, nil
+		// if parse version failed, we still try to graftfully shutdown TiCDC,
+		// as we already checked `http.StatusNotFound` in `DrainCapture`, `ResignOwner` and `IsHealthy`
+		return true, nil
 	}
 	le, err := cmpver.Compare(podVersion, cmpver.LessOrEqual, currentVersion)
 	if err != nil {
-		klog.Errorf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
+		klog.Warningf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
 			action, podVersion, currentVersion, err)
-		return false, nil
+		return true, nil
 	}
 	// Reload TiCDC if the current version matches pod version.
 	if ge && le {
@@ -252,7 +254,7 @@ func isTiCDCPodSupportGracefulUpgrade(
 	// We are performing cross version upgrade.
 	lessThan63, err := cmpver.Compare(podVersion, cmpver.Less, ticdcCrossUpgradeVersion)
 	if err != nil {
-		klog.Errorf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
+		klog.Warningf("ticdc.%s: fail to compare TiCDC pod version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
 			action, podVersion, ticdcCrossUpgradeVersion, err)
 		return false, nil
 	}
@@ -271,7 +273,7 @@ func isTiCDCPodSupportGracefulUpgrade(
 	podVersionPlus2 := podVer.IncMajor().IncMajor()
 	withInTwoMajorVersion, err := cmpver.Compare(currentVersion, cmpver.Less, podVersionPlus2.String())
 	if err != nil {
-		klog.Errorf("ticdc.%s: fail to compare version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
+		klog.Warningf("ticdc.%s: fail to compare version \"%s\", version \"%s\", error: %v, skip graceful shutdown",
 			action, currentVersion, ticdcCrossUpgradeVersion, err)
 		return false, nil
 	}