Add retry mechanism while ContainedInspectContainer return NotFound

tkestack · May 17, 2024 · baff523 · baff523
1 parent 7e70362
commit baff523
Showing 1 changed file with 47 additions and 47 deletions.
diff --git a/pkg/gc/flannel_gc.go b/pkg/gc/flannel_gc.go
@@ -106,7 +106,7 @@ func (gc *flannelGC) Run() {
 		if err := gc.cleanupVeth(); err != nil {
 			glog.Errorf("failed cleanup links: %v", err)
 		}
-	}, *flagFlannelGCInterval*3, gc.quit)
+	}, *flagFlannelGCInterval*2, gc.quit)
 }
 
 func (gc *flannelGC) cleanupIP() error {
@@ -195,12 +195,12 @@ func (gc *flannelGC) cleanupVeth() error {
 }
 
 func (gc *flannelGC) shouldCleanup(cid string) bool {
-    if os.Getenv("CONTAINERD_HOST") != "" {
-        if c, err := gc.dockerCli.ContainedInspectContainer(cid); err != nil {
-            if stausErr, ok := status.FromError(err); ok {
-                if stausErr.Code() == codes.NotFound {
-                    glog.Infof("container %s not found once", cid)
-					// time.Sleep(3 * time.Second)
+	if os.Getenv("CONTAINERD_HOST") != "" {
+		if c, err := gc.dockerCli.ContainedInspectContainer(cid); err != nil {
+			if stausErr, ok := status.FromError(err); ok {
+				if stausErr.Code() == codes.NotFound {
+					glog.Infof("container %s not found once", cid)
+					time.Sleep(3 * time.Second)
 					if _, err := gc.dockerCli.ContainedInspectContainer(cid); err != nil {
 						if stausErr, ok := status.FromError(err); ok {
 							if stausErr.Code() == codes.NotFound {
@@ -209,46 +209,46 @@ func (gc *flannelGC) shouldCleanup(cid string) bool {
 							}
 						}
 					}
-                }
-                glog.Warningf("Error inspect container %s: %v", cid, err)
-            } else {
-                glog.Warningf("Error inspect container %s: %v", cid, err)
-            }
-        } else {
-            if c != nil && (c.State == criapi.PodSandboxState_SANDBOX_NOTREADY) {
-                pod, err := gc.kubeCli.CoreV1().Pods(c.Annotations[SandboxNamespace]).Get(context.Background(), c.Annotations[SandboxName], metav1.GetOptions{})
-                if err != nil {
-                    if apierrors.IsNotFound(err) {
-                        return true
-                    }
-                    glog.Errorf("failed to get pod %s", fmt.Sprintf("%s/%s", c.Annotations[SandboxNamespace], c.Annotations[SandboxName]))
-                    return false
-                }
-                for _, status := range pod.Status.ContainerStatuses {
-                    if status.State.Waiting != nil || status.State.Running != nil {
-                        return false
-                    }
-                }
-                glog.Infof("container %s exited %s", c.Id, c.State.String())
-                return true
-            }
-        }
-        return false
-    }
-    if c, err := gc.dockerCli.DockerInspectContainer(cid); err != nil {
-        if _, ok := err.(docker.ContainerNotFoundError); ok {
-            glog.Infof("container %s not found", cid)
-            return true
-        } else {
-            glog.Warningf("Error inspect container %s: %v", cid, err)
-        }
-    } else {
-        if c.State != nil && (c.State.Status == ContainerExited || c.State.Status == ContainerDead) {
-            glog.Infof("container %s(%s) exited %s", c.ID, c.Name, c.State.Status)
-            return true
-        }
-    }
-    return false
+				}
+				glog.Warningf("Error inspect container %s: %v", cid, err)
+			} else {
+				glog.Warningf("Error inspect container %s: %v", cid, err)
+			}
+		} else {
+			if c != nil && (c.State == criapi.PodSandboxState_SANDBOX_NOTREADY) {
+				pod, err := gc.kubeCli.CoreV1().Pods(c.Annotations[SandboxNamespace]).Get(context.Background(), c.Annotations[SandboxName], metav1.GetOptions{})
+				if err != nil {
+					if apierrors.IsNotFound(err) {
+						return true
+					}
+					glog.Errorf("failed to get pod %s", fmt.Sprintf("%s/%s", c.Annotations[SandboxNamespace], c.Annotations[SandboxName]))
+					return false
+				}
+				for _, status := range pod.Status.ContainerStatuses {
+					if status.State.Waiting != nil || status.State.Running != nil {
+						return false
+					}
+				}
+				glog.Infof("container %s exited %s", c.Id, c.State.String())
+				return true
+			}
+		}
+		return false
+	}
+	if c, err := gc.dockerCli.DockerInspectContainer(cid); err != nil {
+		if _, ok := err.(docker.ContainerNotFoundError); ok {
+			glog.Infof("container %s not found", cid)
+			return true
+		} else {
+			glog.Warningf("Error inspect container %s: %v", cid, err)
+		}
+	} else {
+		if c.State != nil && (c.State.Status == ContainerExited || c.State.Status == ContainerDead) {
+			glog.Infof("container %s(%s) exited %s", c.ID, c.Name, c.State.Status)
+			return true
+		}
+	}
+	return false
 }
 
 func removeLeakyIPFile(ipFile, containerId string) {