compact: retry on cleanPartialMarked errors if possible (thanos-io#5922)

cleanPartialMarked is calling SyncMetas which basically can have retriable errors. By checking for retriable errors and retrying, it can prevent the compact from shutdown the HTTP server. Signed-off-by: Seena Fallah <[email protected]> Signed-off-by: Seena Fallah <[email protected]>
ngraham20 · May 18, 2023 · 3ed8bc1 · 3ed8bc1
1 parent 30664d5
commit 3ed8bc1
Showing 1 changed file with 13 additions and 1 deletion.
diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go
@@ -557,7 +557,19 @@ func runCompact(
 		// since one iteration potentially could take a long time.
 		if conf.cleanupBlocksInterval > 0 {
 			g.Add(func() error {
-				return runutil.Repeat(conf.cleanupBlocksInterval, ctx.Done(), cleanPartialMarked)
+				return runutil.Repeat(conf.cleanupBlocksInterval, ctx.Done(), func() error {
+					err := cleanPartialMarked()
+					if err != nil && compact.IsRetryError(err) {
+						// The RetryError signals that we hit an retriable error (transient error, no connection).
+						// You should alert on this being triggered too frequently.
+						level.Error(logger).Log("msg", "retriable error", "err", err)
+						compactMetrics.retried.Inc()
+
+						return nil
+					}
+
+					return err
+				})
 			}, func(error) {
 				cancel()
 			})