From afe19c032861fa6f00b2401fe98d2d9975ad459a Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Wed, 27 Mar 2019 14:52:22 +0800 Subject: [PATCH 1/3] bugfix: dead loop in AbstractRegistry when IOException. fix #3746 --- .../org/apache/dubbo/registry/support/AbstractRegistry.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java index 31cdbf8d467..309e160a39b 100644 --- a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java +++ b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java @@ -174,6 +174,10 @@ public void doSaveProperties(long version) { } } } catch (Throwable e) { + if (e instanceof IOException) { // may not be recoverable when IOException throws, give up retrying + logger.warn("Failed to save registry cache file, cause: " + e.getMessage(), e); + return; + } if (version < lastCacheChanged.get()) { return; } else { From 2a22ca8e380fb4bff0fb23b32840750eebdf2aa3 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Thu, 28 Mar 2019 10:11:48 +0800 Subject: [PATCH 2/3] limit retry times to save properties to local file --- .../dubbo/registry/support/AbstractRegistry.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java index 309e160a39b..f99c2522324 100644 --- a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java +++ b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java @@ -48,6 +48,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -60,6 +61,8 @@ public abstract class AbstractRegistry implements Registry { private static final char URL_SEPARATOR = ' '; // URL address separated regular expression for parsing the service provider URL list in the file cache private static final String URL_SPLIT = "\\s+"; + // Max times to retry to save properties to local cache file + private static final int MAX_RETRY_TIMES_SAVE_PROPERTIES = 3; // Log output protected final Logger logger = LoggerFactory.getLogger(getClass()); // Local disk cache, where the special key value.registries records the list of registry centers, and the others are the list of notified service providers @@ -69,6 +72,7 @@ public abstract class AbstractRegistry implements Registry { // Is it synchronized to save the file private final boolean syncSaveFile; private final AtomicLong lastCacheChanged = new AtomicLong(); + private final AtomicInteger savePropertiesRetryTimes = new AtomicInteger(); private final Set registered = new ConcurrentHashSet<>(); private final ConcurrentMap> subscribed = new ConcurrentHashMap<>(); private final ConcurrentMap>> notified = new ConcurrentHashMap<>(); @@ -174,8 +178,10 @@ public void doSaveProperties(long version) { } } } catch (Throwable e) { - if (e instanceof IOException) { // may not be recoverable when IOException throws, give up retrying - logger.warn("Failed to save registry cache file, cause: " + e.getMessage(), e); + savePropertiesRetryTimes.incrementAndGet(); + if (savePropertiesRetryTimes.get() >= MAX_RETRY_TIMES_SAVE_PROPERTIES) { + logger.warn("Failed to save registry cache file after retrying " + MAX_RETRY_TIMES_SAVE_PROPERTIES + " times, cause: " + e.getMessage(), e); + savePropertiesRetryTimes.set(0); return; } if (version < lastCacheChanged.get()) { @@ -183,7 +189,7 @@ public void doSaveProperties(long version) { } else { registryCacheExecutor.execute(new SaveProperties(lastCacheChanged.incrementAndGet())); } - logger.warn("Failed to save registry cache file, cause: " + e.getMessage(), e); + logger.warn("Failed to save registry cache file, will retry, cause: " + e.getMessage(), e); } } From b4310a9aa33f484b2469197a4a125f73dba8e26c Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Thu, 28 Mar 2019 10:14:38 +0800 Subject: [PATCH 3/3] limit retry times to save properties to local file --- .../java/org/apache/dubbo/registry/support/AbstractRegistry.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java index f99c2522324..cfe3299eb89 100644 --- a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java +++ b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java @@ -185,6 +185,7 @@ public void doSaveProperties(long version) { return; } if (version < lastCacheChanged.get()) { + savePropertiesRetryTimes.set(0); return; } else { registryCacheExecutor.execute(new SaveProperties(lastCacheChanged.incrementAndGet()));