Fixed to retry when loading plugin fails (#1728)

* Fixed to retry when loading plugin fails * add plugin retry load test
treasure-data · Jun 24, 2022 · bec6e6b · bec6e6b
1 parent 3356327
commit bec6e6b
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 4 deletions.
diff --git a/digdag-core/src/main/java/io/digdag/core/plugin/RemotePluginLoader.java b/digdag-core/src/main/java/io/digdag/core/plugin/RemotePluginLoader.java
@@ -10,6 +10,7 @@
 import com.google.common.collect.ImmutableList;
 
 import io.digdag.commons.ThrowablesUtil;
+import io.digdag.util.RetryExecutor;
 import org.apache.maven.repository.internal.MavenRepositorySystemUtils;
 import org.eclipse.aether.RepositorySystem;
 import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory;
@@ -45,6 +46,10 @@ public class RemotePluginLoader
 {
     private static final Logger logger = LoggerFactory.getLogger(RemotePluginLoader.class);
 
+    private static final int EXTRACT_RETRIES = 3;
+    private static final int EXTRACT_MIN_RETRY_WAIT_MS = 1000;
+    private static final int EXTRACT_MAX_RETRY_WAIT_MS = 10000;
+
     private static final List<RemoteRepository> DEFAULT_REPOSITORIES = ImmutableList.copyOf(new RemoteRepository[] {
         new RemoteRepository.Builder("central", "default", "https://repo1.maven.org/maven2/").build(),
         new RemoteRepository.Builder("jcenter", "default", "https://jcenter.bintray.com/").build(),
@@ -161,10 +166,16 @@ private List<ArtifactResult> resolveArtifacts(List<RemoteRepository> repositorie
     {
         DependencyRequest depRequest = buildDependencyRequest(repositories, dep, JavaScopes.RUNTIME);
         try {
-            return system.resolveDependencies(session, depRequest).getArtifactResults();
-        }
-        catch (DependencyResolutionException ex) {
-            throw ThrowablesUtil.propagate(ex);
+            return RetryExecutor.retryExecutor()
+                    .retryIf(exception -> true)
+                    .withInitialRetryWait(EXTRACT_MIN_RETRY_WAIT_MS)
+                    .withMaxRetryWait(EXTRACT_MAX_RETRY_WAIT_MS)
+                    .onRetry((exception, retryCount, retryLimit, retryWait) ->
+                            logger.warn("Failed to resolve artifacts: retry {} of {}", retryCount, retryLimit, exception))
+                    .withRetryLimit(EXTRACT_RETRIES)
+                    .run(() -> system.resolveDependencies(session, depRequest).getArtifactResults());
+        } catch (RetryExecutor.RetryGiveupException e) {
+            throw ThrowablesUtil.propagate(e.getCause());
         }
     }
 

diff --git a/digdag-tests/src/test/java/acceptance/PluginIT.java b/digdag-tests/src/test/java/acceptance/PluginIT.java
@@ -6,12 +6,15 @@
 import utils.CommandStatus;
 import utils.TestUtils;
 
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
 import static utils.TestUtils.copyResource;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
@@ -40,4 +43,21 @@ public void testRun()
                 new String(Files.readAllBytes(root().resolve("example.out")), UTF_8).trim(),
                 is("Worked? yes"));
     }
+
+    @Test
+    public void testRetryLoadPlugin()
+            throws Exception {
+        try {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            System.setOut(new PrintStream(out));
+            copyResource("acceptance/plugin/plugin.dig", root().resolve("plugin.dig"));
+            copyResource("acceptance/plugin/template.txt", root().resolve("template.txt"));
+            TestUtils.main("run", "-o", root().toString(), "--project", root().toString(), "plugin.dig", "-p", "repository_path=test", "-X", "plugin.local-path=" + root().resolve(".digdag/plugins"), "--log-level", "debug");
+            assertThat(
+                    out.toString(),
+                    containsString("Failed to resolve artifacts: retry 3 of 3"));
+        } finally {
+            System.setOut(System.out);
+        }
+    }
 }