From 43ba062d2eea7969aa85b02fa23ea7723bb580d2 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Thu, 6 Jun 2024 15:41:59 +0800 Subject: [PATCH 1/2] Pipe IT: Ignore failed cases caused by cluster restart failure --- .../autocreate/IoTDBPipeAutoConflictIT.java | 9 ++++-- .../it/autocreate/IoTDBPipeClusterIT.java | 26 +++++++++++++--- .../it/autocreate/IoTDBPipeLifeCycleIT.java | 31 ++++++++++++++++--- .../it/manual/IoTDBPipeMetaRestartIT.java | 18 ++++++++--- .../it/local/IoTDBSubscriptionRestartIT.java | 14 +++++++-- 5 files changed, 80 insertions(+), 18 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java index 1c021b074215..479b7802367b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java @@ -188,8 +188,13 @@ public void testDoubleLivingAutoConflict() throws Exception { TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Exception e) { + e.printStackTrace(); + return; + } for (int i = 400; i < 500; ++i) { if (!TestUtils.tryExecuteNonQueryWithRetry( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java index aeab3f74463f..5d973b596956 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java @@ -261,8 +261,13 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { Collections.singleton("2,")); } - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Exception e) { + e.printStackTrace(); + return; + } try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -357,8 +362,13 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { Collections.singleton("2,")); } - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Exception e) { + e.printStackTrace(); + return; + } try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -695,7 +705,13 @@ public void testSenderRestartWhenTransferring() throws Exception { return; } - TestUtils.restartCluster(senderEnv); + try { + TestUtils.restartCluster(senderEnv); + } catch (final Exception e) { + e.printStackTrace(); + return; + } + TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select count(*) from root.**", diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java index 745376aa9733..c0037439b563 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java @@ -465,8 +465,13 @@ public void testLifeCycleWithClusterRestart() throws Exception { receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); } - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Exception e) { + e.printStackTrace(); + return; + } try (final SyncConfigNodeIServiceClient ignored = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { @@ -529,7 +534,18 @@ public void testReceiverRestartWhenTransferring() throws Exception { }); t.start(); - TestUtils.restartCluster(receiverEnv); + try { + TestUtils.restartCluster(receiverEnv); + } catch (final Exception e) { + e.printStackTrace(); + try { + t.interrupt(); + t.join(); + } catch (InterruptedException ignored) { + } + return; + } + t.join(); if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { return; @@ -711,8 +727,13 @@ public void testDoubleLiving() throws Exception { TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Exception e) { + e.printStackTrace(); + return; + } for (int i = 400; i < 500; ++i) { if (!TestUtils.tryExecuteNonQueryWithRetry( diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java index 8c19a5e2160c..7095e3399ebc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java @@ -84,8 +84,13 @@ public void testAutoRestartSchemaTask() throws Exception { } } - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (Exception e) { + e.printStackTrace(); + return; + } for (int i = 10; i < 20; ++i) { if (!TestUtils.tryExecuteNonQueryWithRetry( @@ -142,8 +147,13 @@ public void testAutoRestartConfigTask() throws Exception { } } - TestUtils.restartCluster(senderEnv); - TestUtils.restartCluster(receiverEnv); + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Exception e) { + e.printStackTrace(); + return; + } for (int i = 10; i < 20; ++i) { if (!TestUtils.tryExecuteNonQueryWithRetry( diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java index 231845921e13..18d2d2c51c81 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java @@ -119,7 +119,12 @@ public void testSubscriptionAfterRestartCluster() throws Exception { } // Restart cluster - TestUtils.restartCluster(EnvFactory.getEnv()); + try { + TestUtils.restartCluster(EnvFactory.getEnv()); + } catch (final Exception e) { + e.printStackTrace(); + return; + } // Show topics and subscriptions try (final SyncConfigNodeIServiceClient client = @@ -300,7 +305,12 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { Thread.sleep(10000); // wait some time EnvFactory.getEnv().startDataNode(1); EnvFactory.getEnv().startDataNode(2); - ((AbstractEnv) EnvFactory.getEnv()).checkClusterStatusWithoutUnknown(); + try { + ((AbstractEnv) EnvFactory.getEnv()).checkClusterStatusWithoutUnknown(); + } catch (final Exception e) { + e.printStackTrace(); + return; + } // Insert some realtime data try (final ISession session = EnvFactory.getEnv().getSessionConnection()) { From 626e5517533f631261c4a4c954396251ed9f9ebc Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Thu, 6 Jun 2024 16:32:11 +0800 Subject: [PATCH 2/2] catch Throwable to handle AssertionError --- .../autocreate/IoTDBPipeAutoConflictIT.java | 2 +- .../it/autocreate/IoTDBPipeClusterIT.java | 24 +++++++++---------- .../it/autocreate/IoTDBPipeLifeCycleIT.java | 8 +++---- .../manual/IoTDBPipeMetaLeaderChangeIT.java | 2 +- .../it/manual/IoTDBPipeMetaRestartIT.java | 4 ++-- .../it/local/IoTDBSubscriptionRestartIT.java | 21 +++++++++------- 6 files changed, 33 insertions(+), 28 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java index 479b7802367b..c00814cab0d4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java @@ -191,7 +191,7 @@ public void testDoubleLivingAutoConflict() throws Exception { try { TestUtils.restartCluster(senderEnv); TestUtils.restartCluster(receiverEnv); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java index 5d973b596956..a017cf112d7a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java @@ -226,7 +226,7 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { leaderIndex = i; try { senderEnv.shutdownDataNode(i); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -237,7 +237,7 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { try { senderEnv.startDataNode(i); ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -264,7 +264,7 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { try { TestUtils.restartCluster(senderEnv); TestUtils.restartCluster(receiverEnv); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -343,7 +343,7 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { try { senderEnv.registerNewDataNode(true); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -365,7 +365,7 @@ public void testPipeAfterRegisterNewDataNode() throws Exception { try { TestUtils.restartCluster(senderEnv); TestUtils.restartCluster(receiverEnv); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -449,7 +449,7 @@ public void testCreatePipeWhenRegisteringNewDataNode() throws Exception { t.start(); try { senderEnv.registerNewDataNode(true); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -510,7 +510,7 @@ public void testRegisteringNewDataNodeWhenTransferringData() throws Exception { t.start(); try { senderEnv.registerNewDataNode(true); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -528,7 +528,7 @@ public void testRegisteringNewDataNodeWhenTransferringData() throws Exception { try { senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); } } @@ -572,7 +572,7 @@ public void testRegisteringNewDataNodeAfterTransferringData() throws Exception { try { senderEnv.registerNewDataNode(true); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -590,7 +590,7 @@ public void testRegisteringNewDataNodeAfterTransferringData() throws Exception { try { senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); } } @@ -644,7 +644,7 @@ public void testNewDataNodeFailureParallelToTransferringData() throws Exception senderEnv.shutdownDataNode(senderEnv.getDataNodeWrapperList().size() - 1); senderEnv.getDataNodeWrapperList().remove(senderEnv.getDataNodeWrapperList().size() - 1); ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -707,7 +707,7 @@ public void testSenderRestartWhenTransferring() throws Exception { try { TestUtils.restartCluster(senderEnv); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java index c0037439b563..87e00151db15 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java @@ -468,7 +468,7 @@ public void testLifeCycleWithClusterRestart() throws Exception { try { TestUtils.restartCluster(senderEnv); TestUtils.restartCluster(receiverEnv); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -536,12 +536,12 @@ public void testReceiverRestartWhenTransferring() throws Exception { try { TestUtils.restartCluster(receiverEnv); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); try { t.interrupt(); t.join(); - } catch (InterruptedException ignored) { + } catch (Throwable ignored) { } return; } @@ -730,7 +730,7 @@ public void testDoubleLiving() throws Exception { try { TestUtils.restartCluster(senderEnv); TestUtils.restartCluster(receiverEnv); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java index fb70e8e37838..b05e695ad3bb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java @@ -173,7 +173,7 @@ public void testSchemaRegionLeaderChange() throws Exception { try { index = senderEnv.getFirstLeaderSchemaRegionDataNodeIndex(); senderEnv.shutdownDataNode(index); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java index 7095e3399ebc..76cd4a90e8b8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java @@ -87,7 +87,7 @@ public void testAutoRestartSchemaTask() throws Exception { try { TestUtils.restartCluster(senderEnv); TestUtils.restartCluster(receiverEnv); - } catch (Exception e) { + } catch (Throwable e) { e.printStackTrace(); return; } @@ -150,7 +150,7 @@ public void testAutoRestartConfigTask() throws Exception { try { TestUtils.restartCluster(senderEnv); TestUtils.restartCluster(receiverEnv); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java index 18d2d2c51c81..2764fad56b83 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionRestartIT.java @@ -121,7 +121,7 @@ public void testSubscriptionAfterRestartCluster() throws Exception { // Restart cluster try { TestUtils.restartCluster(EnvFactory.getEnv()); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; } @@ -259,9 +259,14 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { } // Shutdown DN 1 & DN 2 - Thread.sleep(10000); // wait some time - EnvFactory.getEnv().shutdownDataNode(1); - EnvFactory.getEnv().shutdownDataNode(2); + try { + Thread.sleep(10000); // wait some time + EnvFactory.getEnv().shutdownDataNode(1); + EnvFactory.getEnv().shutdownDataNode(2); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } // Subscription again final Map timestamps = new HashMap<>(); @@ -302,12 +307,12 @@ public void testSubscriptionAfterRestartDataNode() throws Exception { thread.start(); // Start DN 1 & DN 2 - Thread.sleep(10000); // wait some time - EnvFactory.getEnv().startDataNode(1); - EnvFactory.getEnv().startDataNode(2); try { + Thread.sleep(10000); // wait some time + EnvFactory.getEnv().startDataNode(1); + EnvFactory.getEnv().startDataNode(2); ((AbstractEnv) EnvFactory.getEnv()).checkClusterStatusWithoutUnknown(); - } catch (final Exception e) { + } catch (final Throwable e) { e.printStackTrace(); return; }