Mark more tests as flaky

This adds the necessary annotation to repeat flaky tests if necessary for tests described in these issues: * JanusGraph#1457 * JanusGraph#1459 * JanusGraph#1462 * JanusGraph#1464 * JanusGraph#1465 * JanusGraph#1497 * JanusGraph#2272 * JanusGraph#3142 * JanusGraph#3356 * JanusGraph#3392 * JanusGraph#3393 * JanusGraph#3651 * JanusGraph#3931 * JanusGraph#3959 * JanusGraph#3960 Signed-off-by: Florian Hockmann <[email protected]>
GDATASoftwareAG · Sep 7, 2023 · 1f8a634 · 1f8a634 · github-actions · Sep 7, 2023
1 parent e2b0bed
commit 1f8a634
Show file tree

Hide file tree

Showing 10 changed files with 114 additions and 33 deletions.
diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java
@@ -2188,7 +2188,7 @@ public void testIndexReplay() throws Exception {
         assertEquals(4, recoveryStats[1]); //all 4 index transaction had provoked errors in the indexing backend
     }
 
-    @Test
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
         final Object[] settings = new Object[]{option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ofMillis(0),
                 option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50),

diff --git a/...ckend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java b/...ckend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java
@@ -20,6 +20,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -127,7 +128,7 @@ private void verifyLockingOverwrite(long num) {
         }
     }
 
-    @Test
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     public void testIdCounts() {
         makeVertexIndexedUniqueKey("uid", Integer.class);
         mgmt.setConsistency(mgmt.getGraphIndex("uid"), ConsistencyModifier.LOCK);

diff --git a/...eleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java b/...eleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java
@@ -14,7 +14,6 @@
 
 package org.janusgraph.graphdb.berkeleyje;
 
-import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.BerkeleyStorageSetup;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.graphdb.JanusGraphOperationCountingTest;
@@ -27,12 +26,6 @@ public WriteConfiguration getBaseConfiguration() {
         return BerkeleyStorageSetup.getBerkeleyJEGraphConfiguration();
     }
 
-    @Override
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
-    public void testIdCounts() {
-        super.testIdCounts();
-    }
-
     @AfterEach
     public void resetCounts() {
         resetMetrics(); // Metrics is a singleton, so subsequents test runs have wrong counts if we don't clean up.

diff --git a/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java b/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java
@@ -109,7 +109,7 @@ public void createConfigurationShouldSupportMultiHosts() throws Exception {
         }
     }
 
-    @Test
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     public void dropGraphShouldRemoveGraphKeyspace() throws Exception {
         final MapConfiguration graphConfig = getGraphConfig();
         final String graphName = graphConfig.getString(GRAPH_NAME.toStringWithoutRoot());
@@ -137,5 +137,11 @@ public void dropGraphShouldRemoveGraphKeyspace() throws Exception {
     public void updateConfigurationShouldRemoveGraphFromCache() throws Exception {
         super.updateConfigurationShouldRemoveGraphFromCache();
     }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void dropShouldCleanUpTraversalSourceAndBindings() throws Exception {
+        super.dropShouldCleanUpTraversalSourceAndBindings();
+    }
 }
 
diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java
@@ -14,13 +14,18 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.ParameterizedRepeatedIfExceptionsTest;
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.StorageSetup;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.graphdb.JanusGraphTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
+import java.util.concurrent.ExecutionException;
+
 @Testcontainers
 public class CQLGraphCacheTest extends JanusGraphTest {
 
@@ -31,4 +36,46 @@ public class CQLGraphCacheTest extends JanusGraphTest {
     public WriteConfiguration getConfiguration() {
         return StorageSetup.addPermanentCache(cqlContainer.getConfiguration(getClass().getSimpleName()));
     }
+
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    public void simpleLogTest(boolean useStringId) throws InterruptedException {
+        super.simpleLogTest(useStringId);
+    }
+
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException {
+        super.simpleLogTestWithFailure(useStringId);
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testEdgeTTLTiming() throws Exception {
+        super.testEdgeTTLTiming();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testEdgeTTLWithTransactions() throws Exception {
+        super.testEdgeTTLWithTransactions();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testVertexTTLWithCompositeIndex() throws Exception {
+        super.testVertexTTLWithCompositeIndex();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testVertexTTLImplicitKey() throws Exception {
+        super.testVertexTTLImplicitKey();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testReindexingForEdgeIndex() throws ExecutionException, InterruptedException {
+        super.testReindexingForEdgeIndex();
+    }
 }
diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java
@@ -14,6 +14,7 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.ParameterizedRepeatedIfExceptionsTest;
 import io.github.artsok.RepeatedIfExceptionsTest;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -227,18 +228,34 @@ public void testHasTTL() {
         assertTrue(features.hasCellTTL());
     }
 
-    @ParameterizedTest
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     @ValueSource(booleans = {true, false})
-    public void simpleLogTest(boolean useStringId) {
-        for (int i = 0; i < 3; i++) {
-            try {
-                super.simpleLogTest(useStringId);
-                // If the test passes, break out of the loop.
-                break;
-            } catch (Exception ex) {
-                log.info("Attempt #{} fails", i, ex);
-            }
-        }
+    public void simpleLogTest(boolean useStringId) throws InterruptedException {
+        super.simpleLogTest(useStringId);
+    }
+
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException {
+        super.simpleLogTestWithFailure(useStringId);
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testEdgeTTLTiming() throws Exception {
+        super.testEdgeTTLTiming();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testVertexTTLWithCompositeIndex() throws Exception {
+        super.testVertexTTLWithCompositeIndex();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testVertexTTLImplicitKey() throws Exception {
+        super.testVertexTTLImplicitKey();
     }
 
     @RepeatedIfExceptionsTest(repeats = 3)

diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java
@@ -14,6 +14,7 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.olap.OLAPTest;
@@ -29,4 +30,10 @@ public class CQLOLAPTest extends OLAPTest {
     public WriteConfiguration getConfiguration() {
         return cqlContainer.getConfiguration(getClass().getSimpleName()).getConfiguration();
     }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testShortestDistance() throws Exception {
+        super.testShortestDistance();
+    }
 }
diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java
@@ -41,6 +41,12 @@ public ModifiableConfiguration getStorageConfiguration() {
         return getBerkeleyJEConfiguration();
     }
 
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void indexShouldNotExistAfterDeletion() throws Exception {
+        super.indexShouldNotExistAfterDeletion();
+    }
+
     /**
      * Test {@link org.janusgraph.example.GraphOfTheGodsFactory#create(String)}.
      */
@@ -55,9 +61,15 @@ public void testGraphOfTheGodsFactoryCreate() {
         gotg.close();
     }
 
-    @RepeatedIfExceptionsTest(repeats = 3)
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testDisableAndDiscardManuallyAndDropEnabledIndex() throws Exception {
+        super.testDisableAndDiscardManuallyAndDropEnabledIndex();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     @Override
-    public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
-        super.testIndexUpdatesWithoutReindex();
+    public void testDiscardAndDropRegisteredIndex() throws ExecutionException, InterruptedException {
+        super.testDiscardAndDropRegisteredIndex();
     }
 }
diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java
@@ -14,15 +14,12 @@
 
 package org.janusgraph.diskstorage.es;
 
-import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
 import org.junit.jupiter.api.Disabled;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
-import java.util.concurrent.ExecutionException;
-
 @Testcontainers
 public class CQLElasticsearchTest extends ElasticsearchJanusGraphIndexTest {
 
@@ -37,10 +34,4 @@ public ModifiableConfiguration getStorageConfiguration() {
     @Override
     @Disabled("CQL seems to not clear storage correctly")
     public void testClearStorage() {}
-
-    @RepeatedIfExceptionsTest(repeats = 3)
-    @Override
-    public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
-        super.testIndexUpdatesWithoutReindex();
-    }
 }
diff --git a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java
@@ -14,6 +14,7 @@
 
 package org.janusgraph.diskstorage.solr;
 
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
 import org.junit.jupiter.api.Test;
@@ -38,6 +39,12 @@ public boolean supportsWildcardQuery() {
         return false;
     }
 
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testSetIndexing() {
+        super.testSetIndexing();
+    }
+
     @Test
     @Override
     public void testDiscardAndDropRegisteredIndex() {
Benchmark suite	Current: `1f8a634`	Previous: `c7f1414`	Ratio
`org.janusgraph.JanusGraphSpeedBenchmark.basicAddAndDelete`	`14525.979112637224` ms/op	`22832.725066731808` ms/op	`0.64`
`org.janusgraph.GraphCentricQueryBenchmark.getVertices`	`1351.2532596958442` ms/op	`1846.1651353102989` ms/op	`0.73`
`org.janusgraph.MgmtOlapJobBenchmark.runClearIndex`	`221.01526613043478` ms/op	`225.06416455256917` ms/op	`0.98`
`org.janusgraph.MgmtOlapJobBenchmark.runReindex`	`478.82749310555556` ms/op	`577.6853888111111` ms/op	`0.83`
`org.janusgraph.JanusGraphSpeedBenchmark.basicCount`	`437.95260863245505` ms/op	`404.26724358193667` ms/op	`1.08`
`org.janusgraph.CQLMultiQueryMultiSlicesBenchmark.getValuesAllPropertiesWithAllMultiQuerySlicesUnderMaxRequestsPerConnection`	`9879.776841296098` ms/op	`14049.432726261206` ms/op	`0.70`
`org.janusgraph.CQLMultiQueryBenchmark.getElementsWithUsingEmitRepeatSteps`	`35155.55099783333` ms/op	`48032.01077955` ms/op	`0.73`
`org.janusgraph.CQLMultiQueryMultiSlicesBenchmark.getValuesMultiplePropertiesWithSmallBatch`	`35769.49384717191` ms/op	`48915.53240540001` ms/op	`0.73`
`org.janusgraph.CQLMultiQueryMultiSlicesBenchmark.vertexCentricPropertiesFetching`	`71486.1659924` ms/op	`105756.8187758` ms/op	`0.68`
`org.janusgraph.CQLMultiQueryBenchmark.getAllElementsTraversedFromOuterVertex`	`16889.930117528864` ms/op	`22789.450908244446` ms/op	`0.74`
`org.janusgraph.CQLMultiQueryBenchmark.getVerticesWithDoubleUnion`	`645.4643564519697` ms/op	`834.129436581297` ms/op	`0.77`
`org.janusgraph.CQLMultiQueryMultiSlicesBenchmark.getValuesAllPropertiesWithUnlimitedBatch`	`9221.89148492405` ms/op	`12360.737748881762` ms/op	`0.75`
`org.janusgraph.CQLMultiQueryBenchmark.getNames`	`17128.102654976665` ms/op	`22053.325934522858` ms/op	`0.78`
`org.janusgraph.CQLMultiQueryMultiSlicesBenchmark.getValuesThreePropertiesWithAllMultiQuerySlicesUnderMaxRequestsPerConnection`	`12062.862032687164` ms/op	`17268.2833808608` ms/op	`0.70`
`org.janusgraph.CQLMultiQueryBenchmark.getLabels`	`15165.007782775714` ms/op	`19012.308155116116` ms/op	`0.80`
`org.janusgraph.CQLMultiQueryBenchmark.getVerticesFilteredByAndStep`	`698.6864511516683` ms/op	`877.034097341064` ms/op	`0.80`
`org.janusgraph.CQLMultiQueryBenchmark.getVerticesFromMultiNestedRepeatStepStartingFromSingleVertex`	`24054.98091854` ms/op	`29542.13639576095` ms/op	`0.81`
`org.janusgraph.CQLMultiQueryBenchmark.getVerticesWithCoalesceUsage`	`610.3134604542911` ms/op	`801.440679281071` ms/op	`0.76`
`org.janusgraph.CQLMultiQueryMultiSlicesBenchmark.getValuesMultiplePropertiesWithAllMultiQuerySlicesUnderMaxRequestsPerConnection`	`31973.31205771476` ms/op	`44586.59237236953` ms/op	`0.72`
`org.janusgraph.CQLMultiQueryBenchmark.getIdToOutVerticesProjection`	`434.56687233080623` ms/op	`539.5587642404585` ms/op	`0.81`
`org.janusgraph.CQLMultiQueryMultiSlicesBenchmark.getValuesMultiplePropertiesWithUnlimitedBatch`	`34182.616832782565` ms/op	`48771.697695169096` ms/op	`0.70`
`org.janusgraph.CQLMultiQueryBenchmark.getNeighborNames`	`17214.371970422348` ms/op	`22258.082075511105` ms/op	`0.77`
`org.janusgraph.CQLMultiQueryBenchmark.getElementsWithUsingRepeatUntilSteps`	`18852.23238906154` ms/op	`24524.48026238206` ms/op	`0.77`
`org.janusgraph.CQLMultiQueryBenchmark.getAdjacentVerticesLocalCounts`	`17023.190769099605` ms/op	`23609.766987193336` ms/op	`0.72`