Retry only specified tests (#8825)

This list was compiled by finding every test failure that succeeded on a retry in the past month (approx. June 20-July 20, 2023). The [flaky-test-finder.rb][1] script was used to collect this list. ``` ruby flaky-test-finder.rb -s 18000 -e 20681 | sed 's/[0-9]*[[:space:]]\(.*\)\..*/includeClasses\.add(\"\1\")/' | sort | uniq ``` [1]: https://gist.github.com/andrross/ee07a8a05beb63f1173bcb98523918b9 Signed-off-by: Andrew Ross <[email protected]> (cherry picked from commit 5d78de6) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Jul 24, 2023 · fdd88e4 · fdd88e4
1 parent cd71131
commit fdd88e4
Show file tree

Hide file tree

Showing 2 changed files with 81 additions and 0 deletions.
diff --git a/TESTING.md b/TESTING.md
@@ -13,6 +13,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran
   - [Test groups](#test-groups)
   - [Load balancing and caches](#load-balancing-and-caches)
   - [Test compatibility](#test-compatibility)
+  - [Retries](#retries)
   - [Miscellaneous](#miscellaneous)
 - [Running verification tasks](#running-verification-tasks)
 - [Testing the REST layer](#testing-the-rest-layer)
@@ -160,6 +161,10 @@ It is possible to provide a version that allows to adapt the tests' behaviour to
 
     ./gradlew test -Dtests.compatibility=1.0.0
 
+## Retries
+
+The goal of tests is to be completely deterministic such that any test failure can be easily and reliably reproduced. However, the reality is that many OpenSearch integration tests have non-deterministic behavior which results in rare test failures that cannot be easily reproduced even using the same random test seed. To mitigate the pain of frequent non-reproducible test failures, limited retries have been introduced using the Gradle [test-retry](https://plugins.gradle.org/plugin/org.gradle.test-retry) plugin. The known flaky tests are explicitly listed in the test-retry configuration of the build.gradle file. This is intended as a temporary mitigation for existing flakiness, and as such new tests should not be added to the retry list. Any new addition to the retry list must provide a thorough rationale as to why adding retries is the right thing to do as opposed to fixing the underlying flakiness. Existing flaky tests are tracked in GitHub with the [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) label.
+
 ## Miscellaneous
 
 Run all tests without stopping on errors (inspect log files).

diff --git a/build.gradle b/build.gradle
@@ -459,6 +459,82 @@ subprojects {
         maxFailures = 10
       }
       failOnPassedAfterRetry = false
+      classRetry {
+        includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests")
+        includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT")
+        includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT")
+        includeClasses.add("org.opensearch.action.admin.indices.create.ShrinkIndexIT")
+        includeClasses.add("org.opensearch.aliases.IndexAliasesIT")
+        includeClasses.add("org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT")
+        includeClasses.add("org.opensearch.blocks.SimpleBlocksIT")
+        includeClasses.add("org.opensearch.client.PitIT")
+        includeClasses.add("org.opensearch.client.ReindexIT")
+        includeClasses.add("org.opensearch.cluster.ClusterHealthIT")
+        includeClasses.add("org.opensearch.cluster.allocation.AwarenessAllocationIT")
+        includeClasses.add("org.opensearch.cluster.allocation.ClusterRerouteIT")
+        includeClasses.add("org.opensearch.cluster.coordination.AwarenessAttributeDecommissionIT")
+        includeClasses.add("org.opensearch.cluster.metadata.IndexGraveyardTests")
+        includeClasses.add("org.opensearch.cluster.routing.MovePrimaryFirstTests")
+        includeClasses.add("org.opensearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT")
+        includeClasses.add("org.opensearch.cluster.service.MasterServiceTests")
+        includeClasses.add("org.opensearch.common.util.concurrent.QueueResizableOpenSearchThreadPoolExecutorTests")
+        includeClasses.add("org.opensearch.gateway.RecoveryFromGatewayIT")
+        includeClasses.add("org.opensearch.gateway.ReplicaShardAllocatorIT")
+        includeClasses.add("org.opensearch.http.SearchRestCancellationIT")
+        includeClasses.add("org.opensearch.http.netty4.Netty4HttpServerTransportTests")
+        includeClasses.add("org.opensearch.index.IndexServiceTests")
+        includeClasses.add("org.opensearch.index.IndexSettingsTests")
+        includeClasses.add("org.opensearch.index.SegmentReplicationPressureIT")
+        includeClasses.add("org.opensearch.index.ShardIndexingPressureIT")
+        includeClasses.add("org.opensearch.index.ShardIndexingPressureSettingsIT")
+        includeClasses.add("org.opensearch.index.reindex.BulkByScrollResponseTests")
+        includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests")
+        includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests")
+        includeClasses.add("org.opensearch.index.shard.IndexShardIT")
+        includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests")
+        includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests")
+        includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT")
+        includeClasses.add("org.opensearch.indices.replication.RemoteStoreReplicationSourceTests")
+        includeClasses.add("org.opensearch.indices.replication.SegmentReplicationAllocationIT")
+        includeClasses.add("org.opensearch.indices.replication.SegmentReplicationIT")
+        includeClasses.add("org.opensearch.indices.replication.SegmentReplicationRelocationIT")
+        includeClasses.add("org.opensearch.indices.replication.SegmentReplicationTargetServiceTests")
+        includeClasses.add("org.opensearch.indices.state.CloseWhileRelocatingShardsIT")
+        includeClasses.add("org.opensearch.monitor.fs.FsHealthServiceTests")
+        includeClasses.add("org.opensearch.recovery.ReplicationCollectionTests")
+        includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep")
+        includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT")
+        includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteStoreIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteStoreStatsIT")
+        includeClasses.add("org.opensearch.remotestore.SegmentReplicationRemoteStoreIT")
+        includeClasses.add("org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT")
+        includeClasses.add("org.opensearch.remotestore.multipart.RemoteStoreMultipartIT")
+        includeClasses.add("org.opensearch.repositories.azure.AzureBlobContainerRetriesTests")
+        includeClasses.add("org.opensearch.repositories.azure.AzureBlobStoreRepositoryTests")
+        includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests")
+        includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests")
+        includeClasses.add("org.opensearch.repositories.s3.S3BlobStoreRepositoryTests")
+        includeClasses.add("org.opensearch.search.ConcurrentSegmentSearchTimeoutIT")
+        includeClasses.add("org.opensearch.search.SearchTimeoutIT")
+        includeClasses.add("org.opensearch.search.SearchWeightedRoutingIT")
+        includeClasses.add("org.opensearch.search.aggregations.bucket.DoubleTermsIT")
+        includeClasses.add("org.opensearch.search.aggregations.bucket.terms.StringTermsIT")
+        includeClasses.add("org.opensearch.search.aggregations.metrics.CardinalityIT")
+        includeClasses.add("org.opensearch.search.backpressure.SearchBackpressureIT")
+        includeClasses.add("org.opensearch.search.basic.SearchWithRandomIOExceptionsIT")
+        includeClasses.add("org.opensearch.search.pit.DeletePitMultiNodeIT")
+        includeClasses.add("org.opensearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT")
+        includeClasses.add("org.opensearch.snapshots.CloneSnapshotIT")
+        includeClasses.add("org.opensearch.snapshots.DedicatedClusterSnapshotRestoreIT")
+        includeClasses.add("org.opensearch.snapshots.RestoreSnapshotIT")
+        includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT")
+        includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT")
+        includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests")
+      }
     }
   }
 }