diff --git a/TESTING.md b/TESTING.md index 908c456de6cf9..06533cd5c170a 100644 --- a/TESTING.md +++ b/TESTING.md @@ -13,6 +13,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran - [Test groups](#test-groups) - [Load balancing and caches](#load-balancing-and-caches) - [Test compatibility](#test-compatibility) + - [Retries](#retries) - [Miscellaneous](#miscellaneous) - [Running verification tasks](#running-verification-tasks) - [Testing the REST layer](#testing-the-rest-layer) @@ -160,6 +161,10 @@ It is possible to provide a version that allows to adapt the tests' behaviour to ./gradlew test -Dtests.compatibility=1.0.0 +## Retries + +The goal of tests is to be completely deterministic such that any test failure can be easily and reliably reproduced. However, the reality is that many OpenSearch integration tests have non-deterministic behavior which results in rare test failures that cannot be easily reproduced even using the same random test seed. To mitigate the pain of frequent non-reproducible test failures, limited retries have been introduced using the Gradle [test-retry](https://plugins.gradle.org/plugin/org.gradle.test-retry) plugin. The known flaky tests are explicitly listed in the test-retry configuration of the build.gradle file. This is intended as a temporary mitigation for existing flakiness, and as such new tests should not be added to the retry list. Any new addition to the retry list must provide a thorough rationale as to why adding retries is the right thing to do as opposed to fixing the underlying flakiness. Existing flaky tests are tracked in GitHub with the [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) label. + ## Miscellaneous Run all tests without stopping on errors (inspect log files). diff --git a/build.gradle b/build.gradle index 50db96d1e11b0..5f99d0ba078a8 100644 --- a/build.gradle +++ b/build.gradle @@ -459,6 +459,82 @@ subprojects { maxFailures = 10 } failOnPassedAfterRetry = false + classRetry { + includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests") + includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT") + includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT") + includeClasses.add("org.opensearch.action.admin.indices.create.ShrinkIndexIT") + includeClasses.add("org.opensearch.aliases.IndexAliasesIT") + includeClasses.add("org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.blocks.SimpleBlocksIT") + includeClasses.add("org.opensearch.client.PitIT") + includeClasses.add("org.opensearch.client.ReindexIT") + includeClasses.add("org.opensearch.cluster.ClusterHealthIT") + includeClasses.add("org.opensearch.cluster.allocation.AwarenessAllocationIT") + includeClasses.add("org.opensearch.cluster.allocation.ClusterRerouteIT") + includeClasses.add("org.opensearch.cluster.coordination.AwarenessAttributeDecommissionIT") + includeClasses.add("org.opensearch.cluster.metadata.IndexGraveyardTests") + includeClasses.add("org.opensearch.cluster.routing.MovePrimaryFirstTests") + includeClasses.add("org.opensearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT") + includeClasses.add("org.opensearch.cluster.service.MasterServiceTests") + includeClasses.add("org.opensearch.common.util.concurrent.QueueResizableOpenSearchThreadPoolExecutorTests") + includeClasses.add("org.opensearch.gateway.RecoveryFromGatewayIT") + includeClasses.add("org.opensearch.gateway.ReplicaShardAllocatorIT") + includeClasses.add("org.opensearch.http.SearchRestCancellationIT") + includeClasses.add("org.opensearch.http.netty4.Netty4HttpServerTransportTests") + includeClasses.add("org.opensearch.index.IndexServiceTests") + includeClasses.add("org.opensearch.index.IndexSettingsTests") + includeClasses.add("org.opensearch.index.SegmentReplicationPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureSettingsIT") + includeClasses.add("org.opensearch.index.reindex.BulkByScrollResponseTests") + includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests") + includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests") + includeClasses.add("org.opensearch.index.shard.IndexShardIT") + includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests") + includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests") + includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT") + includeClasses.add("org.opensearch.indices.replication.RemoteStoreReplicationSourceTests") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationAllocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationRelocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationTargetServiceTests") + includeClasses.add("org.opensearch.indices.state.CloseWhileRelocatingShardsIT") + includeClasses.add("org.opensearch.monitor.fs.FsHealthServiceTests") + includeClasses.add("org.opensearch.recovery.ReplicationCollectionTests") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT") + includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreStatsIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.multipart.RemoteStoreMultipartIT") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.s3.S3BlobStoreRepositoryTests") + includeClasses.add("org.opensearch.search.ConcurrentSegmentSearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchWeightedRoutingIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.DoubleTermsIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.terms.StringTermsIT") + includeClasses.add("org.opensearch.search.aggregations.metrics.CardinalityIT") + includeClasses.add("org.opensearch.search.backpressure.SearchBackpressureIT") + includeClasses.add("org.opensearch.search.basic.SearchWithRandomIOExceptionsIT") + includeClasses.add("org.opensearch.search.pit.DeletePitMultiNodeIT") + includeClasses.add("org.opensearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.snapshots.CloneSnapshotIT") + includeClasses.add("org.opensearch.snapshots.DedicatedClusterSnapshotRestoreIT") + includeClasses.add("org.opensearch.snapshots.RestoreSnapshotIT") + includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT") + includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests") + } } } }