From fdd88e433152ce2c25d8db28af1bc731f2392e25 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 24 Jul 2023 20:53:36 +0000 Subject: [PATCH] Retry only specified tests (#8825) This list was compiled by finding every test failure that succeeded on a retry in the past month (approx. June 20-July 20, 2023). The [flaky-test-finder.rb][1] script was used to collect this list. ``` ruby flaky-test-finder.rb -s 18000 -e 20681 | sed 's/[0-9]*[[:space:]]\(.*\)\..*/includeClasses\.add(\"\1\")/' | sort | uniq ``` [1]: https://gist.github.com/andrross/ee07a8a05beb63f1173bcb98523918b9 Signed-off-by: Andrew Ross (cherry picked from commit 5d78de61f33bd813913563076d3dfcf8ddaf1498) Signed-off-by: github-actions[bot] --- TESTING.md | 5 ++++ build.gradle | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/TESTING.md b/TESTING.md index 908c456de6cf9..06533cd5c170a 100644 --- a/TESTING.md +++ b/TESTING.md @@ -13,6 +13,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran - [Test groups](#test-groups) - [Load balancing and caches](#load-balancing-and-caches) - [Test compatibility](#test-compatibility) + - [Retries](#retries) - [Miscellaneous](#miscellaneous) - [Running verification tasks](#running-verification-tasks) - [Testing the REST layer](#testing-the-rest-layer) @@ -160,6 +161,10 @@ It is possible to provide a version that allows to adapt the tests' behaviour to ./gradlew test -Dtests.compatibility=1.0.0 +## Retries + +The goal of tests is to be completely deterministic such that any test failure can be easily and reliably reproduced. However, the reality is that many OpenSearch integration tests have non-deterministic behavior which results in rare test failures that cannot be easily reproduced even using the same random test seed. To mitigate the pain of frequent non-reproducible test failures, limited retries have been introduced using the Gradle [test-retry](https://plugins.gradle.org/plugin/org.gradle.test-retry) plugin. The known flaky tests are explicitly listed in the test-retry configuration of the build.gradle file. This is intended as a temporary mitigation for existing flakiness, and as such new tests should not be added to the retry list. Any new addition to the retry list must provide a thorough rationale as to why adding retries is the right thing to do as opposed to fixing the underlying flakiness. Existing flaky tests are tracked in GitHub with the [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) label. + ## Miscellaneous Run all tests without stopping on errors (inspect log files). diff --git a/build.gradle b/build.gradle index 50db96d1e11b0..5f99d0ba078a8 100644 --- a/build.gradle +++ b/build.gradle @@ -459,6 +459,82 @@ subprojects { maxFailures = 10 } failOnPassedAfterRetry = false + classRetry { + includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests") + includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT") + includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT") + includeClasses.add("org.opensearch.action.admin.indices.create.ShrinkIndexIT") + includeClasses.add("org.opensearch.aliases.IndexAliasesIT") + includeClasses.add("org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.blocks.SimpleBlocksIT") + includeClasses.add("org.opensearch.client.PitIT") + includeClasses.add("org.opensearch.client.ReindexIT") + includeClasses.add("org.opensearch.cluster.ClusterHealthIT") + includeClasses.add("org.opensearch.cluster.allocation.AwarenessAllocationIT") + includeClasses.add("org.opensearch.cluster.allocation.ClusterRerouteIT") + includeClasses.add("org.opensearch.cluster.coordination.AwarenessAttributeDecommissionIT") + includeClasses.add("org.opensearch.cluster.metadata.IndexGraveyardTests") + includeClasses.add("org.opensearch.cluster.routing.MovePrimaryFirstTests") + includeClasses.add("org.opensearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT") + includeClasses.add("org.opensearch.cluster.service.MasterServiceTests") + includeClasses.add("org.opensearch.common.util.concurrent.QueueResizableOpenSearchThreadPoolExecutorTests") + includeClasses.add("org.opensearch.gateway.RecoveryFromGatewayIT") + includeClasses.add("org.opensearch.gateway.ReplicaShardAllocatorIT") + includeClasses.add("org.opensearch.http.SearchRestCancellationIT") + includeClasses.add("org.opensearch.http.netty4.Netty4HttpServerTransportTests") + includeClasses.add("org.opensearch.index.IndexServiceTests") + includeClasses.add("org.opensearch.index.IndexSettingsTests") + includeClasses.add("org.opensearch.index.SegmentReplicationPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureIT") + includeClasses.add("org.opensearch.index.ShardIndexingPressureSettingsIT") + includeClasses.add("org.opensearch.index.reindex.BulkByScrollResponseTests") + includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests") + includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests") + includeClasses.add("org.opensearch.index.shard.IndexShardIT") + includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests") + includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests") + includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT") + includeClasses.add("org.opensearch.indices.replication.RemoteStoreReplicationSourceTests") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationAllocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationRelocationIT") + includeClasses.add("org.opensearch.indices.replication.SegmentReplicationTargetServiceTests") + includeClasses.add("org.opensearch.indices.state.CloseWhileRelocatingShardsIT") + includeClasses.add("org.opensearch.monitor.fs.FsHealthServiceTests") + includeClasses.add("org.opensearch.recovery.ReplicationCollectionTests") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT") + includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT") + includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT") + includeClasses.add("org.opensearch.remotestore.RemoteStoreStatsIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT") + includeClasses.add("org.opensearch.remotestore.multipart.RemoteStoreMultipartIT") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.azure.AzureBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests") + includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests") + includeClasses.add("org.opensearch.repositories.s3.S3BlobStoreRepositoryTests") + includeClasses.add("org.opensearch.search.ConcurrentSegmentSearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchTimeoutIT") + includeClasses.add("org.opensearch.search.SearchWeightedRoutingIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.DoubleTermsIT") + includeClasses.add("org.opensearch.search.aggregations.bucket.terms.StringTermsIT") + includeClasses.add("org.opensearch.search.aggregations.metrics.CardinalityIT") + includeClasses.add("org.opensearch.search.backpressure.SearchBackpressureIT") + includeClasses.add("org.opensearch.search.basic.SearchWithRandomIOExceptionsIT") + includeClasses.add("org.opensearch.search.pit.DeletePitMultiNodeIT") + includeClasses.add("org.opensearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.snapshots.CloneSnapshotIT") + includeClasses.add("org.opensearch.snapshots.DedicatedClusterSnapshotRestoreIT") + includeClasses.add("org.opensearch.snapshots.RestoreSnapshotIT") + includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT") + includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT") + includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests") + } } } }