Merge remote-tracking branch 'upstream/main' into fix_maven_publish_task

opensearch-project · Jul 25, 2023 · b8a6f5b · b8a6f5b
2 parents 3ab7a14 + 6786608
commit b8a6f5b
Show file tree

Hide file tree

Showing 167 changed files with 1,067 additions and 896 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1 +1 @@
-*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis @sachinpkale
+*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @dbwiddis @sachinpkale @sohami
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -82,12 +82,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `io.grpc:grpc-context` from 1.46.0 to 1.56.1 ([#8726](https://github.com/opensearch-project/OpenSearch/pull/8726))
 - Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.5 to 12.1.6 ([#8724](https://github.com/opensearch-project/OpenSearch/pull/8724))
 - Bump `commons-codec:commons-codec` from 1.15 to 1.16.0 ([#8725](https://github.com/opensearch-project/OpenSearch/pull/8725))
+- Bump `org.apache.zookeeper:zookeeper` from 3.8.1 to 3.8.2 ([#8844](https://github.com/opensearch-project/OpenSearch/pull/8844))
+- Bump `org.gradle.test-retry` from 1.5.3 to 1.5.4 ([#8842](https://github.com/opensearch-project/OpenSearch/pull/8842))
+- Bump `com.netflix.nebula.ospackage-base` from 11.3.0 to 11.4.0 ([#8838](https://github.com/opensearch-project/OpenSearch/pull/8838))
+- Bump `com.google.http-client:google-http-client-gson` from 1.43.2 to 1.43.3 ([#8840](https://github.com/opensearch-project/OpenSearch/pull/8840))
+- OpenJDK Update (July 2023 Patch releases) ([#8868](https://github.com/opensearch-project/OpenSearch/pull/8868)
+- Bump `hadoop` libraries from 3.3.4 to 3.3.6 ([#6995](https://github.com/opensearch-project/OpenSearch/pull/6995))
 
 ### Changed
 - Perform aggregation postCollection in ContextIndexSearcher after searching leaves ([#8303](https://github.com/opensearch-project/OpenSearch/pull/8303))
 - Make Span exporter configurable ([#8620](https://github.com/opensearch-project/OpenSearch/issues/8620))
 - Change InternalSignificantTerms to sum shard-level superset counts only in final reduce ([#8735](https://github.com/opensearch-project/OpenSearch/pull/8735))
 - Exclude 'benchmarks' from codecov report ([#8805](https://github.com/opensearch-project/OpenSearch/pull/8805))
+- [Refactor] MediaTypeParser to MediaTypeParserRegistry ([#8636](https://github.com/opensearch-project/OpenSearch/pull/8636))
+- Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807))
 
 ### Deprecated
 
@@ -98,4 +106,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.8...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.10...2.x
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -25,6 +25,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Sachin Kale              | [sachinpkale](https://github.com/sachinpkale)           | Amazon      |
 | Sarat Vemulapalli        | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon      |
 | Shweta Thareja           | [shwetathareja](https://github.com/shwetathareja)       | Amazon      |
+| Sorabh Hamirwasia        | [sohami](https://github.com/sohami)                     | Amazon      |
 | Suraj Singh              | [dreamer-89](https://github.com/dreamer-89)             | Amazon      |
 | Tianli Feng              | [tlfeng](https://github.com/tlfeng)                     | Amazon      |
 | Vacha Shah               | [VachaShah](https://github.com/VachaShah)               | Amazon      |

diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@
 [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
 [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
 [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
-[![2.8 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.8.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.8.0")
+[![2.10 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.10.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.10.0")
 [![3.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
 [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
 [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)

diff --git a/TESTING.md b/TESTING.md
@@ -13,6 +13,7 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran
   - [Test groups](#test-groups)
   - [Load balancing and caches](#load-balancing-and-caches)
   - [Test compatibility](#test-compatibility)
+  - [Retries](#retries)
   - [Miscellaneous](#miscellaneous)
 - [Running verification tasks](#running-verification-tasks)
 - [Testing the REST layer](#testing-the-rest-layer)
@@ -160,6 +161,10 @@ It is possible to provide a version that allows to adapt the tests' behaviour to
 
     ./gradlew test -Dtests.compatibility=1.0.0
 
+## Retries
+
+The goal of tests is to be completely deterministic such that any test failure can be easily and reliably reproduced. However, the reality is that many OpenSearch integration tests have non-deterministic behavior which results in rare test failures that cannot be easily reproduced even using the same random test seed. To mitigate the pain of frequent non-reproducible test failures, limited retries have been introduced using the Gradle [test-retry](https://plugins.gradle.org/plugin/org.gradle.test-retry) plugin. The known flaky tests are explicitly listed in the test-retry configuration of the build.gradle file. This is intended as a temporary mitigation for existing flakiness, and as such new tests should not be added to the retry list. Any new addition to the retry list must provide a thorough rationale as to why adding retries is the right thing to do as opposed to fixing the underlying flakiness. Existing flaky tests are tracked in GitHub with the [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) label.
+
 ## Miscellaneous
 
 Run all tests without stopping on errors (inspect log files).

diff --git a/build.gradle b/build.gradle
@@ -55,7 +55,7 @@ plugins {
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
   id "com.diffplug.spotless" version "6.19.0" apply false
-  id "org.gradle.test-retry" version "1.5.3" apply false
+  id "org.gradle.test-retry" version "1.5.4" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }
@@ -470,6 +470,82 @@ subprojects {
         maxFailures = 10
       }
       failOnPassedAfterRetry = false
+      classRetry {
+        includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests")
+        includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT")
+        includeClasses.add("org.opensearch.action.admin.indices.create.CreateIndexIT")
+        includeClasses.add("org.opensearch.action.admin.indices.create.ShrinkIndexIT")
+        includeClasses.add("org.opensearch.aliases.IndexAliasesIT")
+        includeClasses.add("org.opensearch.backwards.MixedClusterClientYamlTestSuiteIT")
+        includeClasses.add("org.opensearch.blocks.SimpleBlocksIT")
+        includeClasses.add("org.opensearch.client.PitIT")
+        includeClasses.add("org.opensearch.client.ReindexIT")
+        includeClasses.add("org.opensearch.cluster.ClusterHealthIT")
+        includeClasses.add("org.opensearch.cluster.allocation.AwarenessAllocationIT")
+        includeClasses.add("org.opensearch.cluster.allocation.ClusterRerouteIT")
+        includeClasses.add("org.opensearch.cluster.coordination.AwarenessAttributeDecommissionIT")
+        includeClasses.add("org.opensearch.cluster.metadata.IndexGraveyardTests")
+        includeClasses.add("org.opensearch.cluster.routing.MovePrimaryFirstTests")
+        includeClasses.add("org.opensearch.cluster.routing.allocation.decider.DiskThresholdDeciderIT")
+        includeClasses.add("org.opensearch.cluster.service.MasterServiceTests")
+        includeClasses.add("org.opensearch.common.util.concurrent.QueueResizableOpenSearchThreadPoolExecutorTests")
+        includeClasses.add("org.opensearch.gateway.RecoveryFromGatewayIT")
+        includeClasses.add("org.opensearch.gateway.ReplicaShardAllocatorIT")
+        includeClasses.add("org.opensearch.http.SearchRestCancellationIT")
+        includeClasses.add("org.opensearch.http.netty4.Netty4HttpServerTransportTests")
+        includeClasses.add("org.opensearch.index.IndexServiceTests")
+        includeClasses.add("org.opensearch.index.IndexSettingsTests")
+        includeClasses.add("org.opensearch.index.SegmentReplicationPressureIT")
+        includeClasses.add("org.opensearch.index.ShardIndexingPressureIT")
+        includeClasses.add("org.opensearch.index.ShardIndexingPressureSettingsIT")
+        includeClasses.add("org.opensearch.index.reindex.BulkByScrollResponseTests")
+        includeClasses.add("org.opensearch.index.reindex.DeleteByQueryBasicTests")
+        includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests")
+        includeClasses.add("org.opensearch.index.shard.IndexShardIT")
+        includeClasses.add("org.opensearch.index.shard.RemoteStoreRefreshListenerTests")
+        includeClasses.add("org.opensearch.index.translog.RemoteFSTranslogTests")
+        includeClasses.add("org.opensearch.indices.DateMathIndexExpressionsIntegrationIT")
+        includeClasses.add("org.opensearch.indices.replication.RemoteStoreReplicationSourceTests")
+        includeClasses.add("org.opensearch.indices.replication.SegmentReplicationAllocationIT")
+        includeClasses.add("org.opensearch.indices.replication.SegmentReplicationIT")
+        includeClasses.add("org.opensearch.indices.replication.SegmentReplicationRelocationIT")
+        includeClasses.add("org.opensearch.indices.replication.SegmentReplicationTargetServiceTests")
+        includeClasses.add("org.opensearch.indices.state.CloseWhileRelocatingShardsIT")
+        includeClasses.add("org.opensearch.monitor.fs.FsHealthServiceTests")
+        includeClasses.add("org.opensearch.recovery.ReplicationCollectionTests")
+        includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep")
+        includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT")
+        includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteStoreIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteStoreStatsIT")
+        includeClasses.add("org.opensearch.remotestore.SegmentReplicationRemoteStoreIT")
+        includeClasses.add("org.opensearch.remotestore.SegmentReplicationUsingRemoteStoreIT")
+        includeClasses.add("org.opensearch.remotestore.multipart.RemoteStoreMultipartIT")
+        includeClasses.add("org.opensearch.repositories.azure.AzureBlobContainerRetriesTests")
+        includeClasses.add("org.opensearch.repositories.azure.AzureBlobStoreRepositoryTests")
+        includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobContainerRetriesTests")
+        includeClasses.add("org.opensearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests")
+        includeClasses.add("org.opensearch.repositories.s3.S3BlobStoreRepositoryTests")
+        includeClasses.add("org.opensearch.search.ConcurrentSegmentSearchTimeoutIT")
+        includeClasses.add("org.opensearch.search.SearchTimeoutIT")
+        includeClasses.add("org.opensearch.search.SearchWeightedRoutingIT")
+        includeClasses.add("org.opensearch.search.aggregations.bucket.DoubleTermsIT")
+        includeClasses.add("org.opensearch.search.aggregations.bucket.terms.StringTermsIT")
+        includeClasses.add("org.opensearch.search.aggregations.metrics.CardinalityIT")
+        includeClasses.add("org.opensearch.search.backpressure.SearchBackpressureIT")
+        includeClasses.add("org.opensearch.search.basic.SearchWithRandomIOExceptionsIT")
+        includeClasses.add("org.opensearch.search.pit.DeletePitMultiNodeIT")
+        includeClasses.add("org.opensearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT")
+        includeClasses.add("org.opensearch.snapshots.CloneSnapshotIT")
+        includeClasses.add("org.opensearch.snapshots.DedicatedClusterSnapshotRestoreIT")
+        includeClasses.add("org.opensearch.snapshots.RestoreSnapshotIT")
+        includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT")
+        includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT")
+        includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests")
+      }
     }
   }
 }

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
@@ -75,9 +75,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.19+7";
+    private static final String SYSTEM_JDK_VERSION = "11.0.20+8";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.7+7";
+    private static final String GRADLE_JDK_VERSION = "17.0.8+7";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,8 +1,8 @@
 opensearch        = 3.0.0
-lucene            = 9.7.0
+lucene            = 9.8.0-snapshot-4373c3b
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 20.0.1+9
+bundled_jdk = 20.0.2+9
 # See please https://github.com/adoptium/temurin-build/issues/3371
 bundled_jdk_linux_ppc64le = 20+36
 

diff --git a/...p-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/...p-api-plugin/src/main/java/org/opensearch/plugin/noop/action/bulk/RestNoopBulkAction.java
@@ -99,7 +99,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             defaultPipeline,
             defaultRequireAlias,
             true,
-            request.getXContentType()
+            request.getMediaType()
         );
 
         // short circuit the call to the transport layer