Merge branch '2.x' into backport/backport-6381-to-2.x

CHANGELOG.md

@@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Segment Replication] Add primary weight factor for balanced primary distribution ([#6017](https://github.com/opensearch-project/OpenSearch/pull/6017))
 - Add a setting to control auto release of OpenSearch managed index creation block ([#6277](https://github.com/opensearch-project/OpenSearch/pull/6277))
 - Fix timeout error when adding a document to an index with extension running ([#6275](https://github.com/opensearch-project/OpenSearch/pull/6275))
+- Handle translog upload during primary relocation for remote-backed indexes ([#5804](https://github.com/opensearch-project/OpenSearch/pull/5804))
+- Batch translog sync/upload per x ms for remote-backed indexes ([#5854](https://github.com/opensearch-project/OpenSearch/pull/5854))
 
 ### Dependencies
 - Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704))
@@ -37,14 +39,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumps `com.azure:azure-storage-common` from 12.18.1 to 12.19.3 ([#6304](https://github.com/opensearch-project/OpenSearch/pull/6304))
 - Bumps `reactor-netty-http` from 1.0.24 to 1.1.2 ([#6309](https://github.com/opensearch-project/OpenSearch/pull/6309))
 - Bumps `com.google.protobuf:protobuf-java` from 3.21.12 to 3.22.0 ([#6387](https://github.com/opensearch-project/OpenSearch/pull/6387))
+- Bumps `io.projectreactor.netty:reactor-netty` from 1.1.2 to 1.1.3 ([#6386](https://github.com/opensearch-project/OpenSearch/pull/6386))
+- Bumps `com.diffplug.spotless` from 6.10.0 to 6.15.0 ([#6385](https://github.com/opensearch-project/OpenSearch/pull/6385))
+- Bumps `jettison` from 1.5.1 to 1.5.3 ([#6391](https://github.com/opensearch-project/OpenSearch/pull/6391))
 
 ### Changed
 - Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725))
 - [Refactor] Use local opensearch.common.SetOnce instead of lucene's utility class ([#5947](https://github.com/opensearch-project/OpenSearch/pull/5947))
 - Cluster health call to throw decommissioned exception for local decommissioned node([#6008](https://github.com/opensearch-project/OpenSearch/pull/6008))
 - [Refactor] core.common to new opensearch-common library ([#5976](https://github.com/opensearch-project/OpenSearch/pull/5976))
-- Cluster local health call to throw exception if node is decommissioned or weighed away ([#6198](https://github.com/opensearch-project/OpenSearch/pull/6198))
+- Update API spec for cluster health API ([#6399](https://github.com/opensearch-project/OpenSearch/pull/6399))
 - Remove deprecated org.gradle.util.DistributionLocator usage ([#6212](https://github.com/opensearch-project/OpenSearch/pull/6212))
+- Require MediaType in Strings.toString API ([#6009](https://github.com/opensearch-project/OpenSearch/pull/6009))
 
 ### Deprecated
 
@@ -55,6 +61,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix weighted shard routing state across search requests([#6004](https://github.com/opensearch-project/OpenSearch/pull/6004))
 - [Segment Replication] Fix bug where inaccurate sequence numbers are sent during replication ([#6122](https://github.com/opensearch-project/OpenSearch/pull/6122))
 - Enable numeric sort optimisation for few numerical sort types ([#6321](https://github.com/opensearch-project/OpenSearch/pull/6321))
+- Fix Opensearch repository-s3 plugin cannot read ServiceAccount token ([#6390](https://github.com/opensearch-project/OpenSearch/pull/6390)
 
 ### Security
 

build.gradle

@@ -54,7 +54,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.11.0" apply false
+  id "com.diffplug.spotless" version "6.15.0" apply false
   id "org.gradle.test-retry" version "1.5.1" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

buildSrc/version.properties

@@ -16,7 +16,7 @@ supercsv          = 2.4.0
 log4j             = 2.17.1
 slf4j             = 1.7.36
 asm               = 9.4
-jettison          = 1.5.1
+jettison          = 1.5.3
 woodstox          = 6.4.0
 kotlin            = 1.7.10
 antlr4            = 4.11.1

client/rest-high-level/src/main/java/org/opensearch/client/slm/SnapshotLifecyclePolicy.java

@@ -39,6 +39,7 @@
 import org.opensearch.common.xcontent.ToXContentObject;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentParser;
+import org.opensearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Map;
@@ -168,6 +169,6 @@ public boolean equals(Object obj) {
 
     @Override
     public String toString() {
-        return Strings.toString(this);
+        return Strings.toString(XContentType.JSON, this);
     }
 }

client/rest-high-level/src/main/java/org/opensearch/client/slm/SnapshotLifecyclePolicyMetadata.java

@@ -39,6 +39,7 @@
 import org.opensearch.common.xcontent.ToXContentObject;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentParser;
+import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.snapshots.SnapshotId;
 
 import java.io.IOException;
@@ -288,7 +289,7 @@ public boolean equals(Object obj) {
 
         @Override
         public String toString() {
-            return Strings.toString(this);
+            return Strings.toString(XContentType.JSON, this);
         }
     }
 }

client/rest-high-level/src/main/java/org/opensearch/client/slm/SnapshotLifecycleStats.java

@@ -40,6 +40,7 @@
 import org.opensearch.common.xcontent.ToXContentObject;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentParser;
+import org.opensearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -187,7 +188,7 @@ public boolean equals(Object obj) {
 
     @Override
     public String toString() {
-        return Strings.toString(this);
+        return Strings.toString(XContentType.JSON, this);
     }
 
     public static class SnapshotPolicyStats implements ToXContentFragment {

client/rest-high-level/src/main/java/org/opensearch/client/slm/SnapshotRetentionConfiguration.java

@@ -40,6 +40,7 @@
 import org.opensearch.common.xcontent.ToXContentObject;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentParser;
+import org.opensearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -150,6 +151,6 @@ public boolean equals(Object obj) {
 
     @Override
     public String toString() {
-        return Strings.toString(this);
+        return Strings.toString(XContentType.JSON, this);
     }
 }

distribution/archives/integ-test-zip/src/test/java/org/opensearch/test/rest/NodeRestUsageIT.java

@@ -36,6 +36,7 @@
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
 import org.opensearch.common.Strings;
+import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.search.aggregations.AggregationBuilders;
 import org.opensearch.search.builder.SearchSourceBuilder;
 
@@ -171,8 +172,8 @@ public void testAggregationUsage() throws IOException {
             .aggregation(AggregationBuilders.terms("str_terms").field("str.keyword"))
             .aggregation(AggregationBuilders.terms("num_terms").field("num"))
             .aggregation(AggregationBuilders.avg("num_avg").field("num"));
-        searchRequest.setJsonEntity(Strings.toString(searchSource));
-        searchRequest.setJsonEntity(Strings.toString(searchSource));
+        searchRequest.setJsonEntity(Strings.toString(XContentType.JSON, searchSource));
+        searchRequest.setJsonEntity(Strings.toString(XContentType.JSON, searchSource));
         client().performRequest(searchRequest);
 
         searchRequest = new Request("GET", "/test/_search");
@@ -181,8 +182,8 @@ public void testAggregationUsage() throws IOException {
             .aggregation(AggregationBuilders.avg("num1").field("num"))
             .aggregation(AggregationBuilders.avg("num2").field("num"))
             .aggregation(AggregationBuilders.terms("foo").field("foo.keyword"));
-        String r = Strings.toString(searchSource);
-        searchRequest.setJsonEntity(Strings.toString(searchSource));
+        String r = Strings.toString(XContentType.JSON, searchSource);
+        searchRequest.setJsonEntity(Strings.toString(XContentType.JSON, searchSource));
         client().performRequest(searchRequest);
 
         Response response = client().performRequest(new Request("GET", "_nodes/usage"));

libs/x-content/src/main/java/org/opensearch/common/xcontent/MediaType.java

@@ -62,4 +62,6 @@ public interface MediaType {
     default String typeWithSubtype() {
         return type() + "/" + subtype();
     }
+
+    XContent xContent();
 }

libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentType.java

@@ -190,8 +190,6 @@ public String mediaType() {
         return mediaTypeWithoutParameters();
     }
 
-    public abstract XContent xContent();
-
     public abstract String mediaTypeWithoutParameters();
 
     @Override

libs/x-content/src/test/java/org/opensearch/common/xcontent/SimpleStruct.java

@@ -101,6 +101,6 @@ public int hashCode() {
 
     @Override
     public String toString() {
-        return Strings.toString(this);
+        return Strings.toString(XContentType.JSON, this);
     }
 }

modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java

@@ -46,6 +46,7 @@
 import org.opensearch.common.xcontent.ToXContentObject;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentParser;
+import org.opensearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -206,6 +207,6 @@ public static MultiSearchTemplateResponse fromXContext(XContentParser parser) {
 
     @Override
     public String toString() {
-        return Strings.toString(this);
+        return Strings.toString(XContentType.JSON, this);
     }
 }

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java

@@ -53,6 +53,7 @@
 import org.opensearch.common.Strings;
 import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.opensearch.common.xcontent.XContentBuilder;
+import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.analysis.AnalyzerScope;
 import org.opensearch.index.analysis.IndexAnalyzers;
@@ -609,15 +610,15 @@ public void testAnalyzerSerialization() throws IOException {
             b.field("type", "search_as_you_type");
             b.field("analyzer", "simple");
         }));
-        String serialized = Strings.toString(ms.documentMapper());
+        String serialized = Strings.toString(XContentType.JSON, ms.documentMapper());
         assertEquals(
             serialized,
             "{\"_doc\":{\"properties\":{\"field\":"
                 + "{\"type\":\"search_as_you_type\",\"doc_values\":false,\"max_shingle_size\":3,\"analyzer\":\"simple\"}}}}"
         );
 
         merge(ms, mapping(b -> {}));
-        assertEquals(serialized, Strings.toString(ms.documentMapper()));
+        assertEquals(serialized, Strings.toString(XContentType.JSON, ms.documentMapper()));
     }
 
     private void documentParsingTestCase(Collection<String> values) throws IOException {

modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalResponse.java

@@ -44,6 +44,7 @@
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentParser;
 import org.opensearch.common.xcontent.XContentParserUtils;
+import org.opensearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -105,7 +106,7 @@ public Map<String, Exception> getFailures() {
 
     @Override
     public String toString() {
-        return Strings.toString(this);
+        return Strings.toString(XContentType.JSON, this);
     }
 
     @Override

modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RankEvalSpec.java

@@ -43,6 +43,7 @@
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentParser;
 import org.opensearch.common.xcontent.XContentParserUtils;
+import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.script.Script;
 
 import java.io.IOException;
@@ -249,7 +250,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public String toString() {
-        return Strings.toString(this);
+        return Strings.toString(XContentType.JSON, this);
     }
 
     @Override

modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedDocument.java

@@ -41,6 +41,7 @@
 import org.opensearch.common.xcontent.ToXContentObject;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentParser;
+import org.opensearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -128,7 +129,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public String toString() {
-        return Strings.toString(this);
+        return Strings.toString(XContentType.JSON, this);
     }
 
     @Override

modules/rank-eval/src/main/java/org/opensearch/index/rankeval/RatedRequest.java

@@ -42,6 +42,7 @@
 import org.opensearch.common.xcontent.ToXContentObject;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentParser;
+import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.index.rankeval.RatedDocument.DocumentKey;
 import org.opensearch.search.builder.SearchSourceBuilder;
 
@@ -340,7 +341,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public String toString() {
-        return Strings.toString(this);
+        return Strings.toString(XContentType.JSON, this);
     }
 
     @Override

modules/rank-eval/src/test/java/org/opensearch/index/rankeval/DiscountedCumulativeGainTests.java

@@ -323,10 +323,13 @@ public void testMetricDetails() {
                     + ",\"unrated_docs\":"
                     + unratedDocs
                     + "}}",
-                Strings.toString(detail)
+                Strings.toString(XContentType.JSON, detail)
             );
         } else {
-            assertEquals("{\"dcg\":{\"dcg\":" + dcg + ",\"unrated_docs\":" + unratedDocs + "}}", Strings.toString(detail));
+            assertEquals(
+                "{\"dcg\":{\"dcg\":" + dcg + ",\"unrated_docs\":" + unratedDocs + "}}",
+                Strings.toString(XContentType.JSON, detail)
+            );
         }
     }
 

plugins/discovery-azure-classic/licenses/jettison-1.5.3.jar.sha1

@@ -0,0 +1 @@
+964d35bbdecbbc33cf2f2044e8a1648d9f6f1474

plugins/repository-azure/build.gradle

@@ -57,9 +57,9 @@ dependencies {
   api 'com.azure:azure-storage-blob:12.15.0'
   api 'org.reactivestreams:reactive-streams:1.0.3'
   api 'io.projectreactor:reactor-core:3.4.18'
-  api 'io.projectreactor.netty:reactor-netty:1.1.2'
+  api 'io.projectreactor.netty:reactor-netty:1.1.3'
   api 'io.projectreactor.netty:reactor-netty-core:1.0.24'
-  api 'io.projectreactor.netty:reactor-netty-http:1.1.2'
+  api 'io.projectreactor.netty:reactor-netty-http:1.1.3'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"

plugins/repository-azure/licenses/reactor-netty-1.1.3.jar.sha1

@@ -0,0 +1 @@
+2745c13811d82b7c57d4ca52764bbaab83e3e2e2

plugins/repository-azure/licenses/reactor-netty-http-1.1.3.jar.sha1

@@ -0,0 +1 @@
+ddebf6e059e10315e9a53417289c98745a8fc7a7

plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -68,6 +68,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -213,8 +214,8 @@ public void testEnforcedCooldownPeriod() throws IOException {
      */
     public static class TestS3RepositoryPlugin extends S3RepositoryPlugin {
 
-        public TestS3RepositoryPlugin(final Settings settings) {
-            super(settings);
+        public TestS3RepositoryPlugin(final Settings settings, final Path configPath) {
+            super(settings, configPath);
         }
 
         @Override