Merge branch 'main' into main-telemetry-mock

Signed-off-by: Gagan Juneja <[email protected]>

.github/workflows/version.yml

@@ -30,7 +30,11 @@ jobs:
           CURRENT_VERSION_ARRAY[2]=$((CURRENT_VERSION_ARRAY[2]+1))
           NEXT_VERSION=$(IFS=. ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}")
           NEXT_VERSION_UNDERSCORE=$(IFS=_ ; echo "V_${CURRENT_VERSION_ARRAY[*]:0:3}")
-          NEXT_VERSION_ID=$(IFS=0 ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}99")
+          if [[ ${#CURRENT_VERSION_ARRAY[2]} -gt 1 ]]; then
+            NEXT_VERSION_ID="${CURRENT_VERSION_ARRAY[0]:0:3}0${CURRENT_VERSION_ARRAY[1]:0:3}${CURRENT_VERSION_ARRAY[2]:0:3}99"
+          else
+            NEXT_VERSION_ID=$(IFS=0 ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}99")
+          fi
           echo "TAG=$TAG" >> $GITHUB_ENV
           echo "BASE=$BASE" >> $GITHUB_ENV
           echo "BASE_X=$BASE_X" >> $GITHUB_ENV

CHANGELOG.md

@@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
 - Pass localNode info to all plugins on node start ([#7919](https://github.com/opensearch-project/OpenSearch/pull/7919))
+- Improved performance of parsing floating point numbers ([#7909](https://github.com/opensearch-project/OpenSearch/pull/7909))
 
 ### Deprecated
 
@@ -75,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460))
 - Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230))
 - Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541))
+- Adds log4j configuration for telemetry LogSpanExporter ([#8393](https://github.com/opensearch-project/OpenSearch/pull/8393))
 
 ### Security
 
@@ -92,6 +94,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038))
 - Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029)
 - Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543))
+- Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168))
+- [Extensions] Support extension additional settings with extension REST initialization ([#8414](https://github.com/opensearch-project/OpenSearch/pull/8414))
 - Adds mock implementation for TelemetryPlugin ([#7545](https://github.com/opensearch-project/OpenSearch/issues/7545))
 
 ### Dependencies
@@ -125,6 +129,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Upgrade] Lucene 9.7.0 release (#8272)
 - Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209))
 - Bump `com.google.api-client:google-api-client` from 1.34.0 to 2.2.0 ([#8276](https://github.com/opensearch-project/OpenSearch/pull/8276))
+- Update Apache HttpCore/ HttpClient and Apache HttpCore5 / HttpClient5 dependencies ([#8434](https://github.com/opensearch-project/OpenSearch/pull/8434))
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
@@ -156,6 +161,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - With only GlobalAggregation in request causes unnecessary wrapping with MultiCollector ([#8125](https://github.com/opensearch-project/OpenSearch/pull/8125))
 - Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591))
 - Fix NPE in multiterms aggregations involving empty buckets ([#7318](https://github.com/opensearch-project/OpenSearch/pull/7318))
+- Precise system clock time in MasterService debug logs ([#7902](https://github.com/opensearch-project/OpenSearch/pull/7902))
 
 ### Security
 

buildSrc/version.properties

@@ -33,10 +33,10 @@ netty             = 4.1.94.Final
 joda              = 2.12.2
 
 # client dependencies
-httpclient5       = 5.1.4
-httpcore5         = 5.1.5
-httpclient        = 4.5.13
-httpcore          = 4.4.15
+httpclient5       = 5.2.1
+httpcore5         = 5.2.2
+httpclient        = 4.5.14
+httpcore          = 4.4.16
 httpasyncclient   = 4.1.5
 commonslogging    = 1.2
 commonscodec      = 1.15

client/rest/licenses/httpclient5-5.2.1.jar.sha1

@@ -0,0 +1 @@
+0c900514d3446d9ce5d9dbd90c21192048125440

client/rest/licenses/httpcore5-5.2.2.jar.sha1

@@ -0,0 +1 @@
+6da28f5aa6c2b129ef49632e041a5203ce7507b2

client/rest/licenses/httpcore5-h2-5.2.2.jar.sha1

@@ -0,0 +1 @@
+54ee1ed58fe8ac40be1083ea9873a6c734939ab9

client/sniffer/licenses/httpclient5-5.2.1.jar.sha1

@@ -0,0 +1 @@
+0c900514d3446d9ce5d9dbd90c21192048125440

client/sniffer/licenses/httpcore5-5.2.2.jar.sha1

@@ -0,0 +1 @@
+6da28f5aa6c2b129ef49632e041a5203ce7507b2

libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java

@@ -36,6 +36,7 @@
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.StreamReadConstraints;
+import com.fasterxml.jackson.core.StreamReadFeature;
 import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
 import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.MediaType;
@@ -75,6 +76,7 @@ public static XContentBuilder contentBuilder() throws IOException {
         cborFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
         cborFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
         cborFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build());
+        cborFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true);
         cborXContent = new CborXContent();
     }
 

libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java

@@ -37,7 +37,7 @@
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.StreamReadConstraints;
-
+import com.fasterxml.jackson.core.StreamReadFeature;
 import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
@@ -78,6 +78,7 @@ public static XContentBuilder contentBuilder() throws IOException {
         jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
         jsonFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
         jsonFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build());
+        jsonFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true);
         jsonXContent = new JsonXContent();
     }
 

libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java

@@ -36,6 +36,7 @@
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.StreamReadConstraints;
+import com.fasterxml.jackson.core.StreamReadFeature;
 import com.fasterxml.jackson.dataformat.smile.SmileFactory;
 import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
 import org.opensearch.core.xcontent.DeprecationHandler;
@@ -77,6 +78,7 @@ public static XContentBuilder contentBuilder() throws IOException {
         smileFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
         smileFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
         smileFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build());
+        smileFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true);
         smileXContent = new SmileXContent();
     }
 

libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java

@@ -35,6 +35,7 @@
 import com.fasterxml.jackson.core.JsonEncoding;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.StreamReadConstraints;
+import com.fasterxml.jackson.core.StreamReadFeature;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
@@ -70,6 +71,7 @@ public static XContentBuilder contentBuilder() throws IOException {
         yamlFactory = new YAMLFactory();
         yamlFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
         yamlFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build());
+        yamlFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true);
         yamlXContent = new YamlXContent();
     }
 

plugins/discovery-azure-classic/licenses/httpclient-4.5.14.jar.sha1

@@ -0,0 +1 @@
+1194890e6f56ec29177673f2f12d0b8e627dec98

plugins/discovery-azure-classic/licenses/httpcore-4.4.16.jar.sha1

@@ -0,0 +1 @@
+51cf043c87253c9f58b539c9f7e44c8894223850

plugins/discovery-ec2/licenses/httpclient-4.5.14.jar.sha1

@@ -0,0 +1 @@
+1194890e6f56ec29177673f2f12d0b8e627dec98

plugins/discovery-ec2/licenses/httpcore-4.4.16.jar.sha1

@@ -0,0 +1 @@
+51cf043c87253c9f58b539c9f7e44c8894223850

plugins/discovery-gce/licenses/httpclient-4.5.14.jar.sha1

@@ -0,0 +1 @@
+1194890e6f56ec29177673f2f12d0b8e627dec98

plugins/discovery-gce/licenses/httpcore-4.4.16.jar.sha1

@@ -0,0 +1 @@
+51cf043c87253c9f58b539c9f7e44c8894223850

plugins/repository-s3/licenses/httpclient-4.5.14.jar.sha1

@@ -0,0 +1 @@
+1194890e6f56ec29177673f2f12d0b8e627dec98

plugins/repository-s3/licenses/httpcore-4.4.16.jar.sha1

@@ -0,0 +1 @@
+51cf043c87253c9f58b539c9f7e44c8894223850

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java

@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.ExceptionsHelper;
+import org.opensearch.action.ActionListener;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.SetOnce;
 import org.opensearch.common.blobstore.BlobContainer;
@@ -296,6 +297,35 @@ private static DeleteObjectsRequest bulkDelete(String bucket, List<String> blobs
             .build();
     }
 
+    @Override
+    public void listBlobsByPrefixInSortedOrder(
+        String blobNamePrefix,
+        int limit,
+        BlobNameSortOrder blobNameSortOrder,
+        ActionListener<List<BlobMetadata>> listener
+    ) {
+        // As AWS S3 returns list of keys in Lexicographic order, we don't have to fetch all the keys in order to sort them
+        // We fetch only keys as per the given limit to optimize the fetch. If provided sort order is not Lexicographic,
+        // we fall-back to default implementation of fetching all the keys and sorting them.
+        if (blobNameSortOrder != BlobNameSortOrder.LEXICOGRAPHIC) {
+            super.listBlobsByPrefixInSortedOrder(blobNamePrefix, limit, blobNameSortOrder, listener);
+        } else {
+            if (limit < 0) {
+                throw new IllegalArgumentException("limit should not be a negative value");
+            }
+            String prefix = blobNamePrefix == null ? keyPath : buildKey(blobNamePrefix);
+            try (AmazonS3Reference clientReference = blobStore.clientReference()) {
+                List<BlobMetadata> blobs = executeListing(clientReference, listObjectsRequest(prefix, limit), limit).stream()
+                    .flatMap(listing -> listing.contents().stream())
+                    .map(s3Object -> new PlainBlobMetadata(s3Object.key().substring(keyPath.length()), s3Object.size()))
+                    .collect(Collectors.toList());
+                listener.onResponse(blobs.subList(0, Math.min(limit, blobs.size())));
+            } catch (final Exception e) {
+                listener.onFailure(new IOException("Exception when listing blobs by prefix [" + prefix + "]", e));
+            }
+        }
+    }
+
     @Override
     public Map<String, BlobMetadata> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
         String prefix = blobNamePrefix == null ? keyPath : buildKey(blobNamePrefix);
@@ -339,10 +369,25 @@ public Map<String, BlobContainer> children() throws IOException {
     }
 
     private static List<ListObjectsV2Response> executeListing(AmazonS3Reference clientReference, ListObjectsV2Request listObjectsRequest) {
+        return executeListing(clientReference, listObjectsRequest, -1);
+    }
+
+    private static List<ListObjectsV2Response> executeListing(
+        AmazonS3Reference clientReference,
+        ListObjectsV2Request listObjectsRequest,
+        int limit
+    ) {
         return SocketAccess.doPrivileged(() -> {
             final List<ListObjectsV2Response> results = new ArrayList<>();
+            int totalObjects = 0;
             ListObjectsV2Iterable listObjectsIterable = clientReference.get().listObjectsV2Paginator(listObjectsRequest);
-            listObjectsIterable.forEach(results::add);
+            for (ListObjectsV2Response listObjectsV2Response : listObjectsIterable) {
+                results.add(listObjectsV2Response);
+                totalObjects += listObjectsV2Response.contents().size();
+                if (limit != -1 && totalObjects > limit) {
+                    break;
+                }
+            }
             return results;
         });
     }
@@ -356,6 +401,10 @@ private ListObjectsV2Request listObjectsRequest(String keyPath) {
             .build();
     }
 
+    private ListObjectsV2Request listObjectsRequest(String keyPath, int limit) {
+        return listObjectsRequest(keyPath).toBuilder().maxKeys(Math.min(limit, 1000)).build();
+    }
+
     private String buildKey(String blobName) {
         return keyPath + blobName;
     }