From 1f2a00d4cfac40f62ca6e0c37b133adad2a807f9 Mon Sep 17 00:00:00 2001 From: dblock Date: Wed, 11 Jan 2023 15:45:10 -0500 Subject: [PATCH] Added support for signing service name. Signed-off-by: dblock --- CHANGELOG.md | 4 +- DEVELOPER_GUIDE.md | 15 +++ USER_GUIDE.md | 31 ++++- java-client/build.gradle.kts | 2 + .../transport/aws/AwsSdk2Transport.java | 116 ++++++++++++++---- .../aws/AwsSdk2TransportTestCase.java | 11 ++ 6 files changed, 153 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07e7abc4de..578b2604c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,13 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] + ### Added - Github workflow for changelog verification ([#239](https://github.com/opensearch-project/opensearch-java/pull/239)) - Github workflow for dependabot PRs ([#247](https://github.com/opensearch-project/opensearch-java/pull/247)) - Add javadoc link for the client ([#255](https://github.com/opensearch-project/opensearch-java/pull/255)) +- Add support for signing service name in AwsSdk2Transport ([#324](https://github.com/opensearch-project/opensearch-java/pull/324)) + ### Dependencies - Bumps `classgraph` from 4.8.149 to 4.8.154 @@ -37,5 +40,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Security - [Unreleased]: https://github.com/opensearch-project/opensearch-java/compare/2.0...HEAD \ No newline at end of file diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index fa2da6946e..e26bedb875 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -7,6 +7,7 @@ - [Run Tests](#run-tests) - [Unit Tests](#unit-tests) - [Integration Tests](#integration-tests) + - [AWS Transport Integration Tests](#aws-transport-integration-tests) - [Use an Editor](#use-an-editor) - [IntelliJ IDEA](#intellij-idea) - [Visual Studio Code](#visual-studio-code) @@ -64,6 +65,20 @@ Run integration tests after starting OpenSearch cluster: ./gradlew clean integrationTest ``` +#### AWS Transport Integration Tests + +To run integration tests for the AWS transport client, ensure working AWS credentials and specify your OpenSearch domain and region as follows: + +``` +./gradlew integrationTest --tests "*AwsSdk2*" -Dtests.awsSdk2support.domainHost=search-...us-west-2.es.amazonaws.com -Dtests.awsSdk2support.domainRegion=us-west-2 +``` + +For OpenSearch Serverless, change the signing service name. + +``` +./gradlew integrationTest --tests "*AwsSdk2*" -Dtests.awsSdk2support.domainHost=....us-west-2.aoss.amazonaws.com -Dtests.awsSdk2support.domainRegion=us-west-2 -Dtests.awsSdk2support.serviceName=aoss +``` + ## Use an Editor ### IntelliJ IDEA diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 6cd7b04b07..a11709e716 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2,15 +2,17 @@ - [User Guide](#user-guide) - [Sample data](#sample-data) + - [IndexData class](#indexdata-class) - [Create a client](#create-a-client) - [Create an index](#create-an-index) - [Index data](#index-data) - - [Search for the document](#search-for-the-documents) + - [Search for the documents](#search-for-the-documents) - [Search documents using a match query](#search-documents-using-a-match-query) - [Aggregations](#aggregations) - [Delete the document](#delete-the-document) - [Delete the index](#delete-the-index) - - [Aggregations](#aggregations) +- [Using different transport options](#using-different-transport-options) + - [Amazon Managed OpenSearch](#amazon-managed-opensearch) ## Sample data @@ -136,3 +138,28 @@ client.delete(d -> d.index(index).id("1")); DeleteIndexRequest deleteIndexRequest = new DeleteRequest.Builder().index(index).build(); DeleteIndexResponse deleteIndexResponse = client.indices().delete(deleteIndexRequest); ``` + +# Using different transport options + +## Amazon Managed OpenSearch + +Use `AwsSdk2Transport` to make requests to Amazon Managed OpenSearch. + +```java +SdkHttpClient httpClient = ApacheHttpClient.builder().build(); + +OpenSearchClient client = new OpenSearchClient( + new AwsSdk2Transport( + httpClient, + "search-...us-west-2.es.amazonaws.com", // OpenSearch endpoint, without https:// + "es" // signing service name + Region.US_WEST_2, // signing service region + AwsSdk2TransportOptions.builder().build() + ) +); + +InfoResponse info = client.info(); +System.out.println(info.version().distribution() + ": " + info.version().number()); + +httpClient.close(); +``` diff --git a/java-client/build.gradle.kts b/java-client/build.gradle.kts index 81e04806b1..43658ab413 100644 --- a/java-client/build.gradle.kts +++ b/java-client/build.gradle.kts @@ -135,6 +135,8 @@ val integrationTest = task("integrationTest") { systemProperty("password", System.getProperty("password", "admin")) systemProperty("tests.awsSdk2support.domainHost", System.getProperty("tests.awsSdk2support.domainHost", null)) + systemProperty("tests.awsSdk2support.serviceName", + System.getProperty("tests.awsSdk2support.serviceName", "es")) systemProperty("tests.awsSdk2support.domainRegion", System.getProperty("tests.awsSdk2support.domainRegion", "us-east-1")) } diff --git a/java-client/src/main/java/org/opensearch/client/transport/aws/AwsSdk2Transport.java b/java-client/src/main/java/org/opensearch/client/transport/aws/AwsSdk2Transport.java index 1069cf7832..af55aa02f7 100644 --- a/java-client/src/main/java/org/opensearch/client/transport/aws/AwsSdk2Transport.java +++ b/java-client/src/main/java/org/opensearch/client/transport/aws/AwsSdk2Transport.java @@ -60,12 +60,12 @@ /** * Implementation of the OpenSearchTransport interface that sends signed requests using - * the AWS v2 SDK HTTP clients, to connect to an AWS OpenSearch service using IAM authentication + * the AWS v2 SDK HTTP clients, to connect to an AWS OpenSearch service using IAM authentication. */ public class AwsSdk2Transport implements OpenSearchTransport { /** * By default, requests that exceed this size will be automatically compressed. - * {@link AwsSdk2TransportOptions} can be used to override this setting or disable compresson. + * {@link AwsSdk2TransportOptions} can be used to override this setting or disable compression. */ public static final Integer DEFAULT_REQUEST_COMPRESSION_SIZE = 8192; @@ -73,21 +73,21 @@ public class AwsSdk2Transport implements OpenSearchTransport { private final SdkHttpClient httpClient; private final SdkAsyncHttpClient asyncHttpClient; private final String host; + private final String signingServiceName; private final Region signingRegion; private final JsonpMapper defaultMapper; private final AwsSdk2TransportOptions transportOptions; /** - * Create an {@link OpenSearchTransport} with a SYNCHRONOUS AWS Http client + * Create an {@link OpenSearchTransport} with a synchronous AWS HTTP client. *

* Note that asynchronous OpenSearch requests sent through this transport will be dispatched * *synchronously* on the calling thread. * - * @param httpClient HTTP client to use for OpenSearch requests - * @param host The fully qualified domain name to connect to - * @param signingRegion The AWS region for which requests will be signed. This should typically match - * the region in `host`. - * @param options Options that apply to all requests. Can be null. Create with + * @param httpClient HTTP client to use for OpenSearch requests. + * @param host The fully qualified domain name to connect to. + * @param signingRegion The AWS region for which requests will be signed. This should typically match the region in `host`. + * @param options Options that apply to all requests. Can be null. Create with * {@link AwsSdk2TransportOptions#builder()} and use these to specify non-default credentials, * compression options, etc. */ @@ -96,20 +96,42 @@ public AwsSdk2Transport( @Nonnull String host, @Nonnull Region signingRegion, @CheckForNull AwsSdk2TransportOptions options) { - this(httpClient, null, host, signingRegion, options); + this(httpClient, null, host, "es", signingRegion, options); } /** - * Create an {@link OpenSearchTransport} with an ASYNCHRONOUS AWS Http client + * Create an {@link OpenSearchTransport} with a synchronous AWS HTTP client. + *

+ * Note that asynchronous OpenSearch requests sent through this transport will be dispatched + * *synchronously* on the calling thread. + * + * @param httpClient HTTP client to use for OpenSearch requests. + * @param host The fully qualified domain name to connect to. + * @param signingServiceName The AWS signing service name, one of `es` (Amazon OpenSearch) or `aoss` (Amazon OpenSearch Serverless). + * @param signingRegion The AWS region for which requests will be signed. This should typically match the region in `host`. + * @param options Options that apply to all requests. Can be null. Create with + * {@link AwsSdk2TransportOptions#builder()} and use these to specify non-default credentials, + * compression options, etc. + */ + public AwsSdk2Transport( + @Nonnull SdkHttpClient httpClient, + @Nonnull String host, + @Nonnull String signingServiceName, + @Nonnull Region signingRegion, + @CheckForNull AwsSdk2TransportOptions options) { + this(httpClient, null, host, signingServiceName, signingRegion, options); + } + + /** + * Create an {@link OpenSearchTransport} with an asynchronous AWS HTTP client *

* Note that synchronous OpenSearch requests sent through this transport will be dispatched * using the asynchronous client, but the calling thread will block until they are complete. * - * @param asyncHttpClient HTTP client to use for OpenSearch requests - * @param host The target host - * @param signingRegion The AWS region for which requests will be signed. This should typically match - * the region in `host`. - * @param options Options that apply to all requests. Can be null. Create with + * @param asyncHttpClient HTTP client to use for OpenSearch requests. + * @param host The target host. + * @param signingRegion The AWS region for which requests will be signed. This should typically match region in `host`. + * @param options Options that apply to all requests. Can be null. Create with * {@link AwsSdk2TransportOptions#builder()} and use these to specify non-default credentials, * compression options, etc. */ @@ -118,7 +140,53 @@ public AwsSdk2Transport( @Nonnull String host, @Nonnull Region signingRegion, @CheckForNull AwsSdk2TransportOptions options) { - this(null, asyncHttpClient, host, signingRegion, options); + this(null, asyncHttpClient, host, "es", signingRegion, options); + } + + /** + * Create an {@link OpenSearchTransport} with an asynchronous AWS HTTP client. + *

+ * Note that synchronous OpenSearch requests sent through this transport will be dispatched + * using the asynchronous client, but the calling thread will block until they are complete. + * + * @param asyncHttpClient HTTP client to use for OpenSearch requests. + * @param host The target host. + * @param signingServiceName The AWS signing service name, one of `es` (Amazon OpenSearch) or `aoss` (Amazon OpenSearch Serverless). + * @param signingRegion The AWS region for which requests will be signed. This should typically match the region in `host`. + * @param options Options that apply to all requests. Can be null. Create with + * {@link AwsSdk2TransportOptions#builder()} and use these to specify non-default credentials, + * compression options, etc. + */ + public AwsSdk2Transport( + @Nonnull SdkAsyncHttpClient asyncHttpClient, + @Nonnull String host, + @Nonnull String signingServiceName, + @Nonnull Region signingRegion, + @CheckForNull AwsSdk2TransportOptions options) { + this(null, asyncHttpClient, host, signingServiceName, signingRegion, options); + } + + /** + * Create an {@link OpenSearchTransport} with both synchronous and asynchronous AWS HTTP clients. + *

+ * The synchronous client will be used for synchronous OpenSearch requests, and the asynchronous client + * will be used for asynchronous HTTP requests. + * + * @param httpClient HTTP client to use for OpenSearch requests. + * @param asyncHttpClient HTTP client to use for synchronous OpenSearch requests. + * @param host The fully qualified domain name to connect to. + * @param signingRegion The AWS region for which requests will be signed. This should typically match the region in `host`. + * @param options Options that apply to all requests. Can be null. Create with + * {@link AwsSdk2TransportOptions#builder()} and use these to specify non-default credentials, + * compression options, etc. + */ + public AwsSdk2Transport( + @CheckForNull SdkHttpClient httpClient, + @CheckForNull SdkAsyncHttpClient asyncHttpClient, + @Nonnull String host, + @Nonnull Region signingRegion, + @CheckForNull AwsSdk2TransportOptions options) { + this(httpClient, asyncHttpClient, host, "es", signingRegion, options); } /** @@ -127,12 +195,12 @@ public AwsSdk2Transport( * The synchronous client will be used for synchronous OpenSearch requests, and the asynchronous client * will be used for asynchronous HTTP requests. * - * @param httpClient HTTP client to use for OpenSearch requests - * @param asyncHttpClient HTTP client to use for synchronous OpenSearch requests - * @param host The fully qualified domain name to connect to - * @param signingRegion The AWS region for which requests will be signed. This should typically match - * the region in `host`. - * @param options Options that apply to all requests. Can be null. Create with + * @param httpClient HTTP client to use for OpenSearch requests. + * @param asyncHttpClient HTTP client to use for synchronous OpenSearch requests. + * @param host The fully qualified domain name to connect to. + * @param signingRegion The AWS region for which requests will be signed. This should typically match the region in `host`. + * @param signingServiceName The AWS signing service name, one of `es` (Amazon OpenSearch) or `aoss` (Amazon OpenSearch Serverless). + * @param options Options that apply to all requests. Can be null. Create with * {@link AwsSdk2TransportOptions#builder()} and use these to specify non-default credentials, * compression options, etc. */ @@ -140,6 +208,7 @@ public AwsSdk2Transport( @CheckForNull SdkHttpClient httpClient, @CheckForNull SdkAsyncHttpClient asyncHttpClient, @Nonnull String host, + @Nonnull String signingServiceName, @Nonnull Region signingRegion, @CheckForNull AwsSdk2TransportOptions options) { if (httpClient == null && asyncHttpClient == null) @@ -150,6 +219,7 @@ public AwsSdk2Transport( this.httpClient = httpClient; this.asyncHttpClient = asyncHttpClient; this.host = host; + this.signingServiceName = signingServiceName; this.signingRegion = signingRegion; this.transportOptions = options != null ? options : AwsSdk2TransportOptions.builder().build(); this.defaultMapper = Optional.ofNullable(options) @@ -314,7 +384,7 @@ private SdkHttpFullRequest prepareRequest( Aws4SignerParams signerParams = Aws4SignerParams.builder() .awsCredentials(credentials.resolveCredentials()) - .signingName("es") + .signingName(this.signingServiceName) .signingRegion(signingRegion) .build(); return Aws4Signer.create().sign(req.build(), signerParams); diff --git a/java-client/src/test/java/org/opensearch/client/opensearch/integTest/aws/AwsSdk2TransportTestCase.java b/java-client/src/test/java/org/opensearch/client/opensearch/integTest/aws/AwsSdk2TransportTestCase.java index 05421559d1..fa1d33f39a 100644 --- a/java-client/src/test/java/org/opensearch/client/opensearch/integTest/aws/AwsSdk2TransportTestCase.java +++ b/java-client/src/test/java/org/opensearch/client/opensearch/integTest/aws/AwsSdk2TransportTestCase.java @@ -83,6 +83,7 @@ protected OpenSearchClient getClient( transport = new AwsSdk2Transport( getAsyncHttpClient(), getTestClusterHost(), + getTestClusterServiceName(), getTestClusterRegion(), getTransportOptions().build() ); @@ -90,6 +91,7 @@ protected OpenSearchClient getClient( transport = new AwsSdk2Transport( getHttpClient(), getTestClusterHost(), + getTestClusterServiceName(), getTestClusterRegion(), getTransportOptions().build() ); @@ -107,6 +109,7 @@ protected OpenSearchAsyncClient getAsyncClient( transport = new AwsSdk2Transport( getAsyncHttpClient(), getTestClusterHost(), + getTestClusterServiceName(), getTestClusterRegion(), getTransportOptions().build() ); @@ -114,6 +117,7 @@ protected OpenSearchAsyncClient getAsyncClient( transport = new AwsSdk2Transport( getHttpClient(), getTestClusterHost(), + getTestClusterServiceName(), getTestClusterRegion(), getTransportOptions().build() ); @@ -131,6 +135,7 @@ protected OpenSearchIndicesClient getIndexesClient( transport = new AwsSdk2Transport( getAsyncHttpClient(), getTestClusterHost(), + getTestClusterServiceName(), getTestClusterRegion(), getTransportOptions().build() ); @@ -138,6 +143,7 @@ protected OpenSearchIndicesClient getIndexesClient( transport = new AwsSdk2Transport( getHttpClient(), getTestClusterHost(), + getTestClusterServiceName(), getTestClusterRegion(), getTransportOptions().build() ); @@ -150,6 +156,11 @@ protected String getTestClusterHost() { return cluster; } + protected String getTestClusterServiceName() { + String cluster = System.getProperty("tests.awsSdk2support.serviceName"); + return cluster; + } + protected Region getTestClusterRegion() { String region = System.getProperty("tests.awsSdk2support.domainRegion"); return region != null ? Region.of(region) : Region.US_EAST_1;