Skip to content

Commit

Permalink
Add AWS v2 SDK and netty dependencies in repository-s3 plugin (opense…
Browse files Browse the repository at this point in the history 
…arch-project#7216)

Signed-off-by: Raghuvansh Raj <[email protected]>
Signed-off-by: Shivansh Arora <[email protected]>
  • Loading branch information
@raghuvanshraj @shiv0408
raghuvanshraj authored and shiv0408 committed Apr 25, 2024
1 parent 7e3e096 commit 3addabd
Show file tree
Hide file tree
Showing 14 changed files with 523 additions and 50 deletions.
244 changes: 194 additions & 50 deletions plugins/repository-s3/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ opensearchplugin {
}

dependencies {
// aws sdk v2 stack
api "software.amazon.awssdk:sdk-core:${versions.aws}"
api "software.amazon.awssdk:annotations:${versions.aws}"
api "software.amazon.awssdk:aws-core:${versions.aws}"
Expand All @@ -67,6 +68,8 @@ dependencies {
api "software.amazon.awssdk:aws-json-protocol:${versions.aws}"
api "software.amazon.awssdk:aws-query-protocol:${versions.aws}"
api "software.amazon.awssdk:sts:${versions.aws}"
api "software.amazon.awssdk:netty-nio-client:${versions.aws}"

api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
Expand All @@ -80,6 +83,18 @@ dependencies {
api "joda-time:joda-time:${versions.joda}"
api "org.slf4j:slf4j-api:${versions.slf4j}"

// network stack
api "io.netty:netty-buffer:${versions.netty}"
api "io.netty:netty-codec:${versions.netty}"
api "io.netty:netty-codec-http:${versions.netty}"
api "io.netty:netty-codec-http2:${versions.netty}"
api "io.netty:netty-common:${versions.netty}"
api "io.netty:netty-handler:${versions.netty}"
api "io.netty:netty-resolver:${versions.netty}"
api "io.netty:netty-transport:${versions.netty}"
api "io.netty:netty-transport-native-unix-common:${versions.netty}"
api "io.netty:netty-transport-classes-epoll:${versions.netty}"

// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
// and allowlist this hack in JarHell
api 'javax.xml.bind:jaxb-api:2.3.1'
Expand All @@ -96,6 +111,7 @@ restResources {
tasks.named("dependencyLicenses").configure {
mapping from: /jackson-.*/, to: 'jackson'
mapping from: /jaxb-.*/, to: 'jaxb'
mapping from: /netty-.*/, to: 'netty'
}

bundlePlugin {
Expand Down Expand Up @@ -375,56 +391,184 @@ TaskProvider s3ThirdPartyTest = tasks.register("s3ThirdPartyTest", Test) {
}
tasks.named("check").configure { dependsOn(s3ThirdPartyTest) }
thirdPartyAudit.ignoreMissingClasses(
// classes are missing
'javax.jms.Message',
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
'org.apache.log.Logger',
'org.slf4j.impl.StaticLoggerBinder',
'org.slf4j.impl.StaticMDCBinder',
'org.slf4j.impl.StaticMarkerBinder',
'software.amazon.awssdk.arns.Arn',
'software.amazon.awssdk.arns.ArnResource',
'software.amazon.awssdk.crt.CRT',
'software.amazon.awssdk.crt.auth.credentials.Credentials',
'software.amazon.awssdk.crt.auth.credentials.CredentialsProvider',
'software.amazon.awssdk.crt.auth.credentials.DelegateCredentialsProvider$DelegateCredentialsProviderBuilder',
'software.amazon.awssdk.crt.http.HttpHeader',
'software.amazon.awssdk.crt.http.HttpMonitoringOptions',
'software.amazon.awssdk.crt.http.HttpProxyOptions',
'software.amazon.awssdk.crt.http.HttpRequest',
'software.amazon.awssdk.crt.http.HttpRequestBodyStream',
'software.amazon.awssdk.crt.io.ClientBootstrap',
'software.amazon.awssdk.crt.io.ExponentialBackoffRetryOptions',
'software.amazon.awssdk.crt.io.StandardRetryOptions',
'software.amazon.awssdk.crt.io.TlsCipherPreference',
'software.amazon.awssdk.crt.io.TlsContext',
'software.amazon.awssdk.crt.io.TlsContextOptions',
'software.amazon.awssdk.crt.s3.ChecksumAlgorithm',
'software.amazon.awssdk.crt.s3.ChecksumConfig',
'software.amazon.awssdk.crt.s3.ChecksumConfig$ChecksumLocation',
'software.amazon.awssdk.crt.s3.ResumeToken',
'software.amazon.awssdk.crt.s3.S3Client',
'software.amazon.awssdk.crt.s3.S3ClientOptions',
'software.amazon.awssdk.crt.s3.S3FinishedResponseContext',
'software.amazon.awssdk.crt.s3.S3MetaRequest',
'software.amazon.awssdk.crt.s3.S3MetaRequestOptions',
'software.amazon.awssdk.crt.s3.S3MetaRequestOptions$MetaRequestType',
'software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler',
'software.amazon.awssdk.crtcore.CrtConfigurationUtils',
'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration',
'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$Builder',
'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$DefaultBuilder',
'software.amazon.awssdk.crtcore.CrtProxyConfiguration',
'software.amazon.awssdk.crtcore.CrtProxyConfiguration$Builder',
'software.amazon.awssdk.crtcore.CrtProxyConfiguration$DefaultBuilder',
'software.amazon.eventstream.HeaderValue',
'software.amazon.eventstream.Message',
'software.amazon.eventstream.MessageDecoder'
)
thirdPartyAudit {
ignoreMissingClasses(
// classes are missing
'javax.jms.Message',
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
'org.apache.log.Logger',
// from io.netty.handler.ssl.OpenSslEngine (netty)
'io.netty.internal.tcnative.AsyncSSLPrivateKeyMethod',
'io.netty.internal.tcnative.AsyncTask',
'io.netty.internal.tcnative.Buffer',
'io.netty.internal.tcnative.CertificateCompressionAlgo',
'io.netty.internal.tcnative.Library',
'io.netty.internal.tcnative.ResultCallback',
'io.netty.internal.tcnative.SSL',
'io.netty.internal.tcnative.SSLContext',
'io.netty.internal.tcnative.SSLPrivateKeyMethod',
'io.netty.internal.tcnative.SSLSession',
'io.netty.internal.tcnative.SSLSessionCache',
'io.netty.internal.tcnative.CertificateCallback',
'io.netty.internal.tcnative.CertificateVerifier',
'io.netty.internal.tcnative.SessionTicketKey',
'io.netty.internal.tcnative.SniHostNameMatcher',
'com.aayushatharva.brotli4j.Brotli4jLoader',
'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
'com.aayushatharva.brotli4j.encoder.Encoder',
'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
'com.google.protobuf.nano.CodedOutputByteBufferNano',
'com.google.protobuf.nano.MessageNano',
'com.ning.compress.BufferRecycler',
'com.ning.compress.lzf.ChunkDecoder',
'com.ning.compress.lzf.ChunkEncoder',
'com.ning.compress.lzf.LZFChunk',
'com.ning.compress.lzf.LZFEncoder',
'com.ning.compress.lzf.util.ChunkDecoderFactory',
'com.ning.compress.lzf.util.ChunkEncoderFactory',
'io.netty.internal.tcnative.AsyncSSLPrivateKeyMethod',
'io.netty.internal.tcnative.AsyncTask',
'io.netty.internal.tcnative.CertificateCallback',
'io.netty.internal.tcnative.CertificateVerifier',
'io.netty.internal.tcnative.ResultCallback',
'io.netty.internal.tcnative.SessionTicketKey',
'io.netty.internal.tcnative.SniHostNameMatcher',
'io.netty.internal.tcnative.SSL',
'io.netty.internal.tcnative.SSLSession',
'io.netty.internal.tcnative.SSLSessionCache',
'lzma.sdk.lzma.Encoder',
'net.jpountz.lz4.LZ4Compressor',
'net.jpountz.lz4.LZ4Factory',
'net.jpountz.lz4.LZ4FastDecompressor',
'net.jpountz.xxhash.XXHash32',
'net.jpountz.xxhash.XXHashFactory',
// from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
'org.bouncycastle.cert.X509v3CertificateBuilder',
'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
'org.bouncycastle.openssl.PEMEncryptedKeyPair',
'org.bouncycastle.openssl.PEMParser',
'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
'org.conscrypt.AllocatedBuffer',
'org.conscrypt.BufferAllocator',
'org.conscrypt.Conscrypt',
'org.conscrypt.HandshakeListener',
'org.eclipse.jetty.alpn.ALPN$ClientProvider',
'org.eclipse.jetty.alpn.ALPN$ServerProvider',
'org.eclipse.jetty.alpn.ALPN',
// from io.netty.handler.ssl.JettyNpnSslEngine (netty)
'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
'org.eclipse.jetty.npn.NextProtoNego$ServerProvider',
'org.eclipse.jetty.npn.NextProtoNego',
// from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty)
'org.jboss.marshalling.ByteInput',
// from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty)
'org.jboss.marshalling.ByteOutput',
// from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty)
'org.jboss.marshalling.Marshaller',
// from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty)
'org.jboss.marshalling.MarshallerFactory',
'org.jboss.marshalling.MarshallingConfiguration',
'org.jboss.marshalling.Unmarshaller',
'org.slf4j.impl.StaticLoggerBinder',
'org.slf4j.impl.StaticMDCBinder',
'org.slf4j.impl.StaticMarkerBinder',
'reactor.blockhound.BlockHound$Builder',
'reactor.blockhound.integration.BlockHoundIntegration',
'software.amazon.awssdk.arns.Arn',
'software.amazon.awssdk.arns.ArnResource',
'software.amazon.awssdk.crt.CRT',
'software.amazon.awssdk.crt.auth.credentials.Credentials',
'software.amazon.awssdk.crt.auth.credentials.CredentialsProvider',
'software.amazon.awssdk.crt.auth.credentials.DelegateCredentialsProvider$DelegateCredentialsProviderBuilder',
'software.amazon.awssdk.crt.http.HttpHeader',
'software.amazon.awssdk.crt.http.HttpMonitoringOptions',
'software.amazon.awssdk.crt.http.HttpProxyOptions',
'software.amazon.awssdk.crt.http.HttpRequest',
'software.amazon.awssdk.crt.http.HttpRequestBodyStream',
'software.amazon.awssdk.crt.io.ClientBootstrap',
'software.amazon.awssdk.crt.io.ExponentialBackoffRetryOptions',
'software.amazon.awssdk.crt.io.StandardRetryOptions',
'software.amazon.awssdk.crt.io.TlsCipherPreference',
'software.amazon.awssdk.crt.io.TlsContext',
'software.amazon.awssdk.crt.io.TlsContextOptions',
'software.amazon.awssdk.crt.s3.ChecksumAlgorithm',
'software.amazon.awssdk.crt.s3.ChecksumConfig',
'software.amazon.awssdk.crt.s3.ChecksumConfig$ChecksumLocation',
'software.amazon.awssdk.crt.s3.ResumeToken',
'software.amazon.awssdk.crt.s3.S3Client',
'software.amazon.awssdk.crt.s3.S3ClientOptions',
'software.amazon.awssdk.crt.s3.S3FinishedResponseContext',
'software.amazon.awssdk.crt.s3.S3MetaRequest',
'software.amazon.awssdk.crt.s3.S3MetaRequestOptions',
'software.amazon.awssdk.crt.s3.S3MetaRequestOptions$MetaRequestType',
'software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler',
'software.amazon.awssdk.crtcore.CrtConfigurationUtils',
'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration',
'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$Builder',
'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$DefaultBuilder',
'software.amazon.awssdk.crtcore.CrtProxyConfiguration',
'software.amazon.awssdk.crtcore.CrtProxyConfiguration$Builder',
'software.amazon.awssdk.crtcore.CrtProxyConfiguration$DefaultBuilder',
'software.amazon.eventstream.HeaderValue',
'software.amazon.eventstream.Message',
'software.amazon.eventstream.MessageDecoder'
)
ignoreViolations (
'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1',
'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2',
'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$3',
'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$4',
'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$5',
'io.netty.util.internal.PlatformDependent0',
'io.netty.util.internal.PlatformDependent0$1',
'io.netty.util.internal.PlatformDependent0$2',
'io.netty.util.internal.PlatformDependent0$3',
'io.netty.util.internal.PlatformDependent0$4',
'io.netty.util.internal.PlatformDependent0$6',
'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef',
'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef',
'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields',
'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields',
'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields',
'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode',
'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
)
}
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
if (BuildParams.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
Expand Down
Loading

0 comments on commit 3addabd

Please sign in to comment.