Implement Digest Authorization Support (#6954)

* Rough idea for Digest Proxy support

* Rough implementation of RFC 2617 and 7616

* Finalize RGC 2617 and RFC 7616 implementation

* Added tests and changes to code structure

* Integrating authentication support into Reactor Netty and OkHttp

* Using channel attributes to pass authorization information

* Support passing proxy authentication header through context

* Fixed incorrect formatting in Proxy-Authorization header, finalized proxy digest authentication support in Reactor Netty

* Adding documentation

* Reverted testing changes in AppConfiguration

* Moved AuthorizationChallengeHandler into util

* Additional tests for AuthorizationChallengeHandler

* Fixed checkstyle issues

* Fixing linting and test issues

* Changes based on PR feedback

* Fix linting issue

* Removed tests that don't have configuration available

* Removed accidental character

* Unit tests for Netty HttpClient

* Fixed invalid name casing

* Additional Netty tests

* Unit tests for OkHttp

* Fixed invalid failing test and removed erroneous dependency

* Renaming classes, updating licensing, and added more tests for authScheme

* Fixed linting issues, synchronizing access to lastChallenge in AuthorizationChallengeHandler

* Fixed linting issue

* Added OkHttp tests for configuration based proxy

* Revert to using AtomicReference

* Updated licensing text to include Netty license

* Add component governance manifest

THIRD-PARTY-NOTICE.txt

@@ -102,4 +102,21 @@ http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software distributed under the License
 is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 or implied. See the License for the specific language governing permissions and limitations under
-the License.
+the License.
+
+License notice for Netty
+------------------------------------------------------------------------------
+
+Copyright 2014 The Netty Project
+
+The Netty Project licenses this file to you under the Apache License,
+version 2.0 (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.

eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml

@@ -35,6 +35,7 @@
   <suppress checks="Header" files=".*CachingKeyResolverTest.java"/>
   <suppress checks="Header" files=".*KeyVaultKeyResolverBCProviderTest.java"/>
   <suppress checks="Header" files=".*KeyVaultKeyResolverDefaultProviderTest.java"/>
+  <suppress checks="Header" files="com.azure.core.http.netty.implementation.HttpProxyHandler"/>
 
   <!-- Cryptography Client exception for service client instantiation as it provides client side crypto and is not entirely based on REST service -->
   <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files=".*CryptographyAsyncClient.java"/>
@@ -205,6 +206,9 @@
   <!-- Report AMQP retry attempts in static withRetry method. -->
   <suppress checks="com.azure.tools.checkstyle.checks.GoodLoggingCheck" files="com.azure.core.amqp.implementation.RetryUtil.java"/>
 
+  <!-- Throws a non-runtime exception -->
+  <suppress checks="com.azure.tools.checkstyle.checks.ThrowFromClientLogger" files="com.azure.core.http.netty.implementation.HttpProxyHandler.java"/>
+
   <!-- Event Hubs uses AMQP, which does not contain an HTTP response. Returning PagedResponse and Response does not apply. -->
   <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.messaging.eventhubs.EventHubClient.java"/>
   <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.messaging.eventhubs.EventHubAsyncClient.java"/>

sdk/core/azure-core-http-netty/cgmanifest.json

@@ -0,0 +1,15 @@
+{
+    "Registrations": [
+        {
+            "Component": {
+                "Type": "Maven",
+                "Maven": {
+                    "ArtifactId": "netty-handler-proxy",
+                    "GroupId": "io.netty",
+                    "Version": "4.1.42.Final"
+                }
+            }
+        }
+    ],
+    "Version": 1
+}

sdk/core/azure-core-http-netty/pom.xml

@@ -138,12 +138,24 @@
       <version>5.4.2</version> <!-- {x-version-update;org.junit.jupiter:junit-jupiter-engine;external_dependency} -->
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <version>5.4.2</version> <!-- {x-version-update;org.junit.jupiter:junit-jupiter-params;external_dependency} -->
+      <scope>test</scope>
+    </dependency>
 
     <dependency>
       <groupId>com.github.tomakehurst</groupId>
       <artifactId>wiremock-standalone</artifactId>
       <version>2.24.1</version> <!-- {x-version-update;com.github.tomakehurst:wiremock-standalone;external_dependency} -->
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <version>3.0.0</version><!-- {x-version-update;org.mockito:mockito-core;external_dependency} -->
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>

sdk/core/azure-core-http-netty/src/main/java/com/azure/core/http/netty/NettyAsyncHttpClient.java

@@ -9,68 +9,118 @@
 import com.azure.core.http.HttpRequest;
 import com.azure.core.http.HttpResponse;
 import com.azure.core.http.ProxyOptions;
+import com.azure.core.http.netty.implementation.HttpProxyExceptionHandler;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.proxy.ProxyHandler;
 import org.reactivestreams.Publisher;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.netty.ByteBufFlux;
 import reactor.netty.Connection;
 import reactor.netty.NettyOutbound;
+import reactor.netty.NettyPipeline;
+import reactor.netty.channel.BootstrapHandlers;
 import reactor.netty.http.client.HttpClientRequest;
 import reactor.netty.http.client.HttpClientResponse;
+import reactor.netty.tcp.TcpClient;
 
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.util.Objects;
 import java.util.function.BiFunction;
+import java.util.function.Supplier;
+import java.util.regex.Pattern;
 
 /**
- * This class provides a Netty-based implementation for the {@link HttpClient} interface. Creating an instance of
- * this class can be achieved by using the {@link NettyAsyncHttpClientBuilder} class, which offers Netty-specific API
- * for features such as {@link NettyAsyncHttpClientBuilder#nioEventLoopGroup(NioEventLoopGroup) thread pooling},
- * {@link NettyAsyncHttpClientBuilder#wiretap(boolean) wiretapping},
- * {@link NettyAsyncHttpClientBuilder#proxy(ProxyOptions) setProxy configuration}, and much more.
+ * This class provides a Netty-based implementation for the {@link HttpClient} interface. Creating an instance of this
+ * class can be achieved by using the {@link NettyAsyncHttpClientBuilder} class, which offers Netty-specific API for
+ * features such as {@link NettyAsyncHttpClientBuilder#nioEventLoopGroup(NioEventLoopGroup) thread pooling}, {@link
+ * NettyAsyncHttpClientBuilder#wiretap(boolean) wiretapping}, {@link NettyAsyncHttpClientBuilder#proxy(ProxyOptions)
+ * setProxy configuration}, and much more.
  *
  * @see HttpClient
  * @see NettyAsyncHttpClientBuilder
  */
 class NettyAsyncHttpClient implements HttpClient {
+    private final NioEventLoopGroup eventLoopGroup;
+    private final Supplier<ProxyHandler> proxyHandlerSupplier;
+    private final Pattern nonProxyHostsPattern;
+
     final reactor.netty.http.client.HttpClient nettyClient;
 
     /**
      * Creates default NettyAsyncHttpClient.
      */
     NettyAsyncHttpClient() {
-        this(reactor.netty.http.client.HttpClient.create());
+        this(reactor.netty.http.client.HttpClient.create(), null, null, null);
     }
 
     /**
      * Creates NettyAsyncHttpClient with provided http client.
      *
      * @param nettyClient the reactor-netty http client
+     * @param eventLoopGroup {@link NioEventLoopGroup} that processes requests.
+     * @param proxyHandlerSupplier Supplier that returns the {@link ProxyHandler} that connects to the configured
+     * proxy.
      */
-    NettyAsyncHttpClient(reactor.netty.http.client.HttpClient nettyClient) {
+    NettyAsyncHttpClient(reactor.netty.http.client.HttpClient nettyClient, NioEventLoopGroup eventLoopGroup,
+        Supplier<ProxyHandler> proxyHandlerSupplier, String nonProxyHosts) {
         this.nettyClient = nettyClient;
+        this.eventLoopGroup = eventLoopGroup;
+        this.proxyHandlerSupplier = proxyHandlerSupplier;
+        this.nonProxyHostsPattern = (nonProxyHosts == null)
+            ? null
+            : Pattern.compile(nonProxyHosts, Pattern.CASE_INSENSITIVE);
     }
 
-    /** {@inheritDoc} */
+    /**
+     * {@inheritDoc}
+     */
     @Override
     public Mono<HttpResponse> send(final HttpRequest request) {
         Objects.requireNonNull(request.getHttpMethod(), "'request.getHttpMethod()' cannot be null.");
         Objects.requireNonNull(request.getUrl(), "'request.getUrl()' cannot be null.");
         Objects.requireNonNull(request.getUrl().getProtocol(), "'request.getUrl().getProtocol()' cannot be null.");
 
         return nettyClient
+            .tcpConfiguration(tcpClient -> configureTcpClient(tcpClient, request.getUrl().getHost()))
             .request(HttpMethod.valueOf(request.getHttpMethod().toString()))
             .uri(request.getUrl().toString())
             .send(bodySendDelegate(request))
             .responseConnection(responseDelegate(request))
             .single();
     }
 
+    /*
+     * Configures the underlying TcpClient that sends the request.
+     */
+    private TcpClient configureTcpClient(TcpClient tcpClient, String host) {
+        if (eventLoopGroup != null) {
+            tcpClient = tcpClient.runOn(eventLoopGroup);
+        }
+
+        // Validate that the request should be proxied.
+        if (nonProxyHostsPattern == null || !nonProxyHostsPattern.matcher(host).matches()) {
+            ProxyHandler proxyHandler = (proxyHandlerSupplier == null) ? null : proxyHandlerSupplier.get();
+            if (proxyHandler != null) {
+                /*
+                 * Configure the request Channel to be initialized with a ProxyHandler. The ProxyHandler is the first
+                 * operation in the pipeline as it needs to handle sending a CONNECT request to the proxy before any
+                 * request data is sent.
+                 */
+                tcpClient = tcpClient.bootstrap(bootstrap -> BootstrapHandlers
+                    .updateConfiguration(bootstrap, NettyPipeline.ProxyHandler, (connectionObserver, channel) ->
+                        channel.pipeline().addFirst(NettyPipeline.ProxyHandler, proxyHandler)
+                            .addLast("azure.proxy.exceptionHandler", new HttpProxyExceptionHandler())));
+            }
+        }
+
+        return tcpClient;
+    }
+
     /**
      * Delegate to send the request content.
      *
@@ -111,7 +161,7 @@ static class ReactorNettyHttpResponse extends HttpResponse {
         private final Connection reactorNettyConnection;
 
         ReactorNettyHttpResponse(HttpClientResponse reactorNettyResponse, Connection reactorNettyConnection,
-                                 HttpRequest httpRequest) {
+            HttpRequest httpRequest) {
             super(httpRequest);
             this.reactorNettyResponse = reactorNettyResponse;
             this.reactorNettyConnection = reactorNettyConnection;

sdk/core/azure-core-http-netty/src/main/java/com/azure/core/http/netty/NettyAsyncHttpClientBuilder.java

@@ -4,14 +4,20 @@
 package com.azure.core.http.netty;
 
 import com.azure.core.http.ProxyOptions;
+import com.azure.core.http.netty.implementation.ChallengeHolder;
+import com.azure.core.http.netty.implementation.HttpProxyHandler;
+import com.azure.core.util.AuthorizationChallengeHandler;
 import com.azure.core.util.Configuration;
 import com.azure.core.util.logging.ClientLogger;
 import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.handler.proxy.ProxyHandler;
+import io.netty.handler.proxy.Socks4ProxyHandler;
+import io.netty.handler.proxy.Socks5ProxyHandler;
 import reactor.netty.http.client.HttpClient;
 import reactor.netty.resources.ConnectionProvider;
-import reactor.netty.tcp.ProxyProvider;
 
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * Builder class responsible for creating instances of {@link NettyAsyncHttpClient}.
@@ -24,6 +30,8 @@
  * @see HttpClient
  */
 public class NettyAsyncHttpClientBuilder {
+    private static final String INVALID_PROXY_MESSAGE = "Unknown Proxy type '%s' in use. Not configuring Netty proxy.";
+
     private final ClientLogger logger = new ClientLogger(NettyAsyncHttpClientBuilder.class);
 
     private final HttpClient baseHttpClient;
@@ -63,65 +71,34 @@ public NettyAsyncHttpClientBuilder(HttpClient nettyHttpClient) {
      */
     public com.azure.core.http.HttpClient build() {
         HttpClient nettyHttpClient;
-        if (this.connectionProvider != null) {
-            if (this.baseHttpClient != null) {
-                throw logger.logExceptionAsError(new IllegalStateException("connectionProvider cannot be set on an "
-                    + "existing reactor netty HttpClient."));
-            }
+        if (this.baseHttpClient != null) {
+            nettyHttpClient = baseHttpClient;
+        } else if (this.connectionProvider != null) {
             nettyHttpClient = HttpClient.create(this.connectionProvider);
         } else {
-            nettyHttpClient = this.baseHttpClient == null ? HttpClient.create() : this.baseHttpClient;
+            nettyHttpClient = HttpClient.create();
         }
 
+        nettyHttpClient = nettyHttpClient
+            .port(port)
+            .wiretap(enableWiretap);
+
         Configuration buildConfiguration = (configuration == null)
             ? Configuration.getGlobalConfiguration()
             : configuration;
 
-        nettyHttpClient = nettyHttpClient
-            .port(port)
-            .wiretap(enableWiretap)
-            .tcpConfiguration(tcpConfig -> {
-                if (nioEventLoopGroup != null) {
-                    tcpConfig = tcpConfig.runOn(nioEventLoopGroup);
-                }
-
-                ProxyOptions buildProxyOptions = (proxyOptions == null)
-                    ? ProxyOptions.fromConfiguration(buildConfiguration)
-                    : proxyOptions;
-
-                if (buildProxyOptions != null) {
-                    tcpConfig = tcpConfig.proxy(typeSpec ->
-                        typeSpec.type(mapProxyType(buildProxyOptions.getType(), logger))
-                            .address(proxyOptions.getAddress())
-                            .username(proxyOptions.getUsername())
-                            .password(user -> proxyOptions.getPassword())
-                            .nonProxyHosts(proxyOptions.getNonProxyHosts()));
-                }
-
-                return tcpConfig;
-            });
-
-        return new NettyAsyncHttpClient(nettyHttpClient);
-    }
+        ProxyOptions buildProxyOptions = (proxyOptions == null && buildConfiguration != Configuration.NONE)
+            ? ProxyOptions.fromConfiguration(buildConfiguration)
+            : proxyOptions;
 
-    /*
-     * Maps a 'ProxyOptions.Type' to a 'ProxyProvider.Proxy', if the type is unknown or cannot be mapped an
-     * IllegalStateException will be thrown.
-     */
-    private static ProxyProvider.Proxy mapProxyType(ProxyOptions.Type type, ClientLogger logger) {
-        Objects.requireNonNull(type, "'ProxyOptions.getType()' cannot be null.");
+        String nonProxyHosts = (buildProxyOptions == null) ? null : buildProxyOptions.getNonProxyHosts();
+        AuthorizationChallengeHandler handler = (buildProxyOptions == null || buildProxyOptions.getUsername() == null)
+            ? null
+            : new AuthorizationChallengeHandler(buildProxyOptions.getUsername(), buildProxyOptions.getPassword());
+        AtomicReference<ChallengeHolder> proxyChallengeHolder = new AtomicReference<>();
 
-        switch (type) {
-            case HTTP:
-                return ProxyProvider.Proxy.HTTP;
-            case SOCKS4:
-                return ProxyProvider.Proxy.SOCKS4;
-            case SOCKS5:
-                return ProxyProvider.Proxy.SOCKS5;
-            default:
-                throw logger.logExceptionAsError(new IllegalStateException(
-                    String.format("Unknown proxy type '%s' in use. Use a proxy type from 'ProxyOptions.Type'.", type)));
-        }
+        return new NettyAsyncHttpClient(nettyHttpClient, nioEventLoopGroup,
+            () -> getProxyHandler(handler, proxyChallengeHolder), nonProxyHosts);
     }
 
     /**
@@ -202,4 +179,28 @@ public NettyAsyncHttpClientBuilder configuration(Configuration configuration) {
         this.configuration = configuration;
         return this;
     }
+
+    /*
+     * Creates a proxy handler based on the passed ProxyOptions.
+     */
+    private ProxyHandler getProxyHandler(AuthorizationChallengeHandler challengeHandler,
+        AtomicReference<ChallengeHolder> proxyChallengeHolder) {
+        if (proxyOptions == null) {
+            return null;
+        }
+
+        switch (proxyOptions.getType()) {
+            case HTTP:
+                return new HttpProxyHandler(proxyOptions.getAddress(), challengeHandler,
+                    proxyChallengeHolder);
+            case SOCKS4:
+                return new Socks4ProxyHandler(proxyOptions.getAddress(), proxyOptions.getUsername());
+            case SOCKS5:
+                return new Socks5ProxyHandler(proxyOptions.getAddress(), proxyOptions.getUsername(),
+                    proxyOptions.getPassword());
+            default:
+                throw logger.logExceptionAsError(new IllegalStateException(
+                    String.format(INVALID_PROXY_MESSAGE, proxyOptions.getType())));
+        }
+    }
 }