tls: override ALPN in transport socket options (envoyproxy#8470)

Description: Override ALPN in transport socket options
Risk Level: Low
Testing: Unit test
Docs Changes: N/A
Release Notes: N/A
Fixes #Issue: Part of envoyproxy#8197

Signed-off-by: crazyxy <[email protected]>

include/envoy/network/transport_socket.h

@@ -171,6 +171,11 @@ class TransportSocketOptions {
    */
   virtual const std::vector<std::string>& verifySubjectAltNameListOverride() const PURE;
 
+  /**
+   * @return the optional overridden application protocols.
+   */
+  virtual const std::vector<std::string>& applicationProtocolListOverride() const PURE;
+
   /**
    * @param vector of bytes to which the option should append hash key data that will be used
    *        to separate connections based on the option. Any data already in the key vector must

include/envoy/upstream/cluster_manager.h

@@ -275,7 +275,8 @@ class ClusterManagerFactory {
   virtual Http::ConnectionPool::InstancePtr
   allocateConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
                    ResourcePriority priority, Http::Protocol protocol,
-                   const Network::ConnectionSocket::OptionsSharedPtr& options) PURE;
+                   const Network::ConnectionSocket::OptionsSharedPtr& options,
+                   const Network::TransportSocketOptionsSharedPtr& transport_socket_options) PURE;
 
   /**
    * Allocate a TCP connection pool for the host. Pools are separated by 'priority' and

source/common/http/http1/conn_pool.cc

@@ -25,9 +25,10 @@ namespace Http1 {
 
 ConnPoolImpl::ConnPoolImpl(Event::Dispatcher& dispatcher, Upstream::HostConstSharedPtr host,
                            Upstream::ResourcePriority priority,
-                           const Network::ConnectionSocket::OptionsSharedPtr& options)
+                           const Network::ConnectionSocket::OptionsSharedPtr& options,
+                           const Network::TransportSocketOptionsSharedPtr& transport_socket_options)
     : ConnPoolImplBase(std::move(host), std::move(priority)), dispatcher_(dispatcher),
-      socket_options_(options),
+      socket_options_(options), transport_socket_options_(transport_socket_options),
       upstream_ready_timer_(dispatcher_.createTimer([this]() { onUpstreamReady(); })) {}
 
 ConnPoolImpl::~ConnPoolImpl() {
@@ -303,8 +304,8 @@ ConnPoolImpl::ActiveClient::ActiveClient(ConnPoolImpl& parent)
 
   parent_.conn_connect_ms_ = std::make_unique<Stats::Timespan>(
       parent_.host_->cluster().stats().upstream_cx_connect_ms_, parent_.dispatcher_.timeSource());
-  Upstream::Host::CreateConnectionData data =
-      parent_.host_->createConnection(parent_.dispatcher_, parent_.socket_options_, nullptr);
+  Upstream::Host::CreateConnectionData data = parent_.host_->createConnection(
+      parent_.dispatcher_, parent_.socket_options_, parent_.transport_socket_options_);
   real_host_description_ = data.host_description_;
   codec_client_ = parent_.createCodecClient(data);
   codec_client_->addConnectionCallbacks(*this);

source/common/http/http1/conn_pool.h

@@ -32,7 +32,8 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase {
 public:
   ConnPoolImpl(Event::Dispatcher& dispatcher, Upstream::HostConstSharedPtr host,
                Upstream::ResourcePriority priority,
-               const Network::ConnectionSocket::OptionsSharedPtr& options);
+               const Network::ConnectionSocket::OptionsSharedPtr& options,
+               const Network::TransportSocketOptionsSharedPtr& transport_socket_options);
 
   ~ConnPoolImpl() override;
 
@@ -122,6 +123,7 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase {
   std::list<ActiveClientPtr> busy_clients_;
   std::list<DrainedCb> drained_callbacks_;
   const Network::ConnectionSocket::OptionsSharedPtr socket_options_;
+  const Network::TransportSocketOptionsSharedPtr transport_socket_options_;
   Event::TimerPtr upstream_ready_timer_;
   bool upstream_ready_enabled_{false};
 };
@@ -133,8 +135,9 @@ class ProdConnPoolImpl : public ConnPoolImpl {
 public:
   ProdConnPoolImpl(Event::Dispatcher& dispatcher, Upstream::HostConstSharedPtr host,
                    Upstream::ResourcePriority priority,
-                   const Network::ConnectionSocket::OptionsSharedPtr& options)
-      : ConnPoolImpl(dispatcher, host, priority, options) {}
+                   const Network::ConnectionSocket::OptionsSharedPtr& options,
+                   const Network::TransportSocketOptionsSharedPtr& transport_socket_options)
+      : ConnPoolImpl(dispatcher, host, priority, options, transport_socket_options) {}
 
   // ConnPoolImpl
   CodecClientPtr createCodecClient(Upstream::Host::CreateConnectionData& data) override;

source/common/http/http2/conn_pool.cc

@@ -17,9 +17,10 @@ namespace Http2 {
 
 ConnPoolImpl::ConnPoolImpl(Event::Dispatcher& dispatcher, Upstream::HostConstSharedPtr host,
                            Upstream::ResourcePriority priority,
-                           const Network::ConnectionSocket::OptionsSharedPtr& options)
+                           const Network::ConnectionSocket::OptionsSharedPtr& options,
+                           const Network::TransportSocketOptionsSharedPtr& transport_socket_options)
     : ConnPoolImplBase(std::move(host), std::move(priority)), dispatcher_(dispatcher),
-      socket_options_(options) {}
+      socket_options_(options), transport_socket_options_(transport_socket_options) {}
 
 ConnPoolImpl::~ConnPoolImpl() {
   if (primary_client_) {
@@ -273,8 +274,8 @@ ConnPoolImpl::ActiveClient::ActiveClient(ConnPoolImpl& parent)
       connect_timer_(parent_.dispatcher_.createTimer([this]() -> void { onConnectTimeout(); })) {
   parent_.conn_connect_ms_ = std::make_unique<Stats::Timespan>(
       parent_.host_->cluster().stats().upstream_cx_connect_ms_, parent_.dispatcher_.timeSource());
-  Upstream::Host::CreateConnectionData data =
-      parent_.host_->createConnection(parent_.dispatcher_, parent_.socket_options_, nullptr);
+  Upstream::Host::CreateConnectionData data = parent_.host_->createConnection(
+      parent_.dispatcher_, parent_.socket_options_, parent_.transport_socket_options_);
   real_host_description_ = data.host_description_;
   client_ = parent_.createCodecClient(data);
   client_->addConnectionCallbacks(*this);

source/common/http/http2/conn_pool.h

@@ -26,7 +26,8 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase {
 public:
   ConnPoolImpl(Event::Dispatcher& dispatcher, Upstream::HostConstSharedPtr host,
                Upstream::ResourcePriority priority,
-               const Network::ConnectionSocket::OptionsSharedPtr& options);
+               const Network::ConnectionSocket::OptionsSharedPtr& options,
+               const Network::TransportSocketOptionsSharedPtr& transport_socket_options);
   ~ConnPoolImpl() override;
 
   // Http::ConnectionPool::Instance
@@ -96,6 +97,7 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase {
   ActiveClientPtr draining_client_;
   std::list<DrainedCb> drained_callbacks_;
   const Network::ConnectionSocket::OptionsSharedPtr socket_options_;
+  const Network::TransportSocketOptionsSharedPtr transport_socket_options_;
 };
 
 /**

source/common/network/BUILD

@@ -31,6 +31,16 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "application_protocol_lib",
+    srcs = ["application_protocol.cc"],
+    hdrs = ["application_protocol.h"],
+    deps = [
+        "//include/envoy/stream_info:filter_state_interface",
+        "//source/common/common:macros",
+    ],
+)
+
 envoy_cc_library(
     name = "cidr_range_lib",
     srcs = ["cidr_range.cc"],

source/common/network/application_protocol.cc

@@ -0,0 +1,12 @@
+#include "common/network/application_protocol.h"
+
+#include "common/common/macros.h"
+
+namespace Envoy {
+namespace Network {
+
+const std::string& ApplicationProtocols::key() {
+  CONSTRUCT_ON_FIRST_USE(std::string, "envoy.network.application_protocols");
+}
+} // namespace Network
+} // namespace Envoy

source/common/network/application_protocol.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "envoy/stream_info/filter_state.h"
+
+namespace Envoy {
+namespace Network {
+
+/**
+ * ALPN to set in the upstream connection. Filters can use this one to override the ALPN in TLS
+ * context.
+ */
+class ApplicationProtocols : public StreamInfo::FilterState::Object {
+public:
+  explicit ApplicationProtocols(const std::vector<std::string>& application_protocols)
+      : application_protocols_(application_protocols) {}
+  const std::vector<std::string>& value() const { return application_protocols_; }
+  static const std::string& key();
+
+private:
+  const std::vector<std::string> application_protocols_;
+};
+
+} // namespace Network
+} // namespace Envoy

source/common/network/transport_socket_options_impl.cc

@@ -6,11 +6,15 @@
 namespace Envoy {
 namespace Network {
 void TransportSocketOptionsImpl::hashKey(std::vector<uint8_t>& key) const {
-  if (!override_server_name_.has_value()) {
-    return;
+  if (override_server_name_.has_value()) {
+    pushScalarToByteVector(StringUtil::CaseInsensitiveHash()(override_server_name_.value()), key);
   }
 
-  pushScalarToByteVector(StringUtil::CaseInsensitiveHash()(override_server_name_.value()), key);
+  if (!override_alpn_list_.empty()) {
+    for (const auto& protocol : override_alpn_list_) {
+      pushScalarToByteVector(StringUtil::CaseInsensitiveHash()(protocol), key);
+    }
+  }
 }
 } // namespace Network
 } // namespace Envoy

source/common/network/transport_socket_options_impl.h

@@ -8,11 +8,13 @@ namespace Network {
 class TransportSocketOptionsImpl : public TransportSocketOptions {
 public:
   TransportSocketOptionsImpl(absl::string_view override_server_name = "",
-                             std::vector<std::string>&& override_verify_san_list = {})
+                             std::vector<std::string>&& override_verify_san_list = {},
+                             std::vector<std::string>&& override_alpn = {})
       : override_server_name_(override_server_name.empty()
                                   ? absl::nullopt
                                   : absl::optional<std::string>(override_server_name)),
-        override_verify_san_list_{std::move(override_verify_san_list)} {}
+        override_verify_san_list_{std::move(override_verify_san_list)},
+        override_alpn_list_{std::move(override_alpn)} {}
 
   // Network::TransportSocketOptions
   const absl::optional<std::string>& serverNameOverride() const override {
@@ -21,11 +23,15 @@ class TransportSocketOptionsImpl : public TransportSocketOptions {
   const std::vector<std::string>& verifySubjectAltNameListOverride() const override {
     return override_verify_san_list_;
   }
+  const std::vector<std::string>& applicationProtocolListOverride() const override {
+    return override_alpn_list_;
+  }
   void hashKey(std::vector<uint8_t>& key) const override;
 
 private:
   const absl::optional<std::string> override_server_name_;
   const std::vector<std::string> override_verify_san_list_;
+  const std::vector<std::string> override_alpn_list_;
 };
 
 } // namespace Network

source/common/router/BUILD

@@ -251,6 +251,8 @@ envoy_cc_library(
         "//source/common/http:headers_lib",
         "//source/common/http:message_lib",
         "//source/common/http:utility_lib",
+        "//source/common/network:application_protocol_lib",
+        "//source/common/network:transport_socket_options_lib",
         "//source/common/stream_info:stream_info_lib",
         "//source/common/tracing:http_tracer_lib",
         "//source/common/upstream:load_balancer_lib",

source/common/router/router.cc

@@ -24,6 +24,8 @@
 #include "common/http/headers.h"
 #include "common/http/message_impl.h"
 #include "common/http/utility.h"
+#include "common/network/application_protocol.h"
+#include "common/network/transport_socket_options_impl.h"
 #include "common/router/config_impl.h"
 #include "common/router/debug_config.h"
 #include "common/router/retry_state_impl.h"
@@ -540,6 +542,16 @@ Http::ConnectionPool::Instance* Filter::getConnPool() {
     protocol = (features & Upstream::ClusterInfo::Features::HTTP2) ? Http::Protocol::Http2
                                                                    : Http::Protocol::Http11;
   }
+
+  if (callbacks_->streamInfo().filterState().hasData<Network::ApplicationProtocols>(
+          Network::ApplicationProtocols::key())) {
+    const auto& alpn =
+        callbacks_->streamInfo().filterState().getDataReadOnly<Network::ApplicationProtocols>(
+            Network::ApplicationProtocols::key());
+    transport_socket_options_ = std::make_shared<Network::TransportSocketOptionsImpl>(
+        "", std::vector<std::string>{}, std::vector<std::string>{alpn.value()});
+  }
+
   return config_.cm_.httpConnPoolForCluster(route_entry_->clusterName(), route_entry_->priority(),
                                             protocol, this);
 }

source/common/router/router.h

@@ -325,6 +325,10 @@ class Filter : Logger::Loggable<Logger::Id::router>,
     return callbacks_->getUpstreamSocketOptions();
   }
 
+  Network::TransportSocketOptionsSharedPtr upstreamTransportSocketOptions() const override {
+    return transport_socket_options_;
+  }
+
   /**
    * Set a computed cookie to be sent with the downstream headers.
    * @param key supplies the size of the cookie
@@ -576,6 +580,8 @@ class Filter : Logger::Loggable<Logger::Id::router>,
   bool attempting_internal_redirect_with_complete_stream_ : 1;
   uint32_t attempt_count_{1};
   uint32_t pending_retries_{0};
+
+  Network::TransportSocketOptionsSharedPtr transport_socket_options_;
 };
 
 class ProdFilter : public Filter {

source/common/tcp_proxy/BUILD

@@ -33,6 +33,7 @@ envoy_cc_library(
         "//source/common/common:empty_string",
         "//source/common/common:macros",
         "//source/common/common:minimal_logger_lib",
+        "//source/common/network:application_protocol_lib",
         "//source/common/network:cidr_range_lib",
         "//source/common/network:filter_lib",
         "//source/common/network:transport_socket_options_lib",

source/common/tcp_proxy/tcp_proxy.cc

@@ -19,6 +19,7 @@
 #include "common/common/macros.h"
 #include "common/common/utility.h"
 #include "common/config/well_known_names.h"
+#include "common/network/application_protocol.h"
 #include "common/network/transport_socket_options_impl.h"
 #include "common/network/upstream_server_name.h"
 #include "common/router/metadatamatchcriteria_impl.h"
@@ -368,14 +369,29 @@ Network::FilterStatus Filter::initializeUpstreamConnection() {
     return Network::FilterStatus::StopIteration;
   }
 
-  if (downstreamConnection() &&
-      downstreamConnection()->streamInfo().filterState().hasData<UpstreamServerName>(
-          UpstreamServerName::key())) {
-    const auto& original_requested_server_name =
-        downstreamConnection()->streamInfo().filterState().getDataReadOnly<UpstreamServerName>(
-            UpstreamServerName::key());
+  if (downstreamConnection()) {
+    absl::string_view server_name = "";
+    std::vector<std::string> application_protocols;
+    if (downstreamConnection()->streamInfo().filterState().hasData<UpstreamServerName>(
+            UpstreamServerName::key())) {
+      const auto& original_requested_server_name =
+          downstreamConnection()->streamInfo().filterState().getDataReadOnly<UpstreamServerName>(
+              UpstreamServerName::key());
+      server_name = original_requested_server_name.value();
+    }
+
+    if (downstreamConnection()->streamInfo().filterState().hasData<Network::ApplicationProtocols>(
+            Network::ApplicationProtocols::key())) {
+      const auto& alpn =
+          downstreamConnection()
+              ->streamInfo()
+              .filterState()
+              .getDataReadOnly<Network::ApplicationProtocols>(Network::ApplicationProtocols::key());
+      application_protocols = alpn.value();
+    }
+
     transport_socket_options_ = std::make_shared<Network::TransportSocketOptionsImpl>(
-        original_requested_server_name.value());
+        server_name, std::vector<std::string>{}, std::vector<std::string>{application_protocols});
   }
 
   Tcp::ConnectionPool::Instance* conn_pool = cluster_manager_.tcpConnPoolForCluster(

source/common/upstream/cluster_manager_impl.cc

@@ -1254,6 +1254,12 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::connPool(
     option->hashKey(hash_key);
   }
 
+  bool have_transport_socket_options = false;
+  if (context && context->upstreamTransportSocketOptions()) {
+    context->upstreamTransportSocketOptions()->hashKey(hash_key);
+    have_transport_socket_options = true;
+  }
+
   ConnPoolsContainer& container = *parent_.getHttpConnPoolsContainer(host, true);
 
   // Note: to simplify this, we assume that the factory is only called in the scope of this
@@ -1262,7 +1268,8 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::connPool(
       container.pools_->getPool(priority, hash_key, [&]() {
         return parent_.parent_.factory_.allocateConnPool(
             parent_.thread_local_dispatcher_, host, priority, protocol,
-            !upstream_options->empty() ? upstream_options : nullptr);
+            !upstream_options->empty() ? upstream_options : nullptr,
+            have_transport_socket_options ? context->upstreamTransportSocketOptions() : nullptr);
       });
 
   if (pool.has_value()) {
@@ -1326,14 +1333,15 @@ ClusterManagerPtr ProdClusterManagerFactory::clusterManagerFromProto(
 
 Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool(
     Event::Dispatcher& dispatcher, HostConstSharedPtr host, ResourcePriority priority,
-    Http::Protocol protocol, const Network::ConnectionSocket::OptionsSharedPtr& options) {
+    Http::Protocol protocol, const Network::ConnectionSocket::OptionsSharedPtr& options,
+    const Network::TransportSocketOptionsSharedPtr& transport_socket_options) {
   if (protocol == Http::Protocol::Http2 &&
       runtime_.snapshot().featureEnabled("upstream.use_http2", 100)) {
-    return Http::ConnectionPool::InstancePtr{
-        new Http::Http2::ProdConnPoolImpl(dispatcher, host, priority, options)};
+    return std::make_unique<Http::Http2::ProdConnPoolImpl>(dispatcher, host, priority, options,
+                                                           transport_socket_options);
   } else {
-    return Http::ConnectionPool::InstancePtr{
-        new Http::Http1::ProdConnPoolImpl(dispatcher, host, priority, options)};
+    return std::make_unique<Http::Http1::ProdConnPoolImpl>(dispatcher, host, priority, options,
+                                                           transport_socket_options);
   }
 }
 

source/common/upstream/cluster_manager_impl.h

@@ -56,10 +56,10 @@ class ProdClusterManagerFactory : public ClusterManagerFactory {
   // Upstream::ClusterManagerFactory
   ClusterManagerPtr
   clusterManagerFromProto(const envoy::config::bootstrap::v2::Bootstrap& bootstrap) override;
-  Http::ConnectionPool::InstancePtr
-  allocateConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
-                   ResourcePriority priority, Http::Protocol protocol,
-                   const Network::ConnectionSocket::OptionsSharedPtr& options) override;
+  Http::ConnectionPool::InstancePtr allocateConnPool(
+      Event::Dispatcher& dispatcher, HostConstSharedPtr host, ResourcePriority priority,
+      Http::Protocol protocol, const Network::ConnectionSocket::OptionsSharedPtr& options,
+      const Network::TransportSocketOptionsSharedPtr& transport_socket_options) override;
   Tcp::ConnectionPool::InstancePtr
   allocateTcpConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
                       ResourcePriority priority,