quiche: make flow control configurable (envoyproxy#15865)

Commit Message: Add initial stream and connection level flow control windows to envoy.config.core.v3.QuicProtocolOptions which is be used in QUIC listener config and Http3 upstream cluster config. Risk Level: low Testing: re-enable more Http3 downstream protocol test. Part of envoyproxy#2557 envoyproxy#12930 envoyproxy#14829 Signed-off-by: Dan Zhang <[email protected]> Co-authored-by: Dan Zhang <[email protected]> Signed-off-by: Gokul Nair <[email protected]>
gokulnair · May 5, 2021 · f8bb0eb · f8bb0eb
1 parent 1fd182e
commit f8bb0eb
Show file tree

Hide file tree

Showing 31 changed files with 326 additions and 64 deletions.
diff --git a/api/envoy/config/core/v3/protocol.proto b/api/envoy/config/core/v3/protocol.proto
@@ -31,6 +31,28 @@ message QuicProtocolOptions {
   // Maximum number of streams that the client can negotiate per connection. 100
   // if not specified.
   google.protobuf.UInt32Value max_concurrent_streams = 1;
+
+  // `Initial stream-level flow-control receive window
+  // <https://tools.ietf.org/html/draft-ietf-quic-transport-34#section-4.1>`_ size. Valid values range from
+  // 1 to 16777216 (2^24, maximum supported by QUICHE) and defaults to 65536 (2^16).
+  //
+  // NOTE: 16384 (2^14) is the minimum window size supported in Google QUIC. If configured smaller than it, we will use 16384 instead.
+  // QUICHE IETF Quic implementation supports 1 bytes window. We only support increasing the default window size now, so it's also the minimum.
+  //
+  // This field also acts as a soft limit on the number of bytes Envoy will buffer per-stream in the
+  // QUIC stream send and receive buffers. Once the buffer reaches this pointer, watermark callbacks will fire to
+  // stop the flow of data to the stream buffers.
+  google.protobuf.UInt32Value initial_stream_window_size = 2
+      [(validate.rules).uint32 = {lte: 16777216 gte: 1}];
+
+  // Similar to *initial_stream_window_size*, but for connection-level
+  // flow-control. Valid values rage from 1 to 25165824 (24MB, maximum supported by QUICHE) and defaults to 65536 (2^16).
+  // window. Currently, this has the same minimum/default as *initial_stream_window_size*.
+  //
+  // NOTE: 16384 (2^14) is the minimum window size supported in Google QUIC. We only support increasing the default
+  // window size now, so it's also the minimum.
+  google.protobuf.UInt32Value initial_connection_window_size = 3
+      [(validate.rules).uint32 = {lte: 25165824 gte: 1}];
 }
 
 message UpstreamHttpProtocolOptions {

diff --git a/api/envoy/config/core/v4alpha/protocol.proto b/api/envoy/config/core/v4alpha/protocol.proto
diff --git a/docs/protodoc_manifest.yaml b/docs/protodoc_manifest.yaml
@@ -52,3 +52,8 @@ fields:
       example: 300s # 5 mins
   envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.use_remote_address:
     edge_config: { example: true }
+  envoy.config.listener.v3.QuicProtocolOptions.quic_protocol_options:
+    edge_config:
+      example:
+        initial_stream_window_size: 65536 # 64 KiB
+        initial_connection_window_size: 65536 # 64 KiB
diff --git a/generated_api_shadow/envoy/config/core/v3/protocol.proto b/generated_api_shadow/envoy/config/core/v3/protocol.proto
diff --git a/generated_api_shadow/envoy/config/core/v4alpha/protocol.proto b/generated_api_shadow/envoy/config/core/v4alpha/protocol.proto
diff --git a/source/common/http/http3/codec_stats.h b/source/common/http/http3/codec_stats.h
@@ -21,6 +21,7 @@ namespace Http3 {
   COUNTER(rx_reset)                                                                                \
   COUNTER(trailers)                                                                                \
   COUNTER(tx_reset)                                                                                \
+  COUNTER(metadata_not_supported_error)                                                            \
   GAUGE(streams_active, Accumulate)
 
 /**

diff --git a/source/common/http/http3/conn_pool.cc b/source/common/http/http3/conn_pool.cc
@@ -14,6 +14,7 @@
 
 #ifdef ENVOY_ENABLE_QUIC
 #include "common/quic/client_connection_factory_impl.h"
+#include "common/quic/envoy_quic_utils.h"
 #else
 #error "http3 conn pool should not be built with QUIC disabled"
 #endif
@@ -44,14 +45,16 @@ class Http3ConnPoolImpl : public FixedHttpConnPoolImpl {
     quic_info_ = std::make_unique<Quic::PersistentQuicInfoImpl>(
         dispatcher, transport_socket_factory, host->cluster().statsScope(), time_source,
         source_address);
+    Quic::configQuicInitialFlowControlWindow(
+        host_->cluster().http3Options().quic_protocol_options(), quic_info_->quic_config_);
   }
 
   // Make sure all connections are torn down before quic_info_ is deleted.
   ~Http3ConnPoolImpl() override { destructAllConnections(); }
 
   // Store quic helpers which can be shared between connections and must live
   // beyond the lifetime of individual connections.
-  std::unique_ptr<PersistentQuicInfo> quic_info_;
+  std::unique_ptr<Quic::PersistentQuicInfoImpl> quic_info_;
 };
 
 ConnectionPool::InstancePtr

diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc
@@ -199,8 +199,13 @@ initializeAndValidateOptions(const envoy::config::core::v3::Http2ProtocolOptions
 
 } // namespace Utility
 } // namespace Http2
+
 namespace Http3 {
 namespace Utility {
+
+const uint32_t OptionsLimits::DEFAULT_INITIAL_STREAM_WINDOW_SIZE;
+const uint32_t OptionsLimits::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE;
+
 envoy::config::core::v3::Http3ProtocolOptions
 initializeAndValidateOptions(const envoy::config::core::v3::Http3ProtocolOptions& options,
                              bool hcm_stream_error_set,

diff --git a/source/common/http/utility.h b/source/common/http/utility.h
@@ -118,11 +118,19 @@ initializeAndValidateOptions(const envoy::config::core::v3::Http2ProtocolOptions
 } // namespace Http2
 namespace Http3 {
 namespace Utility {
+
+// Limits and defaults for `envoy::config::core::v3::Http3ProtocolOptions` protos.
+struct OptionsLimits {
+  // The same as kStreamReceiveWindowLimit in QUICHE which is the maximum supported by QUICHE.
+  static const uint32_t DEFAULT_INITIAL_STREAM_WINDOW_SIZE = 16 * 1024 * 1024;
+  // The same as kSessionReceiveWindowLimit in QUICHE which is the maximum supported by QUICHE.
+  static const uint32_t DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE = 24 * 1024 * 1024;
+};
+
 envoy::config::core::v3::Http3ProtocolOptions
 initializeAndValidateOptions(const envoy::config::core::v3::Http3ProtocolOptions& options,
                              bool hcm_stream_error_set,
                              const Protobuf::BoolValue& hcm_stream_error);
-
 } // namespace Utility
 } // namespace Http3
 namespace Http {

diff --git a/source/common/quic/BUILD b/source/common/quic/BUILD
@@ -359,8 +359,10 @@ envoy_cc_library(
         "//source/common/network:socket_option_factory_lib",
         "//source/common/quic:quic_io_handle_wrapper_lib",
         "//source/extensions/transport_sockets:well_known_names",
+        "@com_googlesource_quiche//:quic_core_config_lib",
         "@com_googlesource_quiche//:quic_core_http_header_list_lib",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+        "@envoy_api//envoy/config/listener/v3:pkg_cc_proto",
     ],
 )
 

diff --git a/source/common/quic/active_quic_listener.cc b/source/common/quic/active_quic_listener.cc
@@ -232,6 +232,7 @@ ActiveQuicListenerFactory::ActiveQuicListenerFactory(
       PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.quic_protocol_options(), max_concurrent_streams, 100);
   quic_config_.SetMaxBidirectionalStreamsToSend(max_streams);
   quic_config_.SetMaxUnidirectionalStreamsToSend(max_streams);
+  configQuicInitialFlowControlWindow(config.quic_protocol_options(), quic_config_);
 }
 
 Network::ConnectionHandler::ActiveUdpListenerPtr ActiveQuicListenerFactory::createActiveUdpListener(

diff --git a/source/common/quic/client_connection_factory_impl.cc b/source/common/quic/client_connection_factory_impl.cc
@@ -31,7 +31,6 @@ namespace {
 // This was preexisting but should either be removed or potentially moved inside
 // PersistentQuicInfoImpl.
 struct StaticInfo {
-  quic::QuicConfig quic_config_;
   quic::QuicClientPushPromiseIndex push_promise_index_;
 
   static StaticInfo& get() { MUTABLE_CONSTRUCT_ON_FIRST_USE(StaticInfo); }
@@ -51,10 +50,15 @@ createQuicNetworkConnection(Http::PersistentQuicInfo& info, Event::Dispatcher& d
       info_impl->alarm_factory_, quic::ParsedQuicVersionVector{info_impl->supported_versions_[0]},
       local_addr, dispatcher, nullptr);
   auto& static_info = StaticInfo::get();
+
+  ASSERT(!info_impl->supported_versions_.empty());
+  // QUICHE client session always use the 1st version to start handshake.
+  // TODO(alyssawilk) pass in ClusterInfo::perConnectionBufferLimitBytes() for
+  // send_buffer_limit instead of using 0.
   auto ret = std::make_unique<EnvoyQuicClientSession>(
-      static_info.quic_config_, info_impl->supported_versions_, std::move(connection),
+      info_impl->quic_config_, info_impl->supported_versions_, std::move(connection),
       info_impl->server_id_, info_impl->crypto_config_.get(), &static_info.push_promise_index_,
-      dispatcher, 0);
+      dispatcher, /*send_buffer_limit=*/0);
   return ret;
 }
 

diff --git a/source/common/quic/client_connection_factory_impl.h b/source/common/quic/client_connection_factory_impl.h
@@ -28,7 +28,10 @@ struct PersistentQuicInfoImpl : public Http::PersistentQuicInfo {
   // given connection pool.
   quic::QuicServerId server_id_;
   quic::ParsedQuicVersionVector supported_versions_{quic::CurrentSupportedVersions()};
+  // TODO(danzh) move this into client transport socket factory so that it can
+  // be updated with SDS.
   std::unique_ptr<quic::QuicCryptoClientConfig> crypto_config_;
+  quic::QuicConfig quic_config_;
 };
 
 std::unique_ptr<Network::ClientConnection>

diff --git a/source/common/quic/envoy_quic_client_stream.cc b/source/common/quic/envoy_quic_client_stream.cc
@@ -105,8 +105,9 @@ void EnvoyQuicClientStream::encodeTrailers(const Http::RequestTrailerMap& traile
 }
 
 void EnvoyQuicClientStream::encodeMetadata(const Http::MetadataMapVector& /*metadata_map_vector*/) {
-  // Metadata Frame is not supported in QUIC.
-  // TODO(danzh): add stats for metadata not supported error.
+  // Metadata Frame is not supported in QUICHE.
+  ENVOY_STREAM_LOG(debug, "METADATA is not supported in Http3.", *this);
+  stats_.metadata_not_supported_error_.inc();
 }
 
 void EnvoyQuicClientStream::resetStream(Http::StreamResetReason reason) {

diff --git a/source/common/quic/envoy_quic_dispatcher.cc b/source/common/quic/envoy_quic_dispatcher.cc
@@ -52,12 +52,6 @@ std::unique_ptr<quic::QuicSession> EnvoyQuicDispatcher::CreateQuicSession(
     const quic::QuicSocketAddress& peer_address, absl::string_view alpn,
     const quic::ParsedQuicVersion& version, absl::string_view sni) {
   quic::QuicConfig quic_config = config();
-  // TODO(danzh) setup flow control window via config.
-  quic_config.SetInitialStreamFlowControlWindowToSend(
-      Http2::Utility::OptionsLimits::MIN_INITIAL_STREAM_WINDOW_SIZE);
-  quic_config.SetInitialSessionFlowControlWindowToSend(
-      1.5 * Http2::Utility::OptionsLimits::MIN_INITIAL_STREAM_WINDOW_SIZE);
-
   Network::ConnectionSocketPtr connection_socket = createServerConnectionSocket(
       listen_socket_.ioHandle(), self_address, peer_address, std::string(sni), alpn);
   const Network::FilterChain* filter_chain =

diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc
@@ -113,7 +113,8 @@ void EnvoyQuicServerStream::encodeTrailers(const Http::ResponseTrailerMap& trail
 
 void EnvoyQuicServerStream::encodeMetadata(const Http::MetadataMapVector& /*metadata_map_vector*/) {
   // Metadata Frame is not supported in QUIC.
-  // TODO(danzh): add stats for metadata not supported error.
+  ENVOY_STREAM_LOG(debug, "METADATA is not supported in Http3.", *this);
+  stats_.metadata_not_supported_error_.inc();
 }
 
 void EnvoyQuicServerStream::resetStream(Http::StreamResetReason reason) {

diff --git a/source/common/quic/envoy_quic_utils.cc b/source/common/quic/envoy_quic_utils.cc
@@ -5,6 +5,7 @@
 #include "envoy/common/platform.h"
 #include "envoy/config/core/v3/base.pb.h"
 
+#include "common/http/utility.h"
 #include "common/network/socket_option_factory.h"
 #include "common/network/utility.h"
 
@@ -243,5 +244,31 @@ createServerConnectionSocket(Network::IoHandle& io_handle,
   return connection_socket;
 }
 
+void configQuicInitialFlowControlWindow(const envoy::config::core::v3::QuicProtocolOptions& config,
+                                        quic::QuicConfig& quic_config) {
+  size_t stream_flow_control_window_to_send = PROTOBUF_GET_WRAPPED_OR_DEFAULT(
+      config, initial_stream_window_size,
+      Http3::Utility::OptionsLimits::DEFAULT_INITIAL_STREAM_WINDOW_SIZE);
+  if (stream_flow_control_window_to_send < quic::kMinimumFlowControlSendWindow) {
+    // If the configured value is smaller than 16kB, only use it for IETF QUIC, because Google QUIC
+    // requires minimum 16kB stream flow control window. The QUICHE default 16kB will be used for
+    // Google QUIC connections.
+    quic_config.SetInitialMaxStreamDataBytesIncomingBidirectionalToSend(
+        stream_flow_control_window_to_send);
+  } else {
+    // Both Google QUIC and IETF Quic can be configured from this.
+    quic_config.SetInitialStreamFlowControlWindowToSend(stream_flow_control_window_to_send);
+  }
+
+  uint32_t session_flow_control_window_to_send = PROTOBUF_GET_WRAPPED_OR_DEFAULT(
+      config, initial_connection_window_size,
+      Http3::Utility::OptionsLimits::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE);
+  // Config connection level flow control window shouldn't be smaller than the minimum flow control
+  // window supported in QUICHE which is 16kB.
+  quic_config.SetInitialSessionFlowControlWindowToSend(
+      std::max(quic::kMinimumFlowControlSendWindow,
+               static_cast<quic::QuicByteCount>(session_flow_control_window_to_send)));
+}
+
 } // namespace Quic
 } // namespace Envoy
diff --git a/source/common/quic/envoy_quic_utils.h b/source/common/quic/envoy_quic_utils.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "envoy/common/platform.h"
+#include "envoy/config/listener/v3/quic_config.pb.h"
 #include "envoy/http/codec.h"
 
 #include "common/common/assert.h"
@@ -16,6 +17,7 @@
 #endif
 
 #include "quiche/quic/core/quic_types.h"
+#include "quiche/quic/core/quic_config.h"
 
 #if defined(__GNUC__)
 #pragma GCC diagnostic pop
@@ -162,5 +164,9 @@ createServerConnectionSocket(Network::IoHandle& io_handle,
                              const quic::QuicSocketAddress& peer_address,
                              const std::string& hostname, absl::string_view alpn);
 
+// Set initial flow control windows in quic_config according to the given Envoy config.
+void configQuicInitialFlowControlWindow(const envoy::config::core::v3::QuicProtocolOptions& config,
+                                        quic::QuicConfig& quic_config);
+
 } // namespace Quic
 } // namespace Envoy
diff --git a/test/common/network/BUILD b/test/common/network/BUILD
@@ -291,7 +291,10 @@ envoy_cc_test(
     name = "udp_listener_impl_batch_writer_test",
     srcs = ["udp_listener_impl_batch_writer_test.cc"],
     # Skipping as quiche quic_gso_batch_writer.h does not exist on Windows
-    tags = ["skip_on_windows"],
+    tags = [
+        "nofips",
+        "skip_on_windows",
+    ],
     deps = [
         ":udp_listener_impl_test_base_lib",
         "//source/common/event:dispatcher_lib",

diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD
@@ -169,6 +169,7 @@ envoy_cc_test(
     deps = [
         ":quic_test_utils_for_envoy_lib",
         ":test_utils_lib",
+        "//source/common/http:utility_lib",
         "//source/common/network:udp_packet_writer_handler_lib",
         "//source/common/quic:active_quic_listener_lib",
         "//source/common/quic:envoy_quic_utils_lib",